added IMAP, LetsEncrypt, update layout

This commit is contained in:
2026-08-12 21:14:19 +01:00
parent 6e103959b0
commit 70fa1a5f2c
222 changed files with 42947 additions and 14038 deletions
+122
View File
@@ -0,0 +1,122 @@
// Package mailstore handles local mailbox storage: per-mailbox encryption at rest
// (a random AES-256 key per mailbox, sealed with one server-held master key so a raw
// DB/backup theft alone can't decrypt mail — see MasterKey below), on-disk ciphertext
// layout, and quota accounting. Message retrieval/IMAP serving is a later milestone;
// this package is the storage engine underneath it.
package mailstore
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"mailgoserver/internal/db"
)
// masterKeySize is 32 bytes (AES-256).
const masterKeySize = 32
// LoadOrCreateMasterKey reads the server's master encryption key from path, generating
// a fresh random one on first run if the file doesn't exist yet — mirrors
// tlsutil.GenerateSelfSignedCert's generate-if-missing pattern. This file must be
// backed up separately from the database: losing it makes every stored mailbox's mail
// permanently unrecoverable, even for admins.
func LoadOrCreateMasterKey(path string) ([]byte, error) {
if b, err := os.ReadFile(path); err == nil {
if len(b) != masterKeySize {
return nil, fmt.Errorf("master key at %s is %d bytes, want %d", path, len(b), masterKeySize)
}
return b, nil
} else if !os.IsNotExist(err) {
return nil, err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, err
}
key := make([]byte, masterKeySize)
if _, err := rand.Read(key); err != nil {
return nil, err
}
if err := os.WriteFile(path, key, 0o600); err != nil {
return nil, err
}
return key, nil
}
// Store is the local mailbox storage engine: one per running server, shared across
// connections (analogous to smtpserver.Backend).
type Store struct {
DB *db.DB
MasterKey []byte
BasePath string
}
func New(database *db.DB, masterKey []byte, basePath string) *Store {
return &Store{DB: database, MasterKey: masterKey, BasePath: basePath}
}
// GenerateDEK returns a fresh random AES-256 data encryption key for one mailbox.
func GenerateDEK() []byte {
dek := make([]byte, masterKeySize)
rand.Read(dek)
return dek
}
// WrapDEK seals dek with the server master key, returning the ciphertext and the
// nonce used for that one seal operation (both stored on the mailbox row).
func (s *Store) WrapDEK(dek []byte) (wrapped, nonce []byte, err error) {
return sealAESGCM(s.MasterKey, dek)
}
// UnwrapDEK reverses WrapDEK.
func (s *Store) UnwrapDEK(wrapped, nonce []byte) ([]byte, error) {
return openAESGCM(s.MasterKey, wrapped, nonce)
}
func sealAESGCM(key, plaintext []byte) (ciphertext, nonce []byte, err error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, nil, err
}
nonce = make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return nil, nil, err
}
return gcm.Seal(nil, nonce, plaintext, nil), nonce, nil
}
func openAESGCM(key, ciphertext, nonce []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
if len(nonce) != gcm.NonceSize() {
return nil, errors.New("mailstore: invalid nonce size")
}
return gcm.Open(nil, nonce, ciphertext, nil)
}
// sanitizePathSegment neuters filesystem-unsafe characters in a mailbox email address
// so it can be used directly as a directory name, mirroring
// smtpserver.sanitizePathSegment's spirit (that one only handles domains; this one
// also strips "@" and ":" since a full address is used here, not just a domain).
func sanitizePathSegment(s string) string {
for _, c := range []string{"/", "\\", ":", "@"} {
s = strings.ReplaceAll(s, c, "_")
}
return s
}
+160
View File
@@ -0,0 +1,160 @@
package mailstore
import (
"bytes"
"os"
"path/filepath"
"testing"
"mailgoserver/internal/db"
)
func newTestDB(t *testing.T) *db.DB {
t.Helper()
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
database, err := db.Open(dbPath)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { database.Close() })
return database
}
// newTestMailbox creates a domain + mailbox with a real wrapped DEK, using a Store
// built against a random master key, and returns both.
func newTestMailbox(t *testing.T, quotaBytes int64) (*Store, int64) {
t.Helper()
database := newTestDB(t)
domainID, err := database.CreateDomain("example.com")
if err != nil {
t.Fatal(err)
}
masterKey := GenerateDEK() // 32 random bytes, reused here as a throwaway master key
s := New(database, masterKey, t.TempDir())
dek := GenerateDEK()
wrapped, nonce, err := s.WrapDEK(dek)
if err != nil {
t.Fatal(err)
}
hash, err := db.HashPassword("irrelevant-portal-password")
if err != nil {
t.Fatal(err)
}
mailboxID, err := database.CreateMailbox("user@example.com", hash, domainID, quotaBytes, wrapped, nonce)
if err != nil {
t.Fatal(err)
}
return s, mailboxID
}
func TestWrapUnwrapDEK(t *testing.T) {
database := newTestDB(t)
s := New(database, GenerateDEK(), t.TempDir())
dek := GenerateDEK()
wrapped, nonce, err := s.WrapDEK(dek)
if err != nil {
t.Fatal(err)
}
got, err := s.UnwrapDEK(wrapped, nonce)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(dek, got) {
t.Fatalf("unwrapped DEK does not match original: got %x, want %x", got, dek)
}
}
func TestStoreFetchRoundTrip(t *testing.T) {
s, mailboxID := newTestMailbox(t, 1024*1024)
raw := []byte("From: a@example.com\r\nSubject: hi\r\n\r\nhello world")
uid, err := s.StoreMessage(mailboxID, "INBOX", raw, "<abc@example.com>", "a@example.com", "hi")
if err != nil {
t.Fatal(err)
}
got, err := s.FetchMessage(mailboxID, uid)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(raw, got) {
t.Fatalf("fetched message does not match stored: got %q, want %q", got, raw)
}
msg, err := s.DB.GetMessageByUID(mailboxID, uid)
if err != nil {
t.Fatal(err)
}
onDisk, err := os.ReadFile(msg.StoragePath)
if err != nil {
t.Fatal(err)
}
if bytes.Equal(onDisk, raw) {
t.Fatal("on-disk file matches plaintext — message was not actually encrypted")
}
mbox, err := s.DB.GetMailboxByID(mailboxID)
if err != nil {
t.Fatal(err)
}
if mbox.UsedBytes != int64(len(raw)) {
t.Fatalf("used_bytes = %d, want %d", mbox.UsedBytes, len(raw))
}
}
func TestQuotaExceeded(t *testing.T) {
s, mailboxID := newTestMailbox(t, 10) // tiny quota
raw := []byte("this message is definitely longer than ten bytes")
_, err := s.StoreMessage(mailboxID, "INBOX", raw, "<abc@example.com>", "a@example.com", "hi")
if err != ErrQuotaExceeded {
t.Fatalf("err = %v, want ErrQuotaExceeded", err)
}
mbox, err := s.DB.GetMailboxByID(mailboxID)
if err != nil {
t.Fatal(err)
}
if mbox.UsedBytes != 0 {
t.Fatalf("used_bytes = %d after a rejected store, want 0", mbox.UsedBytes)
}
entries, err := os.ReadDir(s.BasePath)
if err == nil && len(entries) != 0 {
t.Fatalf("expected no files written under %s after a rejected store, found %d entries", s.BasePath, len(entries))
}
}
func TestDeleteMessageFreesQuota(t *testing.T) {
s, mailboxID := newTestMailbox(t, 1024*1024)
raw := []byte("From: a@example.com\r\nSubject: bye\r\n\r\ngoodbye")
uid, err := s.StoreMessage(mailboxID, "INBOX", raw, "<def@example.com>", "a@example.com", "bye")
if err != nil {
t.Fatal(err)
}
msg, err := s.DB.GetMessageByUID(mailboxID, uid)
if err != nil {
t.Fatal(err)
}
storagePath := msg.StoragePath
if err := s.DeleteMessage(mailboxID, uid); err != nil {
t.Fatal(err)
}
mbox, err := s.DB.GetMailboxByID(mailboxID)
if err != nil {
t.Fatal(err)
}
if mbox.UsedBytes != 0 {
t.Fatalf("used_bytes = %d after delete, want 0", mbox.UsedBytes)
}
if _, err := os.Stat(storagePath); !os.IsNotExist(err) {
t.Fatalf("ciphertext file %s still exists after delete", storagePath)
}
}
+19
View File
@@ -0,0 +1,19 @@
package mailstore
import "fmt"
// QuotaStatus reports a mailbox's storage usage — feeds the "≥90% full" badge on the
// mailboxes list and the dashboard tile in a later milestone.
func (s *Store) QuotaStatus(mailboxID int64) (used, quota int64, pctFull float64, err error) {
mbox, err := s.DB.GetMailboxByID(mailboxID)
if err != nil {
return 0, 0, 0, err
}
if mbox == nil {
return 0, 0, 0, fmt.Errorf("mailstore: mailbox %d not found", mailboxID)
}
if mbox.QuotaBytes == 0 {
return mbox.UsedBytes, 0, 0, nil
}
return mbox.UsedBytes, mbox.QuotaBytes, float64(mbox.UsedBytes) / float64(mbox.QuotaBytes) * 100, nil
}
+17
View File
@@ -0,0 +1,17 @@
package mailstore
import "mailgoserver/internal/db"
// ResolveRecipient looks up a local mailbox for addr — its primary email first, then
// any active alias — so mail sent to an alias lands in the owning mailbox's INBOX.
func (s *Store) ResolveRecipient(addr string) (*db.Mailbox, error) {
mbox, err := s.DB.GetMailboxByEmail(addr)
if err != nil || mbox != nil {
return mbox, err
}
alias, err := s.DB.GetAliasByEmail(addr)
if err != nil || alias == nil {
return nil, err
}
return s.DB.GetMailboxByID(alias.MailboxID)
}
+35
View File
@@ -0,0 +1,35 @@
package mailstore
import (
"bytes"
"encoding/json"
"net/http"
"time"
)
// CheckRspamd sends a message to an optional rspamd instance for scoring, only called
// when [Rspamd] enabled=true — the built-in SpamScore heuristic (spam.go) always runs
// regardless, so this is additive, not a replacement.
func CheckRspamd(url string, raw []byte, mailFrom, rcptTo string) (score float64, action string, err error) {
req, err := http.NewRequest(http.MethodPost, url+"/checkv2", bytes.NewReader(raw))
if err != nil {
return 0, "", err
}
req.Header.Set("From", mailFrom)
req.Header.Set("Rcpt", rcptTo)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
return 0, "", err
}
defer resp.Body.Close()
var result struct {
Score float64 `json:"score"`
Action string `json:"action"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return 0, "", err
}
return result.Score, result.Action, nil
}
+54
View File
@@ -0,0 +1,54 @@
package mailstore
import "strings"
// FilterAction is the outcome of evaluating a mailbox's filter rules against one
// incoming message.
type FilterAction struct {
Folder string // non-empty: store here instead of INBOX
MarkRead bool
Drop bool // don't store at all
}
// ApplyRules evaluates a mailbox's filter rules in priority order and returns the
// first match's action (zero value if none match, meaning "store in INBOX, unread").
// headers should have "from"/"to"/"subject" keys — rules run at delivery time, before
// the message is encrypted and stored, so real header values are available, not just
// the plaintext cache columns used for fast IMAP listing.
func (s *Store) ApplyRules(mailboxID int64, headers map[string]string) (FilterAction, error) {
rules, err := s.DB.ListRulesForMailbox(mailboxID)
if err != nil {
return FilterAction{}, err
}
for _, r := range rules {
if !r.IsActive {
continue
}
if !matchCondition(r.ConditionOp, headers[r.ConditionField], r.ConditionValue) {
continue
}
switch r.Action {
case "move_to_folder":
return FilterAction{Folder: r.ActionValue}, nil
case "delete":
return FilterAction{Drop: true}, nil
case "mark_read":
return FilterAction{MarkRead: true}, nil
}
}
return FilterAction{}, nil
}
func matchCondition(op, value, target string) bool {
value = strings.ToLower(value)
target = strings.ToLower(target)
switch op {
case "contains":
return strings.Contains(value, target)
case "equals":
return value == target
case "starts_with":
return strings.HasPrefix(value, target)
}
return false
}
+58
View File
@@ -0,0 +1,58 @@
package mailstore
import (
"context"
"net"
"strings"
)
// spamKeywords is a tiny, obvious-spam subject keyword list — a coarse signal only.
var spamKeywords = []string{"viagra", "casino", "lottery winner", "click here now", "wire transfer urgent", "nigerian prince"}
// SpamScore is a lightweight built-in heuristic — always runs, regardless of whether
// rspamd (rspamd.go) is also enabled; both are additive, not either/or. Higher is more
// suspicious; compare against [Mailstore] spam_reject_score.
// ponytail: naive keyword/weight heuristic, not a real Bayesian/ML scorer — upgrade or
// lean harder on rspamd if false-positive rate matters.
func SpamScore(peerIP string, headers map[string]string, dkimPass, spfPass bool) int {
score := 0
if !spfPass {
score += 2
}
if !dkimPass {
score++
}
if CheckDNSBL(peerIP) {
score += 5
}
subject := strings.ToLower(headers["subject"])
for _, kw := range spamKeywords {
if strings.Contains(subject, kw) {
score++
}
}
return score
}
// CheckDNSBL looks up peerIP against the Spamhaus ZEN DNSBL. Per RFC 5782, a listing
// response is always an A record in 127.0.0.0/8 — checking for that range (rather than
// "any resolution succeeded") avoids false positives from a resolver that hijacks
// NXDOMAIN into a search/ad page instead of returning an error.
func CheckDNSBL(peerIP string) bool {
ip := net.ParseIP(peerIP)
if ip == nil || ip.To4() == nil {
return false
}
octets := strings.Split(ip.To4().String(), ".")
reversed := octets[3] + "." + octets[2] + "." + octets[1] + "." + octets[0]
addrs, err := net.DefaultResolver.LookupHost(context.Background(), reversed+".zen.spamhaus.org")
if err != nil {
return false
}
for _, a := range addrs {
if resolved := net.ParseIP(a); resolved != nil && resolved.To4() != nil && resolved.To4()[0] == 127 {
return true
}
}
return false
}
+52
View File
@@ -0,0 +1,52 @@
package mailstore
import (
"net"
"testing"
)
// TestSpamScoreArithmetic exercises SpamScore's own weighting logic with synthetic
// dkimPass/spfPass inputs — CheckDNSBL still runs (it does live DNS), so this only
// asserts the score is monotonically at least as high when signals get worse, rather
// than pinning an exact network-dependent number.
func TestSpamScoreArithmetic(t *testing.T) {
clean := SpamScore("203.0.113.1", map[string]string{"subject": "hello"}, true, true)
noDKIM := SpamScore("203.0.113.1", map[string]string{"subject": "hello"}, false, true)
noSPF := SpamScore("203.0.113.1", map[string]string{"subject": "hello"}, true, false)
keyword := SpamScore("203.0.113.1", map[string]string{"subject": "WIN THE LOTTERY WINNER NOW"}, true, true)
if noDKIM <= clean {
t.Fatalf("missing DKIM should raise the score: clean=%d noDKIM=%d", clean, noDKIM)
}
if noSPF <= clean {
t.Fatalf("failing SPF should raise the score: clean=%d noSPF=%d", clean, noSPF)
}
if keyword <= clean {
t.Fatalf("a spam keyword in the subject should raise the score: clean=%d keyword=%d", clean, keyword)
}
}
func TestEvalSPF(t *testing.T) {
ip := net.ParseIP("203.0.113.10")
other := net.ParseIP("198.51.100.5")
tests := []struct {
name string
record string
ip net.IP
want bool
}{
{"ip4 match passes", "v=spf1 ip4:203.0.113.0/24 -all", ip, true},
{"ip4 no match hard fails", "v=spf1 ip4:203.0.113.0/24 -all", other, false},
{"no all and no match is neutral", "v=spf1 ip4:203.0.113.0/24", other, true},
{"soft fail all is neutral for unmatched ip", "v=spf1 ip4:203.0.113.0/24 ~all", other, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := evalSPF(tt.record, tt.ip, "example.com", 0)
if got != tt.want {
t.Fatalf("evalSPF(%q) = %v, want %v", tt.record, got, tt.want)
}
})
}
}
+130
View File
@@ -0,0 +1,130 @@
package mailstore
import (
"context"
"net"
"strings"
)
// CheckSPF is a minimal, single-level SPF check (v=spf1 ip4:/a/mx/include:, no
// recursive include/redirect, no macro expansion) against the sender domain's TXT
// record — one signal feeding the spam heuristic in spam.go, not an authoritative
// pass/fail gate.
// ponytail: not full RFC 7208 (no multi-level includes, no redirect, no macros) —
// good enough as a signal, revisit if a real sender's SPF record depends on it.
func CheckSPF(mailFrom, peerIP string) bool {
domain := domainOf(mailFrom)
if domain == "" {
return true
}
ip := net.ParseIP(peerIP)
if ip == nil {
return true
}
record, ok := lookupSPFRecord(domain)
if !ok {
return true // no SPF record published: neutral, not a penalty
}
return evalSPF(record, ip, domain, 0)
}
func domainOf(address string) string {
i := strings.LastIndex(address, "@")
if i < 0 {
return ""
}
return strings.ToLower(address[i+1:])
}
func lookupSPFRecord(domain string) (string, bool) {
txts, err := net.DefaultResolver.LookupTXT(context.Background(), domain)
if err != nil {
return "", false
}
for _, t := range txts {
if strings.HasPrefix(strings.ToLower(t), "v=spf1") {
return t, true
}
}
return "", false
}
// evalSPF walks mechanisms left to right; depth caps includes at one level.
func evalSPF(record string, ip net.IP, domain string, depth int) bool {
fields := strings.Fields(record)
for _, f := range fields[1:] { // skip "v=spf1"
qualifier := byte('+')
mech := f
if len(f) > 0 && strings.ContainsRune("+-~?", rune(f[0])) {
qualifier = f[0]
mech = f[1:]
}
switch {
case mech == "all":
return qualifier != '-'
case strings.HasPrefix(mech, "ip4:"):
if matchIP4(mech[4:], ip) {
return qualifier != '-'
}
case mech == "a":
if matchA(domain, ip) {
return qualifier != '-'
}
case strings.HasPrefix(mech, "a:"):
if matchA(mech[2:], ip) {
return qualifier != '-'
}
case mech == "mx":
if matchMX(domain, ip) {
return qualifier != '-'
}
case strings.HasPrefix(mech, "mx:"):
if matchMX(mech[3:], ip) {
return qualifier != '-'
}
case strings.HasPrefix(mech, "include:") && depth == 0:
sub, ok := lookupSPFRecord(mech[len("include:"):])
if ok && evalSPF(sub, ip, mech[len("include:"):], depth+1) {
return true
}
}
}
return true // no matching mechanism and no explicit "all": neutral
}
func matchIP4(cidr string, ip net.IP) bool {
if !strings.Contains(cidr, "/") {
cidr += "/32"
}
_, network, err := net.ParseCIDR(cidr)
if err != nil {
return false
}
return network.Contains(ip)
}
func matchA(host string, ip net.IP) bool {
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip4", host)
if err != nil {
return false
}
for _, a := range ips {
if a.Equal(ip) {
return true
}
}
return false
}
func matchMX(domain string, ip net.IP) bool {
mxs, err := net.LookupMX(domain)
if err != nil {
return false
}
for _, mx := range mxs {
if matchA(strings.TrimSuffix(mx.Host, "."), ip) {
return true
}
}
return false
}
+109
View File
@@ -0,0 +1,109 @@
package mailstore
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"time"
)
// ErrQuotaExceeded is returned by StoreMessage when storing raw would push the
// mailbox over its quota. No row, file, or used_bytes change occurs in that case.
var ErrQuotaExceeded = errors.New("mailstore: mailbox quota exceeded")
// StoreMessage encrypts raw with the mailbox's own data encryption key and persists
// it to disk, then indexes it in esrv_mailbox_messages and updates the mailbox's
// cached used_bytes. from/subject are cached in the DB in plain text by design (see
// schema.go) so IMAP LIST/basic SEARCH don't need to decrypt every message.
func (s *Store) StoreMessage(mailboxID int64, folder string, raw []byte, messageIDHeader, from, subject string) (uid int64, err error) {
mbox, err := s.DB.GetMailboxByID(mailboxID)
if err != nil {
return 0, err
}
if mbox == nil {
return 0, fmt.Errorf("mailstore: mailbox %d not found", mailboxID)
}
if mbox.UsedBytes+int64(len(raw)) > mbox.QuotaBytes {
return 0, ErrQuotaExceeded
}
dek, err := s.UnwrapDEK(mbox.DEKWrapped, mbox.DEKNonce)
if err != nil {
return 0, err
}
ciphertext, nonce, err := sealAESGCM(dek, raw)
if err != nil {
return 0, err
}
now := time.Now()
dir := filepath.Join(s.BasePath, sanitizePathSegment(mbox.Email), folder, now.Format("2006-02-Jan"))
if err := os.MkdirAll(dir, 0o755); err != nil {
return 0, err
}
name := make([]byte, 8)
rand.Read(name)
storagePath := filepath.Join(dir, hex.EncodeToString(name)+".eml.enc")
if err := os.WriteFile(storagePath, ciphertext, 0o600); err != nil {
return 0, err
}
uid, err = s.DB.InsertMessage(mailboxID, folder, messageIDHeader, "", now, int64(len(raw)), storagePath, nonce, from, subject)
if err != nil {
os.Remove(storagePath)
return 0, err
}
if err := s.DB.AddMailboxUsedBytes(mailboxID, int64(len(raw))); err != nil {
return 0, err
}
return uid, nil
}
// FetchMessage decrypts a stored message on demand. Plaintext is never written to disk
// or cached — only returned to the caller.
func (s *Store) FetchMessage(mailboxID, uid int64) ([]byte, error) {
msg, err := s.DB.GetMessageByUID(mailboxID, uid)
if err != nil {
return nil, err
}
if msg == nil {
return nil, fmt.Errorf("mailstore: message %d not found in mailbox %d", uid, mailboxID)
}
mbox, err := s.DB.GetMailboxByID(mailboxID)
if err != nil {
return nil, err
}
if mbox == nil {
return nil, fmt.Errorf("mailstore: mailbox %d not found", mailboxID)
}
dek, err := s.UnwrapDEK(mbox.DEKWrapped, mbox.DEKNonce)
if err != nil {
return nil, err
}
ciphertext, err := os.ReadFile(msg.StoragePath)
if err != nil {
return nil, err
}
return openAESGCM(dek, ciphertext, msg.Nonce)
}
// DeleteMessage removes the on-disk ciphertext, the index row, and frees the quota.
func (s *Store) DeleteMessage(mailboxID, uid int64) error {
msg, err := s.DB.GetMessageByUID(mailboxID, uid)
if err != nil {
return err
}
if msg == nil {
return nil
}
if err := os.Remove(msg.StoragePath); err != nil && !os.IsNotExist(err) {
return err
}
if err := s.DB.DeleteMessage(mailboxID, uid); err != nil {
return err
}
return s.DB.AddMailboxUsedBytes(mailboxID, -msg.SizeBytes)
}