updated layout for webmail and added http dns letsencrypt

internal/smtpserver/bounce_test.go

@@ -0,0 +1,101 @@
+package smtpserver
+
+import (
+	"net/smtp"
+	"testing"
+
+	"mailgoserver/internal/db"
+	"mailgoserver/internal/mailstore"
+)
+
+// createTestMailboxWithQuota mirrors newTestBackendWithMailbox's inline mailbox setup,
+// parameterized by quota so this file can create both a normal and an
+// effectively-always-full mailbox.
+func createTestMailboxWithQuota(t *testing.T, backend *Backend, store *mailstore.Store, email string, quotaBytes int64) int64 {
+	t.Helper()
+	dek := mailstore.GenerateDEK()
+	wrapped, nonce, err := store.WrapDEK(dek)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hash, err := db.HashPassword("portal-password-unused")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mailboxID, err := backend.DB.CreateMailbox(email, hash, 1, quotaBytes, wrapped, nonce)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return mailboxID
+}
+
+// TestPartialLocalDeliveryFailureBouncesAndAccepts confirms a multi-recipient
+// transaction where one local mailbox accepts the message and another can't (quota
+// exceeded, discovered only during DATA — RCPT can't catch it) is accepted (250, not
+// 550 — the successful recipient already has it, so the client must not retry the
+// whole transaction) and that the sender gets a bounce in their own mailbox describing
+// the recipient that failed.
+func TestPartialLocalDeliveryFailureBouncesAndAccepts(t *testing.T) {
+	backend, okMailboxID := newTestBackendWithMailbox(t)
+	store := backend.Mailstore
+
+	fullMailboxID := createTestMailboxWithQuota(t, backend, store, "full@example.com", 1)
+	senderMailboxID := createTestMailboxWithQuota(t, backend, store, "test@example.com", 5*1024*1024*1024)
+
+	addr := startTestServer(t, backend)
+	c, err := smtp.Dial(addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	if err := c.Auth(smtp.PlainAuth("", "test@example.com", "testpass123", "127.0.0.1")); err != nil {
+		t.Fatalf("auth: %v", err)
+	}
+	if err := c.Mail("test@example.com"); err != nil {
+		t.Fatalf("MAIL FROM: %v", err)
+	}
+	if err := c.Rcpt("inbox@example.com"); err != nil {
+		t.Fatalf("RCPT (ok mailbox): %v", err)
+	}
+	if err := c.Rcpt("full@example.com"); err != nil {
+		t.Fatalf("RCPT (over-quota mailbox, still accepted at RCPT time): %v", err)
+	}
+	w, err := c.Data()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := w.Write([]byte("Subject: hello\r\n\r\nhi there")); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatalf("expected DATA to succeed (250, partial success) despite one recipient failing, got: %v", err)
+	}
+
+	okMsgs, err := backend.DB.ListMessagesInFolder(okMailboxID, "INBOX")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(okMsgs) != 1 {
+		t.Fatalf("expected the message delivered to inbox@example.com, got %d messages", len(okMsgs))
+	}
+
+	fullMsgs, err := backend.DB.ListMessagesInFolder(fullMailboxID, "INBOX")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fullMsgs) != 0 {
+		t.Fatalf("expected no message delivered to the over-quota mailbox, got %d", len(fullMsgs))
+	}
+
+	bounces, err := backend.DB.ListMessagesInFolder(senderMailboxID, "INBOX")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(bounces) != 1 {
+		t.Fatalf("expected 1 bounce message in the sender's own mailbox, got %d", len(bounces))
+	}
+	if bounces[0].CachedSubject != "Undelivered Mail Returned to Sender" {
+		t.Errorf("bounce subject = %q", bounces[0].CachedSubject)
+	}
+}

internal/smtpserver/log_privacy_test.go

@@ -0,0 +1,86 @@
+package smtpserver
+
+import "testing"
+
+// TestEmailLogBodyOmittedByDefault confirms the admin-visible email log gets the
+// message headers but never the body by default — only Subject/headers are diagnostic
+// metadata; the body is content, which shouldn't sit in a log unless explicitly opted
+// into (store_message_content) or the message needed spam review (see
+// TestEmailLogBodyKeptWhenQuarantined).
+func TestEmailLogBodyOmittedByDefault(t *testing.T) {
+	backend, _ := newTestBackendWithMailbox(t) // spam_reject_score set sky-high, so nothing quarantines here
+	addr := startTestServer(t, backend)
+
+	if err := sendTestMessage(t, addr, "hello"); err != nil {
+		t.Fatalf("send: %v", err)
+	}
+
+	logs, err := backend.DB.ListEmailLogsPage(0, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(logs) != 1 {
+		t.Fatalf("expected 1 email log entry, got %d", len(logs))
+	}
+	if logs[0].MessageBody != "" {
+		t.Errorf("expected no body logged by default, got %q", logs[0].MessageBody)
+	}
+	if logs[0].EmailHeaders == "" {
+		t.Error("expected headers to still be logged even with body omitted")
+	}
+}
+
+// TestEmailLogBodyKeptWhenStoreMessageContentEnabled confirms the sender's own
+// "Store Full Message Content" opt-in (esrv_senders.store_message_content) still works
+// despite the new default-off body logging.
+func TestEmailLogBodyKeptWhenStoreMessageContentEnabled(t *testing.T) {
+	backend, _ := newTestBackendWithMailbox(t)
+	if _, err := backend.DB.Exec(`UPDATE esrv_senders SET store_message_content = 1 WHERE email = 'test@example.com'`); err != nil {
+		t.Fatal(err)
+	}
+	addr := startTestServer(t, backend)
+
+	if err := sendTestMessage(t, addr, "hello"); err != nil {
+		t.Fatalf("send: %v", err)
+	}
+
+	logs, err := backend.DB.ListEmailLogsPage(0, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(logs) != 1 || logs[0].MessageBody == "" {
+		t.Fatalf("expected the opted-in sender's message body to be logged, got %+v", logs)
+	}
+}
+
+// TestEmailLogBodyKeptWhenQuarantined confirms a message quarantined to Spam still
+// gets its body logged even without any opt-in, so an admin can actually review a
+// spam/abuse report — the one deliberate exception to the default-off rule.
+func TestEmailLogBodyKeptWhenQuarantined(t *testing.T) {
+	backend, mailboxID := newTestBackendWithMailbox(t)
+	rspamd := fakeRspamd(t, 20, "add header")
+	backend.Cfg.Section("Rspamd").Key("enabled").SetValue("true")
+	backend.Cfg.Section("Rspamd").Key("url").SetValue(rspamd.URL)
+	backend.Cfg.Section("Rspamd").Key("reject_score").SetValue("15")
+	addr := startTestServer(t, backend)
+
+	if err := sendTestMessage(t, addr, "hello"); err != nil {
+		t.Fatalf("send: %v", err)
+	}
+
+	spamMsgs, err := backend.DB.ListMessagesInFolder(mailboxID, "Spam")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(spamMsgs) != 1 {
+		t.Fatalf("expected the message quarantined to Spam, got %d Spam messages", len(spamMsgs))
+	}
+
+	logs, err := backend.DB.ListEmailLogsPage(0, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(logs) != 1 || logs[0].MessageBody == "" {
+		t.Fatalf("expected the quarantined message's body to be logged for review, got %+v", logs)
+	}
+}

internal/smtpserver/mailbox_delivery_test.go

@@ -18,6 +18,7 @@ func newTestBackendWithMailbox(t *testing.T) (*Backend, int64) {
 
 	store := mailstore.New(backend.DB, mailstore.GenerateDEK(), t.TempDir())
 	backend.Mailstore = store
+	backend.Relay.Mailstore = store // mirrors main.go's wiring, needed for SendBounce's local-delivery shortcut
 	// Spam/SPF/DNSBL checks make live DNS calls (see internal/mailstore) — deliberately
 	// so in production, but that makes their exact score environment-dependent (e.g. a
 	// resolver that hijacks NXDOMAIN, or a real SPF record on the test domain). These
@@ -80,6 +81,52 @@ func TestLocalDeliveryToKnownMailbox(t *testing.T) {
 	}
 }
 
+// TestLocalDeliveryCachesFromHeaderDisplayName confirms cached_from is the message's
+// own From: header (e.g. "Bob Marley <bob@example.com>"), not the bare SMTP envelope
+// address — the envelope rarely carries a display name, but the header usually does,
+// and webmail's folder list wants the display name to show.
+func TestLocalDeliveryCachesFromHeaderDisplayName(t *testing.T) {
+	backend, mailboxID := newTestBackendWithMailbox(t)
+	addr := startTestServer(t, backend)
+
+	c, err := smtp.Dial(addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	if err := c.Auth(smtp.PlainAuth("", "test@example.com", "testpass123", "127.0.0.1")); err != nil {
+		t.Fatalf("auth: %v", err)
+	}
+	if err := c.Mail("test@example.com"); err != nil {
+		t.Fatalf("MAIL FROM: %v", err)
+	}
+	if err := c.Rcpt("inbox@example.com"); err != nil {
+		t.Fatalf("RCPT: %v", err)
+	}
+	w, err := c.Data()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := w.Write([]byte("From: Bob Marley <test@example.com>\r\nSubject: hello\r\n\r\nhi there")); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatalf("DATA: %v", err)
+	}
+
+	msgs, err := backend.DB.ListMessagesInFolder(mailboxID, "INBOX")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(msgs) != 1 {
+		t.Fatalf("expected 1 message, got %d", len(msgs))
+	}
+	if msgs[0].CachedFrom != "Bob Marley <test@example.com>" {
+		t.Errorf("cached_from = %q, want the From: header value with display name", msgs[0].CachedFrom)
+	}
+}
+
 func TestLocalDeliveryUnknownMailboxRejected(t *testing.T) {
 	backend, _ := newTestBackendWithMailbox(t)
 	addr := startTestServer(t, backend)

internal/smtpserver/session.go

@@ -248,6 +248,14 @@ func (s *Session) Data(r io.Reader) error {
 	toHeader := rebuiltHeaders["to"]
 	ccHeader := rebuiltHeaders["cc"]
 	subject := rebuiltHeaders["subject"]
+	// The message's own From: header (e.g. "Bob Marley <bob@example.com>"), not the
+	// bare SMTP envelope address — used only for what's cached/displayed (webmail's
+	// folder list), never for delivery/auth decisions, which stay on s.mailFrom
+	// throughout. Falls back to the envelope address if the header's missing/empty.
+	fromHeader := rebuiltHeaders["from"]
+	if fromHeader == "" {
+		fromHeader = s.mailFrom
+	}
 
 	// Attachment storage: only if the authenticated sender or whitelisted IP opted in.
 	storeMessage := false
@@ -321,23 +329,66 @@ func (s *Session) Data(r io.Reader) error {
 		results = s.backend.Relay.RelayEmailAsync(s.mailFrom, relayRcpts, signedContent, relayTypes)
 	}
 	if len(localRcpts) > 0 {
-		results = append(results, s.deliverLocally(localRcpts, localTypes, signedContent, messageID, subject)...)
+		results = append(results, s.deliverLocally(localRcpts, localTypes, signedContent, messageID, subject, fromHeader)...)
 	}
 
-	allSucceeded := len(results) > 0
+	var failed []relay.Result
 	for _, res := range results {
 		if res.Status != "success" {
-			allSucceeded = false
+			failed = append(failed, res)
+		}
+	}
+	allSucceeded := len(results) > 0 && len(failed) == 0
+	anySucceeded := len(results) > len(failed)
+
+	// A single SMTP response to DATA can't express "delivered to some recipients, not
+	// others" — rejecting the whole transaction here would make the connecting
+	// server's own retry logic re-deliver to the recipients that already succeeded.
+	// So: accept (below) and bounce the failed subset back to our own sender instead,
+	// exactly like a real MTA splitting a multi-recipient transaction's outcome. A
+	// bounce is never sent for a *total* failure — that gets rejected outright (550)
+	// below instead, letting the connecting server's own MTA generate the bounce to
+	// its user, avoiding a double notification. Skipped entirely for a null-sender
+	// message (s.mailFrom == "", already itself a bounce/DSN — replying to one is the
+	// classic bounce-loop bug) and for a currently-blacklisted peer, so a delivery
+	// failure never becomes a free "yes, that mailbox doesn't exist" oracle for abuse.
+	if len(failed) > 0 && anySucceeded && s.mailFrom != "" {
+		if blacklisted, _ := s.backend.DB.IsIPBlacklisted(s.peerIP); !blacklisted {
+			if err := s.backend.Relay.SendBounce(s.mailFrom, subject, messageID, failed); err != nil {
+				s.backend.Logger.Error("send bounce to %s: %v", s.mailFrom, err)
+			}
 		}
 	}
 
-	var emailHeaders, messageBody string
+	var emailHeaders string
 	if parseErr == nil {
 		emailHeaders = strings.Join(parsed.HeaderLines, "\n")
-		messageBody = parsed.BodyText
 	}
 
-	logID, logErr := s.backend.Relay.LogEmail(s.backend.Cfg, s.peerIP, s.mailFrom, toHeader, ccHeader, "", subject, emailHeaders, messageBody, messageID, s.username, dkimSigned, results)
+	// Privacy default: only headers (and the Subject field, logged separately below
+	// regardless) go into the admin-visible log, never the message itself — unless this
+	// sender/IP explicitly opted in via "Store Full Message Content" (storeMessage
+	// above), or the message was quarantined to Spam for at least one recipient, in
+	// which case an admin genuinely needs to see it to judge a spam/abuse report. When
+	// stored, it's the *entire* raw message (not a plain-text extraction) so the log
+	// viewer can render the real HTML body, inline images, and attachments — re-parsed
+	// on demand via internal/mailview, the same parser webmail's own message view uses
+	// — rather than a degraded text-only approximation.
+	storeContent := storeMessage
+	if !storeContent {
+		for _, res := range results {
+			if res.Quarantined {
+				storeContent = true
+				break
+			}
+		}
+	}
+	loggedBody := ""
+	if storeContent {
+		loggedBody = signedContent
+	}
+
+	logID, logErr := s.backend.Relay.LogEmail(s.backend.Cfg, s.peerIP, s.mailFrom, toHeader, ccHeader, "", subject, emailHeaders, loggedBody, messageID, s.username, dkimSigned, results)
 	if logErr != nil {
 		s.backend.Logger.Error("Failed to log email: %v", logErr)
 	} else {
@@ -353,6 +404,12 @@ func (s *Session) Data(r io.Reader) error {
 	if allSucceeded {
 		return &smtp.SMTPError{Code: 250, EnhancedCode: smtp.NoEnhancedCode, Message: "Message accepted for delivery"}
 	}
+	if anySucceeded {
+		// Some recipients already have the message — 250 it (see the bounce comment
+		// above for why), not 550, which would tell the connecting server to retry
+		// the whole thing and re-deliver to those recipients a second time.
+		return &smtp.SMTPError{Code: 250, EnhancedCode: smtp.NoEnhancedCode, Message: "Message accepted for delivery to some recipients"}
+	}
 	return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.NoEnhancedCode, Message: "Message relay failed"}
 }
 
@@ -361,7 +418,7 @@ func (s *Session) Data(r io.Reader) error {
 // and stores it into each resolved local mailbox, producing one relay.Result per
 // recipient so it can be merged into the same LogEmail/allSucceeded logic as relay
 // results.
-func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID, subject string) []relay.Result {
+func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID, subject, fromDisplay string) []relay.Result {
 	senderDomain := domainOfAddr(s.mailFrom)
 	dkimPass := senderDomain != "" && dkim.VerifyInbound(signedContent, senderDomain)
 	spfPass := mailstore.CheckSPF(s.mailFrom, s.peerIP)
@@ -430,7 +487,7 @@ func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID
 			markRead = action.MarkRead
 		}
 
-		uid, err := s.backend.Mailstore.StoreMessage(mbox.ID, folder, []byte(signedContent), messageID, s.mailFrom, subject)
+		uid, err := s.backend.Mailstore.StoreMessage(mbox.ID, folder, []byte(signedContent), messageID, fromDisplay, subject)
 		if err != nil {
 			errCode, errMsg := "450", err.Error()
 			if err == mailstore.ErrQuotaExceeded {
@@ -448,7 +505,7 @@ func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID
 		if spamGated {
 			serverResponse = "Quarantined to Spam folder"
 		}
-		results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: serverResponse})
+		results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: serverResponse, Quarantined: spamGated})
 	}
 	return results
 }