adding message viewer - fixing log view, change to aiosmtplib

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 *$py.class
 
 # Certs, db, private test files
+attachments/
 settings.ini
 *.crt
 *.key

email_server/auth.py

@@ -53,6 +53,7 @@ class EnhancedAuthenticator:
                 # Store authenticated sender info in session for later validation
                 session.authenticated_sender = sender
                 session.auth_type = 'sender'
+                session.username = username  # Store username in session
                 # Log successful authentication
                 log_auth_attempt(
                     auth_type='sender',
@@ -167,6 +168,7 @@ def validate_sender_authorization(session, mail_from: str) -> tuple[bool, str]:
         # Store IP auth info in session
         session.auth_type = 'ip'
         session.authorized_domain = from_domain
+        session.username = f"IP:{peer_ip}"  # Store IP as username for IP authentication
         
         log_auth_attempt(
             auth_type='ip',

email_server/dkim_manager.py

@@ -8,7 +8,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
 from datetime import datetime
 from email_server.models import Session, Domain, DKIMKey, CustomHeader
 from email_server.settings_loader import load_settings
-from email_server.tool_box import get_logger
+from email_server.tool_box import get_logger, get_current_time
 import random
 import string
 
@@ -55,7 +55,7 @@ class DKIMManager:
             existing_active_keys = session.query(DKIMKey).filter_by(domain_id=domain.id, is_active=True).all()
             for existing_key in existing_active_keys:
                 existing_key.is_active = False
-                existing_key.replaced_at = datetime.now()
+                existing_key.replaced_at = get_current_time()
                 logger.debug(f"Marked DKIM key as replaced for domain {domain_name} selector {existing_key.selector}")
             
             # Check if we're reusing an existing selector - if so, reactivate instead of creating new
@@ -75,7 +75,7 @@ class DKIMManager:
                     ).all()
                     for key in other_active_keys:
                         key.is_active = False
-                        key.replaced_at = datetime.now()
+                        key.replaced_at = get_current_time()
                         logger.debug(f"Deactivated other active DKIM key for domain {domain_name} selector {key.selector}")
                     # Reactivate existing key with same selector, clear replaced_at timestamp
                     existing_key_with_selector.is_active = True
@@ -114,7 +114,7 @@ class DKIMManager:
                 selector=use_selector,
                 private_key=private_pem,
                 public_key=public_pem,
-                created_at=datetime.now(),
+                created_at=get_current_time(),
                 is_active=True
             )
             session.add(dkim_key)

email_server/email_relay.py

@@ -2,156 +2,272 @@
 Email relay functionality for the SMTP server.
 """
 
+import asyncio
 import dns.resolver
-import smtplib
+from email_server.models import Session, EmailLog, EmailRecipientLog
-import ssl
-from datetime import datetime
-from email_server.models import Session, EmailLog
 from email_server.settings_loader import load_settings
-from email_server.tool_box import get_logger
+from email_server.tool_box import get_logger, get_current_time
+import aiosmtplib
 
 logger = get_logger()
 
+settings = load_settings()
+_relay_tls_timeout = settings['Server'].get('relay_timeout', 15)
+
 class EmailRelay:
     """Handles relaying emails to recipient mail servers."""
 
     def __init__(self):
-        self.timeout = 30  # Increased timeout for TLS negotiations
+
+        self.timeout = _relay_tls_timeout  # Increased timeout for TLS negotiations
         # Get the configured hostname for HELO/EHLO identification
-        settings = load_settings()
         self.hostname = settings['Server'].get('helo_hostname',
                                              settings['Server'].get('hostname', 'localhost'))
         logger.debug(f"EmailRelay initialized with hostname: {self.hostname}")
 
-    def relay_email(self, mail_from, rcpt_tos, content):
+    def _modify_headers_for_recipients(self, content, to_addresses, cc_addresses=None):
-        """Relay email to recipient's mail server with opportunistic TLS."""
+        """Modify email headers to set To and Cc fields, removing Bcc."""
-        try:
+        lines = content.splitlines()
-            for rcpt in rcpt_tos:
+        new_headers = []
-                domain = rcpt.split('@')[1]
+        body_start = 0
+        for i, line in enumerate(lines):
+            if line.strip() == '':
+                body_start = i + 1
+                break
+            # Skip existing To, Cc, Bcc headers
 
-                # Resolve MX record for the domain
+            if not line.lower().startswith(('to:', 'cc:', 'bcc:')):
+                new_headers.append(line)
+
+        # Add new To and Cc headers
+        if to_addresses:
+            new_headers.append(f"To: {', '.join(to_addresses)}")
+        if cc_addresses:
+            new_headers.append(f"Cc: {', '.join(cc_addresses)}")
+
+        # Reconstruct the message
+        body = '\n'.join(lines[body_start:]) if body_start < len(lines) else ''
+        return '\r\n'.join(new_headers) + '\r\n\r\n' + body
+
+    async def relay_email_async(
+        self,
+        mail_from: str,
+        rcpt_tos: list[str],
+        content: str,
+        username: str = None,
+        cc_addresses: list[str] = None,
+        bcc_addresses: list[str] = None,
+        recipient_types: list[str] = None
+    ) -> list[dict]:
+        """Relay email to recipients' mail servers asynchronously with encryption.
+
+        Args:
+            mail_from (str): Sender email address.
+            rcpt_tos (list[str]): List of recipient email addresses.
+            content (str): Raw email content.
+            username (str, optional): Authenticated username.
+            cc_addresses (list[str], optional): CC addresses.
+            bcc_addresses (list[str], optional): BCC addresses.
+            recipient_types (list[str], optional): Recipient types (to/cc/bcc).
+
+        Returns:
+            list[dict]: Per-recipient delivery results.
+        """
+        results = []
+        recipient_type_map = {}
+        if recipient_types and len(recipient_types) == len(rcpt_tos):
+            for addr, rtype in zip(rcpt_tos, recipient_types):
+                recipient_type_map[addr] = rtype
+        else:
+            for addr in rcpt_tos:
+                recipient_type_map[addr] = 'to'
+
+        domain_groups = {}
+        for rcpt in rcpt_tos:
+            domain = rcpt.split('@')[1].lower()
+            rtype = recipient_type_map.get(rcpt, 'to')
+            if domain not in domain_groups:
+                domain_groups[domain] = {'to': [], 'cc': [], 'bcc': []}
+            domain_groups[domain][rtype].append(rcpt)
+
+        for domain, recipients in domain_groups.items():
+            to_recipients = recipients['to']
+            cc_recipients = recipients['cc']
+            bcc_recipients = recipients['bcc']
             try:
                 mx_records = dns.resolver.resolve(domain, 'MX')
-                    # Sort by priority (lower number = higher priority)
                 mx_records = sorted(mx_records, key=lambda x: x.preference)
-                    mx_host = mx_records[0].exchange.to_text().rstrip('.')
+                mx_hosts = [mx.exchange.to_text().rstrip('.') for mx in mx_records]
-                    logger.debug(f'Found MX record for {domain}: {mx_host}')
+                logger.debug(f'Found MX records for {domain}: {mx_hosts}')
             except Exception as e:
                 logger.error(f'Failed to resolve MX for {domain}: {e}')
-                    return False
+                for rcpt in to_recipients + cc_recipients + bcc_recipients:
-
+                    results.append({
-                # Try to relay with opportunistic TLS
+                        'recipient': rcpt,
-                if not self._relay_with_opportunistic_tls(mail_from, rcpt, content, mx_host):
+                        'status': 'failed',
-                    return False
+                        'error_code': 'MX',
-                    
+                        'error_message': str(e),
-            return True
+                        'server_response': None,
-        except Exception as e:
+                        'recipient_type': recipient_type_map.get(rcpt, 'to')
-            logger.error(f'General relay error: {e}')
+                    })
-            return False
-    
-    def _relay_with_opportunistic_tls(self, mail_from, rcpt, content, mx_host):
-        """Relay email with opportunistic TLS (like Gmail does)."""
-        try:
-            # First, try with STARTTLS (encrypted)
-            try:
-                with smtplib.SMTP(mx_host, 25, timeout=self.timeout) as relay_server:
-                    relay_server.set_debuglevel(1)
-                    
-                    # Try to enable TLS if the server supports it
-                    try:
-                        # Check if server supports STARTTLS - use proper hostname for EHLO
-                        logger.debug(f'Sending EHLO {self.hostname} to {mx_host}')
-                        relay_server.ehlo(self.hostname)
-                        if relay_server.has_extn('starttls'):
-                            logger.debug(f'Starting TLS connection to {mx_host}')
-                            context = ssl.create_default_context()
-                            # Allow self-signed certificates for mail servers
-                            context.check_hostname = False
-                            context.verify_mode = ssl.CERT_NONE
-                            relay_server.starttls(context=context)
-                            logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to {mx_host}')
-                            relay_server.ehlo(self.hostname)  # Say hello again after STARTTLS with proper hostname
-                            logger.debug(f'TLS connection established to {mx_host}')
-                        else:
-                            logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text')
-                    except Exception as tls_e:
-                        logger.warning(f'STARTTLS failed with {mx_host}, continuing with plain text: {tls_e}')
-                    
-                    # Send the email
-                    relay_server.sendmail(mail_from, rcpt, content)
-                    logger.debug(f'Successfully relayed email to {rcpt} via {mx_host}')
-                    return True
-                    
-            except Exception as e:
-                logger.error(f'Failed to relay email to {rcpt} via {mx_host}: {e}')
-                
-                # Fallback: try alternative MX records if available
-                try:
-                    domain = rcpt.split('@')[1]
-                    mx_records = dns.resolver.resolve(domain, 'MX')
-                    mx_records = sorted(mx_records, key=lambda x: x.preference)
-                    
-                    # Try other MX records
-                    for mx_record in mx_records[1:3]:  # Try up to 2 backup MX records
-                        backup_mx = mx_record.exchange.to_text().rstrip('.')
-                        logger.debug(f'Trying backup MX record: {backup_mx}')
-                        
-                        try:
-                            with smtplib.SMTP(backup_mx, 25, timeout=self.timeout) as backup_server:
-                                backup_server.set_debuglevel(1)
-                                
-                                # Try TLS with backup server too
-                                try:
-                                    logger.debug(f'Sending EHLO {self.hostname} to backup {backup_mx}')
-                                    backup_server.ehlo(self.hostname)
-                                    if backup_server.has_extn('starttls'):
-                                        context = ssl.create_default_context()
-                                        context.check_hostname = False
-                                        context.verify_mode = ssl.CERT_NONE
-                                        backup_server.starttls(context=context)
-                                        logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to backup {backup_mx}')
-                                        backup_server.ehlo(self.hostname)
-                                        logger.debug(f'TLS connection established to backup {backup_mx}')
-                                except Exception:
-                                    logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text')
-                                
-                                backup_server.sendmail(mail_from, rcpt, content)
-                                logger.debug(f'Successfully relayed email to {rcpt} via backup {backup_mx}')
-                                return True
-                        except Exception as backup_e:
-                            logger.warning(f'Backup MX {backup_mx} also failed: {backup_e}')
                 continue
 
-                except Exception as fallback_e:
+            # Relay to To and Cc recipients together
-                    logger.error(f'All MX records failed for {rcpt}: {fallback_e}')
+            if to_recipients or cc_recipients:
-                
+                modified_content = self._modify_headers_for_recipients(content, to_recipients, cc_recipients)
-                return False
+                delivered = False
-                
+                last_error = None
+                for mx_host in mx_hosts:
+                    try:
+                        # Only use port 25 for MX delivery
+                        port = 25
+                        smtp = aiosmtplib.SMTP(hostname=mx_host, port=port, timeout=self.timeout, local_hostname=self.hostname)
+                        await smtp.connect()
+                        ext = getattr(smtp, 'extensions', None)
+                        if ext is None:
+                            ext = getattr(smtp, 'esmtp_extensions', None)
+                        if ext is None:
+                            logger.error(f"SMTP object has no 'extensions' or 'esmtp_extensions'. Available attributes: {dir(smtp)}")
+                            ext = {}
+                        if 'starttls' in ext:
+                            logger.debug(f'STARTTLS supported by {mx_host}:{port}, upgrading to TLS')
+                            await smtp.starttls()
+                        else:
+                            logger.warning(f'STARTTLS not supported by {mx_host}:{port}, sending in plain text!')
+                        response = await smtp.sendmail(mail_from, to_recipients + cc_recipients, modified_content)
+                        logger.debug(f'Successfully relayed email to {to_recipients + cc_recipients} via {mx_host}:{port}')
+                        for rcpt in to_recipients + cc_recipients:
+                            results.append({
+                                'recipient': rcpt,
+                                'status': 'success',
+                                'error_code': None,
+                                'error_message': None,
+                                'server_response': str(response),
+                                'recipient_type': recipient_type_map.get(rcpt, 'to')
+                            })
+                        await smtp.quit()
+                        delivered = True
+                        break
                     except Exception as e:
-            logger.error(f'Unexpected error in TLS relay: {e}')
+                        logger.error(f'Failed to relay email to {to_recipients + cc_recipients} via {mx_host}:{port}: {e}')
-            return False
+                        last_error = {
+                            'status': 'failed',
+                            'error_code': 'RELAY',
+                            'error_message': str(e),
+                            'server_response': None
+                        }
+                        continue
+                if not delivered and last_error:
+                    for rcpt in to_recipients + cc_recipients:
+                        results.append({
+                            'recipient': rcpt,
+                            'status': last_error['status'],
+                            'error_code': last_error['error_code'],
+                            'error_message': last_error['error_message'],
+                            'server_response': last_error['server_response'],
+                            'recipient_type': recipient_type_map.get(rcpt, 'to')
+                        })
 
-    def log_email(self, message_id, peer, mail_from, rcpt_tos, content, status, dkim_signed=False):
+            # Relay to Bcc recipients individually
-        """Log email activity to database."""
+            for bcc in bcc_recipients:
+                modified_content = self._modify_headers_for_recipients(content, [bcc], None)
+                delivered = False
+                last_error = None
+                for mx_host in mx_hosts:
+                    try:
+                        port = 25
+                        smtp = aiosmtplib.SMTP(hostname=mx_host, port=port, timeout=self.timeout, local_hostname=self.hostname)
+                        await smtp.connect()
+                        ext = getattr(smtp, 'extensions', None)
+                        if ext is None:
+                            ext = getattr(smtp, 'esmtp_extensions', None)
+                        if ext is None:
+                            logger.error(f"SMTP object has no 'extensions' or 'esmtp_extensions'. Available attributes: {dir(smtp)}")
+                            ext = {}
+                        if 'starttls' in ext:
+                            logger.debug(f'STARTTLS supported by {mx_host}:{port} for BCC, upgrading to TLS')
+                            await smtp.starttls()
+                        else:
+                            logger.warning(f'STARTTLS not supported by {mx_host}:{port} for BCC, sending in plain text!')
+                        response = await smtp.sendmail(mail_from, [bcc], modified_content)
+                        logger.debug(f'Successfully relayed BCC email to {bcc} via {mx_host}:{port}')
+                        results.append({
+                            'recipient': bcc,
+                            'status': 'success',
+                            'error_code': None,
+                            'error_message': None,
+                            'server_response': str(response),
+                            'recipient_type': 'bcc'
+                        })
+                        await smtp.quit()
+                        delivered = True
+                        break
+                    except Exception as e:
+                        logger.error(f'Failed to relay BCC email to {bcc} via {mx_host}:{port}: {e}')
+                        last_error = {
+                            'status': 'failed',
+                            'error_code': 'RELAY',
+                            'error_message': str(e),
+                            'server_response': None,
+                            'recipient_type': 'bcc'
+                        }
+                        continue
+                if not delivered and last_error:
+                    results.append({
+                        'recipient': bcc,
+                        **last_error
+                    })
+        return results
+
+    def relay_email(self, *args, **kwargs):
+        """Synchronous wrapper for relay_email_async for compatibility."""
+        return asyncio.run(self.relay_email_async(*args, **kwargs))
+
+    def log_email(self, message_id, peer, mail_from, to_address, cc_addresses, bcc_addresses, subject, email_headers, message_body, status, dkim_signed=False, username=None, recipient_results=None):
+        """Log email activity to database, including per-recipient results."""
         session_db = Session()
         try:
-            # Convert content to string if it's bytes
+            # Determine status: relayed, partial, failed
-            if isinstance(content, bytes):
+            delivered = [r for r in (recipient_results or []) if r['status'] == 'success']
-                content_str = content.decode('utf-8', errors='replace')
+            failed = [r for r in (recipient_results or []) if r['status'] != 'success']
+            if delivered and failed:
+                overall_status = 'partial'
+            elif delivered:
+                overall_status = 'relayed'
             else:
-                content_str = content
+                overall_status = 'failed'
             
             email_log = EmailLog(
                 message_id=message_id,
-                timestamp=datetime.now(),
+                timestamp=get_current_time(),
-                peer=str(peer),
+                peer_ip=peer,
                 mail_from=mail_from,
-                rcpt_tos=', '.join(rcpt_tos),
+                to_address=to_address or '',
-                content=content_str,
+                cc_addresses=cc_addresses or '',
-                status=status,
+                bcc_addresses=bcc_addresses or '',
-                dkim_signed=dkim_signed
+                subject=subject,
+                email_headers=email_headers,
+                message_body=message_body,
+                status=overall_status,
+                dkim_signed=dkim_signed,
+                username=username
             )
             session_db.add(email_log)
+            session_db.flush()
+            
+            # Log per-recipient results
+            if recipient_results:
+                for r in recipient_results:
+                    recipient_log = EmailRecipientLog(
+                        email_log_id=email_log.id,
+                        recipient=r['recipient'],
+                        recipient_type=r.get('recipient_type', 'to'),
+                        status=r['status'],
+                        error_code=r.get('error_code'),
+                        error_message=r.get('error_message'),
+                        server_response=r.get('server_response')
+                    )
+                    session_db.add(recipient_log)
             session_db.commit()
             logger.debug(f'Logged email: {message_id}')
         except Exception as e:

email_server/models.py

@@ -63,6 +63,7 @@ class Sender(Base):
     can_send_as_domain = Column(Boolean, default=False)
     is_active = Column(Boolean, default=True)
     created_at = Column(DateTime, default=func.now())
+    store_message_content = Column(Boolean, default=False)  # Store message body/attachments
     
     def can_send_as(self, from_address: str) -> bool:
         """
@@ -100,6 +101,7 @@ class WhitelistedIP(Base):
     domain_id = Column(Integer, ForeignKey('esrv_domains.id'), nullable=False)
     is_active = Column(Boolean, default=True)
     created_at = Column(DateTime, default=func.now())
+    store_message_content = Column(Boolean, default=False)  # Store message body/attachments
     
     def can_send_for_domain(self, domain_name: str) -> bool:
         """
@@ -133,26 +135,44 @@ class EmailLog(Base):
     __tablename__ = 'esrv_email_logs'
     
     id = Column(Integer, primary_key=True)
-    
-    # Legacy columns (from original schema)
     message_id = Column(String, unique=True, nullable=False)
     timestamp = Column(DateTime, nullable=False)
-    peer = Column(String, nullable=False)
+    peer_ip = Column(String, nullable=False)  # Store only IP address
     mail_from = Column(String, nullable=False)
-    rcpt_tos = Column(String, nullable=False)
+    to_address = Column(String, nullable=False, server_default='')
-    content = Column(Text, nullable=False)
+    cc_addresses = Column(String, nullable=True, server_default='')  # Comma-separated CC
+    bcc_addresses = Column(String, nullable=True, server_default='')  # Comma-separated BCC
+    subject = Column(Text, nullable=True)
+    email_headers = Column(Text, nullable=False)  # Store only email headers
+    message_body = Column(Text, nullable=True)  # Store actual message content
     status = Column(String, nullable=False)
     dkim_signed = Column(Boolean, default=False)
-    
+    username = Column(String, nullable=True)  # Authenticated username
-    # New columns (added later)
-    from_address = Column(String, nullable=False, server_default='unknown')
-    to_address = Column(String, nullable=False, server_default='unknown')
-    subject = Column(Text, nullable=True)
-    message = Column(Text, nullable=True)
     created_at = Column(DateTime, default=func.now())
 
+    recipients = relationship("EmailRecipientLog", back_populates="email_log", cascade="all, delete-orphan")
+    attachments = relationship("EmailAttachment", back_populates="email_log", cascade="all, delete-orphan")
+
     def __repr__(self):
-        return f"<EmailLog(id={self.id}, message_id='{self.message_id}', from='{self.mail_from}', to='{self.rcpt_tos}', status='{self.status}')>"
+        return f"<EmailLog(id={self.id}, message_id='{self.message_id}', from='{self.mail_from}', to='{self.to_address}', status='{self.status}')>"
+
+class EmailRecipientLog(Base):
+    """Log for each recipient of an email, including status and error details."""
+    __tablename__ = 'esrv_email_recipient_logs'
+
+    id = Column(Integer, primary_key=True)
+    email_log_id = Column(Integer, ForeignKey('esrv_email_logs.id'), nullable=False)
+    recipient = Column(String, nullable=False)
+    recipient_type = Column(String, nullable=False)  # 'to', 'cc', 'bcc'
+    status = Column(String, nullable=False)  # 'success', 'failed', etc.
+    error_code = Column(String, nullable=True)
+    error_message = Column(Text, nullable=True)
+    server_response = Column(Text, nullable=True)
+
+    email_log = relationship("EmailLog", back_populates="recipients")
+
+    def __repr__(self):
+        return f"<EmailRecipientLog(id={self.id}, recipient='{self.recipient}', type='{self.recipient_type}', status='{self.status}')>"
 
 class AuthLog(Base):
     """Authentication log model for security auditing."""
@@ -199,6 +219,23 @@ class CustomHeader(Base):
     def __repr__(self):
         return f"<CustomHeader(id={self.id}, domain_id={self.domain_id}, header='{self.header_name}: {self.header_value}', active={self.is_active})>"
 
+class EmailAttachment(Base):
+    """Attachment metadata and file path, linked to EmailLog."""
+    __tablename__ = 'esrv_email_attachments'
+
+    id = Column(Integer, primary_key=True)
+    email_log_id = Column(Integer, ForeignKey('esrv_email_logs.id'), nullable=False)
+    filename = Column(String, nullable=False)
+    content_type = Column(String, nullable=True)
+    file_path = Column(String, nullable=False)  # Path on disk
+    size = Column(Integer, nullable=True)
+    uploaded_at = Column(DateTime, default=func.now())
+
+    email_log = relationship("EmailLog", back_populates="attachments")
+
+    def __repr__(self):
+        return f"<EmailAttachment(id={self.id}, filename='{self.filename}', file_path='{self.file_path}')>"
+
 
 def create_tables():
     """Create all database tables using ESRV schema."""

email_server/server_web_ui/__init__.py

@@ -15,3 +15,4 @@ from .ip_whitelist import *
 from .dkim import *
 from .settings import *
 from .logs import *
+from .view_message import *

email_server/server_web_ui/dashboard.py

@@ -5,7 +5,7 @@ This module provides the main dashboard view and overview functionality.
 """
 
 from flask import render_template
-from email_server.models import Session, Domain, Sender, DKIMKey, EmailLog, AuthLog
+from email_server.models import Session, Domain, Sender, DKIMKey, EmailLog, AuthLog, EmailRecipientLog
 from email_server.tool_box import get_logger
 from .routes import email_bp
 
@@ -24,6 +24,8 @@ def dashboard():
         
         # Get recent email logs
         recent_emails = session.query(EmailLog).order_by(EmailLog.created_at.desc()).limit(10).all()
+        # Get recipient logs for each recent email
+        recipient_logs_map = {email.id: session.query(EmailRecipientLog).filter_by(email_log_id=email.id).all() for email in recent_emails}
         
         # Get recent auth logs
         recent_auths = session.query(AuthLog).order_by(AuthLog.created_at.desc()).limit(10).all()
@@ -33,6 +35,7 @@ def dashboard():
                              sender_count=sender_count,
                              dkim_count=dkim_count,
                              recent_emails=recent_emails,
-                             recent_auths=recent_auths)
+                             recent_auths=recent_auths,
+                             recipient_logs_map=recipient_logs_map)
     finally:
         session.close()

email_server/server_web_ui/dkim.py

@@ -9,12 +9,12 @@ This module provides DKIM key management functionality including:
 - DKIM DNS verification
 """
 
-from flask import render_template, request, redirect, url_for, flash, jsonify
+from flask import render_template, request, redirect, url_for, flash, jsonify, current_app
 from datetime import datetime
 import re
 from email_server.models import Session, Domain, DKIMKey
 from email_server.dkim_manager import DKIMManager
-from email_server.tool_box import get_logger
+from email_server.tool_box import get_logger, get_current_time
 from .utils import get_public_ip, check_dns_record, generate_spf_record
 from .routes import email_bp
 
@@ -107,7 +107,7 @@ def create_dkim():
         active_keys = session.query(DKIMKey).filter_by(domain_id=domain.id, is_active=True).all()
         for key in active_keys:
             key.is_active = False
-            key.replaced_at = datetime.now()
+            key.replaced_at = get_current_time()
         # Create new DKIM key
         dkim_manager = DKIMManager()
         created = dkim_manager.generate_dkim_keypair(domain_name, selector=selector, force_new_key=True)
@@ -146,7 +146,7 @@ def regenerate_dkim(domain_id: int):
         # Mark existing keys as replaced
         for key in existing_keys:
             key.is_active = False
-            key.replaced_at = datetime.now()  # Mark when this key was replaced
+            key.replaced_at = get_current_time()  # Mark when this key was replaced
         
         # Generate new DKIM key preserving the existing selector
         dkim_manager = DKIMManager()
@@ -331,7 +331,7 @@ def toggle_dkim(dkim_id: int):
             ).all()
             for key in other_active_keys:
                 key.is_active = False
-                key.replaced_at = datetime.now()
+                key.replaced_at = get_current_time()
         
         dkim_key.is_active = not old_status
         if dkim_key.is_active:

email_server/server_web_ui/ip_whitelist.py

@@ -37,6 +37,7 @@ def add_ip():
         if request.method == 'POST':
             ip_address = request.form.get('ip_address', '').strip()
             domain_id = request.form.get('domain_id', type=int)
+            store_message_content = bool(request.form.get('store_message_content'))
             
             if not all([ip_address, domain_id]):
                 flash('All fields are required', 'error')
@@ -58,7 +59,8 @@ def add_ip():
             # Create whitelisted IP
             whitelist = WhitelistedIP(
                 ip_address=ip_address,
-                domain_id=domain_id
+                domain_id=domain_id,
+                store_message_content=store_message_content
             )
             session.add(whitelist)
             session.commit()
@@ -166,6 +168,7 @@ def edit_ip(ip_id: int):
         if request.method == 'POST':
             ip_address = request.form.get('ip_address', '').strip()
             domain_id = request.form.get('domain_id', type=int)
+            store_message_content = bool(request.form.get('store_message_content'))
             
             if not all([ip_address, domain_id]):
                 flash('All fields are required', 'error')
@@ -191,6 +194,7 @@ def edit_ip(ip_id: int):
             # Update IP record
             ip_record.ip_address = ip_address
             ip_record.domain_id = domain_id
+            ip_record.store_message_content = store_message_content
             session.commit()
             
             flash(f'IP whitelist record updated', 'success')

email_server/server_web_ui/logs.py

@@ -4,12 +4,11 @@ Logs blueprint for the SMTP server web UI.
 This module provides email and authentication log viewing functionality.
 """
 
-from flask import render_template, request, jsonify
+from flask import render_template, request, send_file, redirect, url_for, flash, Response
-from email_server.models import Session, EmailLog, AuthLog, Domain
+from email_server.models import Session, EmailLog, AuthLog, EmailRecipientLog, EmailAttachment
 from email_server.tool_box import get_logger
-from sqlalchemy import desc
-from datetime import datetime, timedelta
 from .routes import email_bp
+import os
 
 logger = get_logger()
 
@@ -40,10 +39,15 @@ def logs():
             # Convert to unified format
             combined_logs = []
             for log in email_logs:
+                # Fetch recipient logs and attachments for each email log
+                recipient_logs = session.query(EmailRecipientLog).filter_by(email_log_id=log.id).all()
+                attachments = session.query(EmailAttachment).filter_by(email_log_id=log.id).all()
                 combined_logs.append({
                     'type': 'email',
                     'timestamp': log.created_at,
-                    'data': log
+                    'data': log,
+                    'recipients': recipient_logs,
+                    'attachments': attachments
                 })
             for log in auth_logs:
                 combined_logs.append({
@@ -69,12 +73,20 @@ def logs():
         
         has_next = offset + per_page < total
         has_prev = page > 1
-        
+        # Fetch recipient logs and attachments for each email log if emails
+        recipient_logs_map = {}
+        attachments_map = {}
+        if filter_type == 'emails':
+            for log in logs:
+                recipient_logs_map[log.id] = session.query(EmailRecipientLog).filter_by(email_log_id=log.id).all()
+                attachments_map[log.id] = session.query(EmailAttachment).filter_by(email_log_id=log.id).all()
         return render_template('logs.html', 
                              logs=logs, 
                              filter_type=filter_type,
                              page=page,
                              has_next=has_next,
-                             has_prev=has_prev)
+                             has_prev=has_prev,
+                             recipient_logs_map=recipient_logs_map,
+                             attachments_map=attachments_map)
     finally:
         session.close()

email_server/server_web_ui/routes.py

@@ -1,9 +1,12 @@
 """
 Main routes and blueprint definition for the SMTP server web UI.
 """
-from flask import Blueprint, render_template
+from flask import Blueprint, render_template, request, jsonify, current_app
-from email_server.tool_box import get_logger
+from email_server.models import Session, EmailLog, AuthLog
+from email_server.tool_box import get_logger, get_current_time
+from email_server.settings_loader import load_settings
 from datetime import datetime
+import pytz
 
 
 # Create the main email blueprint
@@ -15,14 +18,35 @@ email_bp = Blueprint('email', __name__,
 
 logger = get_logger()
 
+# Get timezone from settings
+settings = load_settings()
+timezone = pytz.timezone(settings['Server'].get('time_zone', 'UTC'))
+
+@email_bp.app_template_filter('format_datetime')
+def format_datetime(value, timezone=None):
+    """Format datetime with the correct timezone from settings or argument."""
+    if value is None:
+        return ''
+    import pytz
+    if timezone is None:
+        settings = load_settings()
+        timezone = settings['Server'].get('time_zone', 'UTC')
+    tz = pytz.timezone(timezone)
+    if value.tzinfo is None:
+        value = pytz.UTC.localize(value)
+    local_dt = value.astimezone(tz)
+    return local_dt.strftime('%Y-%m-%d %H:%M:%S')
+
+from .view_message import *  # Import view_message routes
+
 # Error handlers
 @email_bp.errorhandler(404)
 def not_found(error):
     """Handle 404 errors."""
     return render_template('error.html', 
                          error_code=404,
-                         error_message='Page not found',
+                         error_message="Page not found",
-                         current_time=datetime.now()), 404
+                         current_time=get_current_time()), 404
 
 @email_bp.errorhandler(500)
 def internal_error(error):
@@ -30,5 +54,5 @@ def internal_error(error):
     logger.error(f"Internal error: {error}")
     return render_template('error.html',
                          error_code=500,
-                         error_message='Internal server error',
+                         error_message=str(error),
-                         current_time=datetime.now()), 500
+                         current_time=get_current_time()), 500

email_server/server_web_ui/senders.py

@@ -40,6 +40,7 @@ def add_sender():
             password = request.form.get('password', '').strip()
             domain_id = request.form.get('domain_id', type=int)
             can_send_as_domain = request.form.get('can_send_as_domain') == 'on'
+            store_message_content = request.form.get('store_message_content') == 'on'
             
             if not all([email, password, domain_id]):
                 flash('All fields are required', 'error')
@@ -61,7 +62,8 @@ def add_sender():
                 email=email,
                 password_hash=hash_password(password),
                 domain_id=domain_id,
-                can_send_as_domain=can_send_as_domain
+                can_send_as_domain=can_send_as_domain,
+                store_message_content=store_message_content
             )
             session.add(sender)
             session.commit()
@@ -171,6 +173,7 @@ def edit_sender(user_id: int):
             password = request.form.get('password', '').strip()
             domain_id = request.form.get('domain_id', type=int)
             can_send_as_domain = request.form.get('can_send_as_domain') == 'on'
+            store_message_content = request.form.get('store_message_content') == 'on'
             
             if not all([email, domain_id]):
                 flash('Email and domain are required', 'error')
@@ -194,6 +197,7 @@ def edit_sender(user_id: int):
             sender.email = email
             sender.domain_id = domain_id
             sender.can_send_as_domain = can_send_as_domain
+            sender.store_message_content = store_message_content
             
             # Update password if provided
             if password:

email_server/server_web_ui/templates/add_ip.html

@@ -67,6 +67,18 @@
                             </div>
                         </div>
                         
+                        <div class="mb-3">
+                            <div class="form-check">
+                                <input class="form-check-input" type="checkbox" id="store_message_content" name="store_message_content">
+                                <label class="form-check-label" for="store_message_content">
+                                    <strong>Store Full Message Content</strong>
+                                </label>
+                                <div class="form-text">
+                                    If enabled, the full message body and attachments will be stored and viewable in logs. Otherwise, only headers and subject are stored.
+                                </div>
+                            </div>
+                        </div>
+                        
                         <div class="alert alert-warning">
                             <h6 class="alert-heading">
                                 <i class="bi bi-exclamation-triangle me-2"></i>

email_server/server_web_ui/templates/add_sender.html

@@ -70,6 +70,18 @@
                             </div>
                         </div>
                         
+                        <div class="mb-3">
+                            <div class="form-check">
+                                <input class="form-check-input" type="checkbox" id="store_message_content" name="store_message_content">
+                                <label class="form-check-label" for="store_message_content">
+                                    <strong>Store Full Message Content</strong>
+                                </label>
+                                <div class="form-text">
+                                    If enabled, the full message body and attachments will be stored and viewable in logs. Otherwise, only headers and subject are stored.
+                                </div>
+                            </div>
+                        </div>
+                        
                         <div class="alert alert-info">
                             <h6 class="alert-heading">
                                 <i class="bi bi-info-circle me-2"></i>

email_server/server_web_ui/templates/dashboard.html

@@ -144,7 +144,7 @@
                                 <tr>
                                     <td>
                                         <small class="text-muted">
-                                            {{ email.created_at.strftime('%H:%M:%S') }}
+                                            {{ email.created_at|format_datetime }}
                                         </small>
                                     </td>
                                     <td>
@@ -153,16 +153,31 @@
                                         </span>
                                     </td>
                                     <td>
-                                        <span class="text-truncate d-inline-block" style="max-width: 150px;" title="{{ email.to_address }}">
+                                        <span class="text-truncate d-inline-block" style="max-width: 150px;" title="{{ email.rcpt_tos }}">
-                                            {{ email.to_address }}
+                                            {{ email.rcpt_tos }}
                                         </span>
                                     </td>
                                     <td>
-                                        {% if email.status == 'relayed' %}
+                                        {% set delivered = recipient_logs_map[email.id]|selectattr('status', 'equalto', 'success')|list %}
+                                        {% set failed = recipient_logs_map[email.id]|selectattr('status', 'ne', 'success')|list %}
+                                        {% if delivered and failed %}
+                                            {% set overall_status = 'partial' %}
+                                        {% elif delivered %}
+                                            {% set overall_status = 'relayed' %}
+                                        {% else %}
+                                            {% set overall_status = 'failed' %}
+                                        {% endif %}
+                                        <td>
+                                            {% if overall_status == 'relayed' %}
                                                 <span class="badge bg-success">
                                                     <i class="bi bi-check-circle me-1"></i>
                                                     Sent
                                                 </span>
+                                            {% elif overall_status == 'partial' %}
+                                                <span class="badge bg-warning text-dark">
+                                                    <i class="bi bi-exclamation-triangle me-1"></i>
+                                                    Partial Fail
+                                                </span>
                                             {% else %}
                                                 <span class="badge bg-danger">
                                                     <i class="bi bi-x-circle me-1"></i>
@@ -170,6 +185,7 @@
                                                 </span>
                                             {% endif %}
                                         </td>
+                                    </td>
                                     <td>
                                         {% if email.dkim_signed %}
                                             <span class="text-success">
@@ -227,7 +243,7 @@
                                 </small>
                                 <br>
                                 <small class="text-muted">
-                                    {{ auth.created_at.strftime('%H:%M:%S') }}
+                                    {{ auth.created_at|format_datetime }}
                                 </small>
                             </div>
                             <small class="text-muted">

email_server/server_web_ui/templates/edit_ip.html

@@ -44,6 +44,18 @@
                         </div>
                     </div>
                     
+                    <div class="mb-3">
+                        <div class="form-check">
+                            <input class="form-check-input" type="checkbox" id="store_message_content" name="store_message_content" {% if ip_record.store_message_content %}checked{% endif %}>
+                            <label class="form-check-label" for="store_message_content">
+                                <strong>Store Full Message Content</strong>
+                            </label>
+                            <div class="form-text">
+                                If enabled, the full message body and attachments will be stored and viewable in logs. Otherwise, only headers and subject are stored.
+                            </div>
+                        </div>
+                    </div>
+                    
                     <div class="d-flex gap-2">
                         <button type="submit" class="btn btn-primary">
                             <i class="bi bi-check-lg me-1"></i>
@@ -95,6 +107,20 @@
                             </span>
                         {% endif %}
                     </dd>
+                    <dt class="col-sm-4">Store Message:</dt>
+                    <dd class="col-sm-8">
+                        {% if ip_record.store_message_content %}
+                            <span class="badge bg-info text-dark">
+                                <i class="bi bi-file-earmark-text me-1"></i>
+                                Full Message
+                            </span>
+                        {% else %}
+                            <span class="badge bg-secondary">
+                                <i class="bi bi-file-earmark me-1"></i>
+                                Headers Only
+                            </span>
+                        {% endif %}
+                    </dd>
                     <dt class="col-sm-4">Created:</dt>
                     <dd class="col-sm-8">
                         <small class="text-muted">

email_server/server_web_ui/templates/edit_sender.html

@@ -65,6 +65,18 @@
                         </div>
                     </div>
                     
+                    <div class="mb-3">
+                        <div class="form-check">
+                            <input class="form-check-input" type="checkbox" id="store_message_content" name="store_message_content" {% if sender.store_message_content %}checked{% endif %}>
+                            <label class="form-check-label" for="store_message_content">
+                                <strong>Store Full Message Content</strong>
+                            </label>
+                            <div class="form-text">
+                                If enabled, the full message body and attachments will be stored and viewable in logs. Otherwise, only headers and subject are stored.
+                            </div>
+                        </div>
+                    </div>
+                    
                     <div class="d-flex gap-2">
                         <button type="submit" class="btn btn-primary">
                             <i class="bi bi-check-lg me-1"></i>
@@ -130,6 +142,20 @@
                             </span>
                         {% endif %}
                     </dd>
+                    <dt class="col-sm-4">Store Message:</dt>
+                    <dd class="col-sm-8">
+                        {% if sender.store_message_content %}
+                            <span class="badge bg-info text-dark">
+                                <i class="bi bi-file-earmark-text me-1"></i>
+                                Full Message
+                            </span>
+                        {% else %}
+                            <span class="badge bg-secondary">
+                                <i class="bi bi-file-earmark me-1"></i>
+                                Headers Only
+                            </span>
+                        {% endif %}
+                    </dd>
                     <dt class="col-sm-4">Created:</dt>
                     <dd class="col-sm-8">
                         <small class="text-muted">

email_server/server_web_ui/templates/ips.html

@@ -33,6 +33,7 @@
                                         <th>IP Address</th>
                                         <th>Domain</th>
                                         <th>Status</th>
+                                        <th>Storage Type</th>
                                         <th>Added</th>
                                         <th>Actions</th>
                                     </tr>
@@ -59,6 +60,19 @@
                                                 </span>
                                             {% endif %}
                                         </td>
+                                        <td>
+                                            {% if ip.store_message_content %}
+                                                <span class="badge bg-info text-dark">
+                                                    <i class="bi bi-file-earmark-text me-1"></i>
+                                                    Stores Full Message
+                                                </span>
+                                            {% else %}
+                                                <span class="badge bg-secondary">
+                                                    <i class="bi bi-file-earmark me-1"></i>
+                                                    Headers Only
+                                                </span>
+                                            {% endif %}
+                                        </td>
                                         <td>
                                             <small class="text-muted">
                                                 {{ ip.created_at.strftime('%Y-%m-%d %H:%M') }}

email_server/server_web_ui/templates/logs.html

@@ -16,16 +16,9 @@
     .log-error { border-left-color: #dc3545; }
     .log-success { border-left-color: #198754; }
     .log-failed { border-left-color: #dc3545; }
+    .log-partial { border-left-color: #fd7e14; } /* Orange for partial fail */
     
-    .log-content {
+    /* Message display styles are now in view_message_content.html */
-        font-family: 'Courier New', monospace;
-        font-size: 0.875rem;
-        background-color: var(--bs-gray-100);
-        border-radius: 0.25rem;
-        padding: 0.5rem;
-        max-height: 150px;
-        overflow-y: auto;
-    }
 </style>
 {% endblock %}
 
@@ -83,11 +76,30 @@
                             {% for log_entry in logs %}
                                 {% if log_entry.type == 'email' %}
                                     {% set log = log_entry.data %}
-                                    <div class="log-entry log-email log-{{ 'success' if log.status == 'relayed' else 'failed' }}">
+                                    {% set recipients = log_entry.recipients %}
+                                    {% set delivered = recipients|selectattr('status', 'equalto', 'success')|list %}
+                                    {% set failed = recipients|selectattr('status', 'ne', 'success')|list %}
+                                    {% if delivered and failed %}
+                                        {% set overall_status = 'partial' %}
+                                    {% elif delivered %}
+                                        {% set overall_status = 'relayed' %}
+                                    {% else %}
+                                        {% set overall_status = 'failed' %}
+                                    {% endif %}
+                                    <div class="log-entry log-email log-{% if overall_status == 'relayed' %}success{% elif overall_status == 'partial' %}partial{% else %}failed{% endif %}">
                                         <div class="d-flex justify-content-between align-items-start mb-2">
                                             <div>
                                                 <span class="badge bg-primary me-2">EMAIL</span>
-                                                <strong>{{ log.mail_from }}</strong> → {{ log.rcpt_tos }}
+                                                <strong>{{ log.mail_from }}</strong>
+                                                {% if log.to_address %}
+                                                    → <span class="text-primary">To:</span> {{ log.to_address }}
+                                                {% endif %}
+                                                {% if log.cc_addresses %}
+                                                    <br><span class="ms-4 text-info">CC:</span> {{ log.cc_addresses }}
+                                                {% endif %}
+                                                {% if log.bcc_addresses %}
+                                                    <br><span class="ms-4 text-warning">BCC:</span> {{ log.bcc_addresses }}
+                                                {% endif %}
                                                 {% if log.dkim_signed %}
                                                     <span class="badge bg-success ms-2">
                                                         <i class="bi bi-shield-check me-1"></i>
@@ -95,13 +107,15 @@
                                                     </span>
                                                 {% endif %}
                                             </div>
-                                            <small class="text-muted">{{ log.created_at.strftime('%Y-%m-%d %H:%M:%S') }}</small>
+                                            <small class="text-muted">{{ log.timestamp.strftime('%Y-%m-%d %H:%M:%S') }}</small>
                                         </div>
                                         <div class="row">
                                             <div class="col-md-6">
                                                 <strong>Status:</strong> 
-                                                {% if log.status == 'relayed' %}
+                                                {% if overall_status == 'relayed' %}
                                                     <span class="text-success">Sent Successfully</span>
+                                                {% elif overall_status == 'partial' %}
+                                                    <span class="text-warning">Partial Fail</span>
                                                 {% else %}
                                                     <span class="text-danger">Failed</span>
                                                 {% endif %}
@@ -115,6 +129,11 @@
                                             <strong>Subject:</strong> {{ log.subject }}
                                         </div>
                                         {% endif %}
+                                        <div class="mt-2">
+                                            <a href="{{ url_for('email.view_message_content', log_id=log.id) }}" class="btn btn-sm btn-primary">
+                                                <i class="fas fa-envelope-open-text"></i> View Message Details
+                                            </a>
+                                        </div>
                                     </div>
                                 {% else %}
                                     {% set log = log_entry.data %}
@@ -127,7 +146,7 @@
                                                     {{ 'Success' if log.success else 'Failed' }}
                                                 </span>
                                             </div>
-                                            <small class="text-muted">{{ log.created_at.strftime('%Y-%m-%d %H:%M:%S') }}</small>
+                                            <small class="text-muted">{{ log.created_at|format_datetime }}</small>
                                         </div>
                                         <div class="row">
                                             <div class="col-md-6">
@@ -148,10 +167,28 @@
                         {% elif filter_type == 'emails' %}
                             <!-- Email logs only -->
                             {% for log in logs %}
-                                <div class="log-entry log-email log-{{ 'success' if log.status == 'relayed' else 'failed' }}">
+                                {% set delivered = recipient_logs_map[log.id]|selectattr('status', 'equalto', 'success')|list %}
+                                {% set failed = recipient_logs_map[log.id]|selectattr('status', 'ne', 'success')|list %}
+                                {% if delivered and failed %}
+                                    {% set overall_status = 'partial' %}
+                                {% elif delivered %}
+                                    {% set overall_status = 'relayed' %}
+                                {% else %}
+                                    {% set overall_status = 'failed' %}
+                                {% endif %}
+                                <div class="log-entry log-email log-{{ overall_status }}">
                                     <div class="d-flex justify-content-between align-items-start mb-2">
                                         <div>
-                                            <strong>{{ log.mail_from }}</strong> → {{ log.rcpt_tos }}
+                                            <strong>{{ log.mail_from }}</strong>
+                                            {% if log.to_address %}
+                                                → <span class="text-primary">To:</span> {{ log.to_address }}
+                                            {% endif %}
+                                            {% if log.cc_addresses %}
+                                                <br><span class="ms-4 text-info">CC:</span> {{ log.cc_addresses }}
+                                            {% endif %}
+                                            {% if log.bcc_addresses %}
+                                                <br><span class="ms-4 text-warning">BCC:</span> {{ log.bcc_addresses }}
+                                            {% endif %}
                                             {% if log.dkim_signed %}
                                                 <span class="badge bg-success ms-2">
                                                     <i class="bi bi-shield-check me-1"></i>
@@ -159,24 +196,64 @@
                                                 </span>
                                             {% endif %}
                                         </div>
-                                        <small class="text-muted">{{ log.created_at.strftime('%Y-%m-%d %H:%M:%S') }}</small>
+                                        <small class="text-muted">{{ log.timestamp.strftime('%Y-%m-%d %H:%M:%S') }}</small>
                                     </div>
                                     <div class="row">
                                         <div class="col-md-3">
                                             <strong>Status:</strong> 
-                                            {% if log.status == 'relayed' %}
+                                            {% if overall_status == 'relayed' %}
                                                 <span class="text-success">Sent</span>
+                                            {% elif overall_status == 'partial' %}
+                                                <span class="text-warning">Partial Fail</span>
                                             {% else %}
                                                 <span class="text-danger">Failed</span>
                                             {% endif %}
                                         </div>
                                         <div class="col-md-3">
-                                            <strong>Peer:</strong> <code>{{ log.peer }}</code>
+                                            <strong>Peer:</strong> <code>{{ log.peer_ip }}</code>
                                         </div>
                                         <div class="col-md-6">
                                             <strong>Message ID:</strong> <code>{{ log.message_id }}</code>
                                         </div>
                                     </div>
+                                    <div class="row mt-2">
+                                        <div class="col-md-4">
+                                            <strong>Username:</strong> {{ log.username or 'N/A' }}
+                                        </div>
+                                        <div class="col-md-4">
+                                            <strong>CC:</strong> {{ log.cc_addresses or 'None' }}
+                                        </div>
+                                        <div class="col-md-4">
+                                            <strong>BCC:</strong> {{ log.bcc_addresses or 'None' }}
+                                        </div>
+                                    </div>
+                                    {% if recipient_logs_map and log.id in recipient_logs_map and recipient_logs_map[log.id] %}
+                                    <div class="mt-2">
+                                        <strong>Recipient Delivery Results:</strong>
+                                        <ul class="list-group">
+                                            {% for r in recipient_logs_map[log.id] %}
+                                            <li class="list-group-item d-flex justify-content-between align-items-center">
+                                                <span>
+                                                    <strong>{{ r.recipient_type|upper }}:</strong> {{ r.recipient }}
+                                                    {% if r.status == 'success' %}
+                                                        <span class="badge bg-success ms-2">Delivered</span>
+                                                    {% else %}
+                                                        <span class="badge bg-danger ms-2">Failed</span>
+                                                    {% endif %}
+                                                </span>
+                                                {% if r.error_code or r.error_message %}
+                                                <span class="text-danger ms-2">
+                                                    {{ r.error_code }} {{ r.error_message }}
+                                                </span>
+                                                {% endif %}
+                                                {% if r.server_response %}
+                                                <span class="text-muted ms-2">{{ r.server_response }}</span>
+                                                {% endif %}
+                                            </li>
+                                            {% endfor %}
+                                        </ul>
+                                    </div>
+                                    {% endif %}
                                     {% if log.subject %}
                                     <div class="mt-2">
                                         <strong>Subject:</strong> {{ log.subject }}
@@ -195,6 +272,13 @@
                                         </div>
                                     </div>
                                     {% endif %}
+                                    {% if log.has_message_content %}
+                                    <div class="mt-2">
+                                        <a href="{{ url_for('email.view_message_content', log_id=log.id) }}" class="btn btn-outline-info btn-sm">
+                                            <i class="bi bi-file-earmark-text me-1"></i> View Full Message
+                                        </a>
+                                    </div>
+                                    {% endif %}
                                 </div>
                             {% endfor %}
                         {% else %}
@@ -208,7 +292,7 @@
                                                 {{ 'Success' if log.success else 'Failed' }}
                                             </span>
                                         </div>
-                                        <small class="text-muted">{{ log.created_at.strftime('%Y-%m-%d %H:%M:%S') }}</small>
+                                        <small class="text-muted">{{ log.created_at|format_datetime }}</small>
                                     </div>
                                     <div class="row">
                                         <div class="col-md-4">

email_server/server_web_ui/templates/senders.html

@@ -85,6 +85,7 @@
                             <th>Domain</th>
                             <th>Permissions</th>
                             <th>Status</th>
+                            <th>Storage</th>
                             <th>Created</th>
                             <th>Actions</th>
                         </tr>
@@ -128,6 +129,19 @@
                                     </span>
                                 {% endif %}
                             </td>
+                            <td>
+                                {% if sender.store_message_content %}
+                                    <span class="badge bg-info text-dark">
+                                        <i class="bi bi-file-earmark-text me-1"></i>
+                                        Stores Full Message
+                                    </span>
+                                {% else %}
+                                    <span class="badge bg-secondary">
+                                        <i class="bi bi-file-earmark me-1"></i>
+                                        Headers Only
+                                    </span>
+                                {% endif %}
+                            </td>
                             <td>
                                 <small class="text-muted">
                                     {{ sender.created_at.strftime('%Y-%m-%d %H:%M') }}

email_server/server_web_ui/templates/view_message_content.html

@@ -0,0 +1,98 @@
+{% extends "base.html" %}
+
+{% block title %}View Full Message - Email Log{% endblock %}
+
+{% block content %}
+<div class="container mt-4">
+    <h2>Full Message Content</h2>
+    <div class="mb-3">
+        <strong>From:</strong> {{ log.mail_from }}<br>
+        <strong>To:</strong> {{ log.to_address }}<br>
+        <strong>CC:</strong> {{ log.cc_addresses or 'None' }}<br>
+        <strong>BCC:</strong> {{ log.bcc_addresses or 'None' }}<br>
+        <strong>Subject:</strong> {{ log.subject or 'N/A' }}<br>
+        <strong>Date:</strong> {{ log.created_at.strftime('%Y-%m-%d %H:%M:%S') }}<br>
+    </div>
+
+    {% if log.attachments %}
+    <div class="card mb-3">
+        <div class="card-header">
+            <strong>Attachments:</strong>
+        </div>
+        <div class="card-body">
+            <ul class="list-group">
+            {% for attachment in log.attachments %}
+                <li class="list-group-item d-flex justify-content-between align-items-center">
+                    <div>
+                        <i class="fas fa-paperclip"></i> {{ attachment.filename }}
+                        <small class="text-muted">({{ attachment.size|filesizeformat }})</small>
+                    </div>
+                    <div class="btn-group" role="group">
+                        {% set content_type = attachment.content_type.lower() if attachment.content_type else 'application/octet-stream' %}
+                        {% set extension = attachment.filename.split('.')[-1].lower() if '.' in attachment.filename else '' %}
+                        
+                        {% set is_image = content_type.startswith('image/') or extension in ['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'] %}
+                        {% set is_text = content_type.startswith('text/') or extension in ['txt', 'log', 'json', 'xml', 'csv', 'md'] %}
+                        {% set is_pdf = content_type == 'application/pdf' or extension == 'pdf' %}
+                        {% set is_html = content_type in ['text/html', 'application/xhtml+xml'] or extension in ['html', 'htm'] %}
+                        
+                        {% if is_image or is_text or is_pdf or is_html %}
+                            <a href="{{ url_for('email.download_attachment', attachment_id=attachment.id) }}" 
+                               class="btn btn-sm btn-outline-primary" 
+                               target="_blank"
+                               data-bs-toggle="tooltip"
+                               title="Open in new tab">
+                               <i class="fas fa-external-link-alt"></i>
+                               {% if is_image %}<i class="fas fa-image"></i> View Image
+                               {% elif is_pdf %}<i class="fas fa-file-pdf"></i> View PDF
+                               {% elif extension == 'csv' %}<i class="fas fa-table"></i> View CSV
+                               {% elif is_text %}<i class="fas fa-file-alt"></i> View Text
+                               {% elif is_html %}<i class="fas fa-file-code"></i> View HTML
+                               {% else %}View in Browser
+                               {% endif %}
+                            </a>
+                        {% endif %}
+                        <a href="{{ url_for('email.download_attachment', attachment_id=attachment.id, download='true') }}" 
+                           class="btn btn-sm btn-outline-secondary"
+                           title="Download file">
+                           <i class="fas fa-download"></i> Download
+                        </a>
+                        <form method="POST" 
+                              action="{{ url_for('email.delete_attachment', attachment_id=attachment.id) }}" 
+                              style="display: inline;" 
+                              onsubmit="return confirm('Are you sure you want to delete this attachment?');">
+                            <button type="submit" 
+                                    class="btn btn-sm btn-outline-danger"
+                                    title="Delete attachment">
+                                    <i class="fas fa-trash-alt"></i> Delete
+                            </button>
+                        </form>
+                    </div>
+                </li>
+            {% endfor %}
+            </ul>
+        </div>
+    </div>
+    {% endif %}
+
+    <div class="card">
+        <div class="card-header">
+            <strong>Message Content:</strong>
+        </div>
+        <div class="card-body">
+            <pre style="white-space: pre-wrap; word-break: break-all;">{{ log.message_body }}</pre>
+        </div>
+    </div>
+
+    <div class="card mt-3">
+        <div class="card-header">
+            <strong>Message Headers:</strong>
+        </div>
+        <div class="card-body">
+            <pre style="white-space: pre-wrap;">{{ log.email_headers }}</pre>
+        </div>
+    </div>
+
+    <a href="{{ url_for('email.logs', type='emails') }}" class="btn btn-secondary mt-3">Back to Logs</a>
+</div>
+{% endblock %}

email_server/server_web_ui/utils.py

@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
 def get_public_ip() -> str:
     """Get the public IP address of the server."""
     try:
-        response1 = requests.get('https://ifconfig.me/ip', timeout=3, verify=False)
+        response1 = requests.get('http://ifconfig.me/ip', timeout=3, verify=False)
 
         ip = response1.text.strip()
         if ip and ip != 'unknown':
@@ -24,7 +24,7 @@ def get_public_ip() -> str:
     except Exception:
         try:
             # Fallback method
-            response = requests.get('https://httpbin.org/ip', timeout=3, verify=False)
+            response = requests.get('http://httpbin.org/ip', timeout=3, verify=False)
             ip = response.json()['origin'].split(',')[0].strip()
             if ip and ip != 'unknown':
                 return ip

email_server/server_web_ui/view_message.py

@@ -0,0 +1,131 @@
+"""
+Route to view full email message content if stored.
+"""
+from flask import render_template, abort, flash, redirect, Response, send_file, request, url_for
+from email_server.models import Session, EmailLog, EmailAttachment
+from email_server.tool_box import get_logger
+from .routes import email_bp
+import os
+
+logger = get_logger()
+
+@email_bp.route('/msg/content/<int:log_id>')
+def view_message_content(log_id):
+    """View the full message content for an email log if stored."""
+    session = Session()
+    try:
+        # Get log with attachments
+        log = session.query(EmailLog).filter_by(id=log_id).first()
+        if not log:
+            abort(404)
+            
+        # Get attachments for this log
+        attachments = session.query(EmailAttachment).filter_by(email_log_id=log_id).all()
+        log.attachments = attachments
+        
+        return render_template('view_message_content.html', log=log)
+    finally:
+        session.close()
+@email_bp.route('/msg/attachment/<int:attachment_id>/download')
+def download_attachment(attachment_id):
+    session = Session()
+    try:
+        attachment = session.query(EmailAttachment).get(attachment_id)
+        if not attachment or not os.path.isfile(attachment.file_path):
+            flash('Attachment not found.', 'danger')
+            return redirect(url_for('email.logs', type='emails'))
+
+        # Get the normalized content type and handle special cases
+        content_type = attachment.content_type.lower() if attachment.content_type else 'application/octet-stream'
+        extension = os.path.splitext(attachment.filename.lower())[1][1:] if '.' in attachment.filename else ''
+        
+        # Force download if requested
+        as_attachment = request.args.get('download', '').lower() == 'true'
+
+        # Map of extensions to content types for common files
+        content_type_map = {
+            'txt': 'text/plain',
+            'csv': 'text/csv',
+            'pdf': 'application/pdf',
+            'jpg': 'image/jpeg',
+            'jpeg': 'image/jpeg',
+            'png': 'image/png',
+            'gif': 'image/gif',
+            'svg': 'image/svg+xml',
+            'html': 'text/html',
+            'htm': 'text/html',
+            'json': 'application/json',
+            'xml': 'text/xml',
+            'md': 'text/markdown',
+        }
+
+        # Update content type based on file extension if needed
+        if content_type == 'application/octet-stream' and extension in content_type_map:
+            content_type = content_type_map[extension]
+
+        # Special handling for CSV files
+        if content_type == 'text/csv' and not as_attachment:
+            try:
+                with open(attachment.file_path, 'r') as f:
+                    csv_content = f.read()
+                # Create a simple HTML table view for CSV
+                html_content = '<html><head><style>'
+                html_content += 'table {border-collapse: collapse; width: 100%;} '
+                html_content += 'th, td {border: 1px solid #ddd; padding: 8px; text-align: left;} '
+                html_content += 'tr:nth-child(even) {background-color: #f2f2f2;} '
+                html_content += 'th {background-color: #4CAF50; color: white;}'
+                html_content += '</style></head><body><table>'
+                
+                # Convert CSV to HTML table
+                for i, line in enumerate(csv_content.split('\n')):
+                    if not line.strip():
+                        continue
+                    html_content += '<tr>'
+                    if i == 0:  # Header row
+                        html_content += ''.join(f'<th>{cell}</th>' for cell in line.split(','))
+                    else:
+                        html_content += ''.join(f'<td>{cell}</td>' for cell in line.split(','))
+                    html_content += '</tr>'
+                
+                html_content += '</table></body></html>'
+                return Response(html_content, mimetype='text/html')
+            except Exception as e:
+                logger.warning(f"Failed to create CSV preview: {e}")
+                # Fall back to normal file handling
+
+        # Determine if the file should be viewed in browser
+        if as_attachment:
+            # Force download
+            return send_file(
+                attachment.file_path,
+                as_attachment=True,
+                download_name=attachment.filename
+            )
+        else:
+            # Try to display in browser
+            return send_file(
+                attachment.file_path,
+                mimetype=content_type
+            )
+    finally:
+        session.close()
+
+
+@email_bp.route('/msg/attachment/<int:attachment_id>/delete', methods=['POST', 'GET'])
+def delete_attachment(attachment_id):
+    session = Session()
+    try:
+        attachment = session.query(EmailAttachment).get(attachment_id)
+        if not attachment:
+            flash('Attachment not found.', 'danger')
+            return redirect(url_for('email.logs', type='emails'))
+        # Remove file from disk
+        if os.path.isfile(attachment.file_path):
+            os.remove(attachment.file_path)
+        # Remove from DB
+        session.delete(attachment)
+        session.commit()
+        flash('Attachment deleted.', 'success')
+        return redirect(url_for('email.logs', type='emails'))
+    finally:
+        session.close()

email_server/settings_loader.py

@@ -24,6 +24,8 @@ DEFAULTS = {
         'BIND_IP': '0.0.0.0',
         '; Custom server banner (to make it empty use "" must be double quotes)': None,
         'server_banner': "",
+        '; Time zone for the server': None,
+        'TIME_ZONE': 'Europe/London',
     },
     'Database': {
         '; Database configuration': None,

email_server/smtp_handler.py

@@ -9,23 +9,27 @@ Security Features:
 """
 
 import uuid
-from datetime import datetime
+import email.utils
+import os
 from aiosmtpd.smtp import SMTP as AIOSMTP, AuthResult
 from aiosmtpd.controller import Controller
-from email_server.auth import EnhancedAuthenticator, EnhancedIPAuthenticator, validate_sender_authorization, get_authenticated_domain_id
+from email_server.auth import EnhancedAuthenticator, EnhancedIPAuthenticator, validate_sender_authorization
 from email_server.email_relay import EmailRelay
 from email_server.dkim_manager import DKIMManager
 from email_server.settings_loader import load_settings
-from email_server.tool_box import get_logger
+from email_server.tool_box import get_logger, ensure_folder_exists
+from email import policy
+from email.parser import BytesParser
+from email_server.models import Session, EmailAttachment, EmailLog
 
 logger = get_logger()
+settings = load_settings()
 
 class CustomSMTP(AIOSMTP):
     """Custom SMTP class with configurable banner and secure AUTH handling."""
     
     def __init__(self, *args, **kwargs):
         # Sets Custom SMTP banner from settings
-        settings = load_settings()
         _banner_message = settings['Server'].get('server_banner', '')
         if _banner_message == '""':
             _banner_message = ''
@@ -97,57 +101,41 @@ class EnhancedCustomSMTPHandler:
         self.auth_methods = ['LOGIN', 'PLAIN']
 
     def _ensure_required_headers(self, content: str, envelope, message_id: str, custom_headers: list = None) -> str:
-        """Ensure all required email headers are present and properly formatted.
+        """Ensure all required email headers are present and properly formatted."""
-        
-        Following RFC 5322 header order and best practices for spam score reduction.
-        Optimized based on Gmail's header structure for better deliverability.
-        
-        Args:
-            content (str): Email content.
-            envelope: SMTP envelope.
-            message_id (str): Generated message ID.
-            custom_headers (list): List of (name, value) tuples for custom headers.
-            
-        Returns:
-            str: Email content with all required headers properly formatted.
-        """
-        import email.utils
-        from email_server.settings_loader import load_settings
-        
         try:
-            settings = load_settings()
+            lines = content.splitlines()
-            fallback_hostname = settings.get('Server', 'HOSTNAME', fallback='localhost')
+            for idx, line in enumerate(lines):
-            server_hostname = settings.get('Server', 'helo_hostname', fallback=fallback_hostname)
+                if not isinstance(line, str):
-            
+                    logger.error(f"_ensure_required_headers: Non-string line at index {idx}: {type(line)}: {line}")
-            logger.debug(f"Processing headers for message {message_id}")
+                    logger.error(f"_ensure_required_headers: Full content object: {repr(content)}")
-            
+                    raise TypeError(f"_ensure_required_headers: Non-string line in content.splitlines(): {type(line)} at index {idx}")
-            # Parse the message properly
-            if isinstance(content, bytes):
-                content = content.decode('utf-8', errors='replace')
-            
-            # Split content into lines and normalize line endings
-            lines = content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
-            
             # Find header/body boundary and collect existing headers
             body_start = 0
             existing_headers = {}
             original_header_order = []
-            
             for i, line in enumerate(lines):
                 if line.strip() == '':
                     body_start = i + 1
                     break
+                if not isinstance(line, str):
+                    logger.error(f"_ensure_required_headers: Header line is not a string: {type(line)}: {line}")
+                    continue
                 if ':' in line and not line.startswith((' ', '\t')):
+                    try:
                         header_name, header_value = line.split(':', 1)
+                    except Exception as e:
+                        logger.error(f"_ensure_required_headers: Failed to split header line: {line} - {e}")
+                        continue
+                    if not isinstance(header_name, str) or not isinstance(header_value, str):
+                        logger.error(f"_ensure_required_headers: Non-string header_name or header_value: {type(header_name)}, {type(header_value)}: {header_name}, {header_value}")
+                        continue
                     header_name_lower = header_name.strip().lower()
                     header_value = header_value.strip()
-                    
                     # Handle continuation lines
                     j = i + 1
                     while j < len(lines) and lines[j].startswith((' ', '\t')):
                         header_value += ' ' + lines[j].strip()
                         j += 1
-                    
                     existing_headers[header_name_lower] = header_value
                     original_header_order.append((header_name.strip(), header_value))
                     logger.debug(f"Found existing header: {header_name_lower} = {header_value}")
@@ -165,8 +153,12 @@ class EnhancedCustomSMTPHandler:
             if 'message-id' in existing_headers:
                 required_headers.append(f"Message-ID: {existing_headers['message-id']}")
             else:
-                domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else server_hostname.replace('mail.', '')
+                # Use helo_hostname from settings for FQDN in Message-ID
-                required_headers.append(f"Message-ID: <{message_id}@{domain}>")
+                helo_hostname = settings['Server'].get('helo_hostname', 'localhost')
+                if not helo_hostname:
+                    # fallback to domain from mail_from
+                    helo_hostname = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else 'localhost'
+                required_headers.append(f"Message-ID: <{message_id}@{helo_hostname}>")
             
             # 2. Date (critical for spam filters)
             if 'date' in existing_headers:
@@ -181,75 +173,49 @@ class EnhancedCustomSMTPHandler:
             else:
                 required_headers.append("MIME-Version: 1.0")
             
-            # 4. User-Agent (if present, helps with reputation)
+            # 4. To (primary recipients - critical)
-            if 'user-agent' in existing_headers:
-                required_headers.append(f"User-Agent: {existing_headers['user-agent']}")
-            
-            # 5. Content-Language (if present)
-            if 'content-language' in existing_headers:
-                required_headers.append(f"Content-Language: {existing_headers['content-language']}")
-            
-            # 6. To (primary recipients - critical)
             if 'to' in existing_headers:
                 required_headers.append(f"To: {existing_headers['to']}")
             else:
-                to_list = ', '.join(envelope.rcpt_tos)
+                required_headers.append(f"To: {', '.join([rcpt for rcpt in envelope.rcpt_tos])}")
-                required_headers.append(f"To: {to_list}")
 
-            # 7. From (sender identification - critical)
+            # 5. Cc (if present)
+            if 'cc' in existing_headers:
+                required_headers.append(f"Cc: {existing_headers['cc']}")
+            
+            # 6. From (sender identification - critical)
             if 'from' in existing_headers:
                 required_headers.append(f"From: {existing_headers['from']}")
             else:
                 required_headers.append(f"From: {envelope.mail_from}")
             
-            # 8. Subject (message topic - critical)
+            # 7. Subject (message topic - critical)
             if 'subject' in existing_headers:
                 required_headers.append(f"Subject: {existing_headers['subject']}")
             else:
                 required_headers.append("Subject: ")
             
-            # 9. Content-Type (media type information)
+            # 8. Content-Type (media type information)
             if 'content-type' in existing_headers:
                 required_headers.append(f"Content-Type: {existing_headers['content-type']}")
             else:
                 required_headers.append("Content-Type: text/plain; charset=UTF-8; format=flowed")
             
-            # 10. Content-Transfer-Encoding
+            # 9. Content-Transfer-Encoding
             if 'content-transfer-encoding' in existing_headers:
                 required_headers.append(f"Content-Transfer-Encoding: {existing_headers['content-transfer-encoding']}")
             else:
                 required_headers.append("Content-Transfer-Encoding: 7bit")
 
-            # Add custom headers after essential headers but before misc headers
+            # Add custom headers after essential headers
             if custom_headers:
                 for header_name, header_value in custom_headers:
-                    # Skip if already added in essential headers
+                    header_name_lower = header_name.lower()
-                    if header_name.lower() not in ['message-id', 'date', 'mime-version', 'user-agent',
+                    # Skip if header already exists
-                                                  'content-language', 'to', 'from', 'subject', 
+                    if header_name_lower not in existing_headers:
-                                                  'content-type', 'content-transfer-encoding']:
                         required_headers.append(f"{header_name}: {header_value}")
                         logger.debug(f"Added custom header: {header_name}: {header_value}")
 
-            # Add any other existing headers that weren't handled above
-            essential_headers = {
-                'message-id', 'date', 'from', 'to', 'subject', 
-                'mime-version', 'content-type', 'content-transfer-encoding',
-                'user-agent', 'content-language'
-            }
-            
-            # Preserve original header names and values for non-essential headers
-            for header_name, header_value in original_header_order:
-                if header_name.lower() not in essential_headers:
-                    # Skip custom headers we already added
-                    skip = False
-                    if custom_headers:
-                        for custom_name, _ in custom_headers:
-                            if header_name.lower() == custom_name.lower():
-                                skip = True
-                                break
-                    if not skip:
-                        required_headers.append(f"{header_name}: {header_value}")
-            
             # Build final message
             final_content = '\r\n'.join(required_headers)
             if body.strip():
@@ -257,30 +223,43 @@ class EnhancedCustomSMTPHandler:
             else:
                 final_content += '\r\n\r\n'
             
-            logger.debug(f"Final headers for message {message_id}:")
-            for header in required_headers:
-                logger.debug(f"  {header}")
-            
             return final_content
             
         except Exception as e:
-            logger.error(f"Error ensuring headers: {e}")
             import traceback
+            logger.error(f"Error ensuring headers: {e}")
             logger.error(f"Traceback: {traceback.format_exc()}")
+            logger.error(f"Locals: {locals()}")
             # Fallback to original content if parsing fails
             return content
 
     async def handle_DATA(self, server, session, envelope):
-        """Handle incoming email data with improved header management."""
+        """Handle incoming email data with improved header management and logging."""
         try:
             message_id = str(uuid.uuid4())
             logger.debug(f'Received email {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}')
 
+            # Get authenticated username from session
+            username = getattr(session, 'username', None)
+            if not username:
+                # Check if IP authentication was used
+                client_ip = getattr(session, 'peer', ['unknown'])[0].split(':')[0] if hasattr(session, 'peer') else None
+                if client_ip:
+                    from email_server.models import get_whitelisted_ip
+                    sender_domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else None
+                    ip_auth = get_whitelisted_ip(client_ip, sender_domain)
+                    if ip_auth:
+                        username = f"IP:{client_ip}"
+            
+            logger.debug(f'Authenticated username: {username}')
+
             # Convert content to string if it's bytes
             if isinstance(envelope.content, bytes):
                 content = envelope.content.decode('utf-8', errors='replace')
+                raw_bytes = envelope.content
             else:
                 content = envelope.content
+                raw_bytes = envelope.content.encode('utf-8', errors='replace')
 
             # Extract domain from sender for DKIM signing
             sender_domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else None
@@ -293,13 +272,8 @@ class EnhancedCustomSMTPHandler:
             # Add beneficial headers for spam score improvement
             client_ip = getattr(session, 'peer', ['unknown'])[0] if hasattr(session, 'peer') else None
             if client_ip:
-                # Add X-Originating-IP header (helps with reputation)
                 custom_headers.append(('X-Originating-IP', f'[{client_ip}]'))
-            
-            # Add X-Mailer header for identification
             custom_headers.append(('X-Mailer', 'NetBro Mail Server 1.0'))
-            
-            # Add X-Priority header (normal priority)
             custom_headers.append(('X-Priority', '3'))
 
             # Ensure required headers are present (including custom headers)
@@ -310,38 +284,260 @@ class EnhancedCustomSMTPHandler:
             dkim_signed = False
             if sender_domain:
                 signed_content = self.dkim_manager.sign_email(content, sender_domain)
+                if not isinstance(signed_content, (str, bytes)):
+                    logger.error(f"DKIMManager.sign_email returned non-str/bytes: {type(signed_content)}: {signed_content}")
+                    raise TypeError(f"DKIMManager.sign_email returned non-str/bytes: {type(signed_content)}")
                 dkim_signed = signed_content != content
                 if dkim_signed:
                     logger.debug(f'Email {message_id} signed with DKIM for domain {sender_domain}')
 
-            # Relay the email (no further modifications allowed)
+            # Extract headers for logging
-            success = self.email_relay.relay_email(
+            to_address = ''
+            cc_addresses = ''
+            bcc_addresses = ''
+            subject = ''
+            split_lines = content.splitlines()
+            for idx, line in enumerate(split_lines):
+                if not isinstance(line, str):
+                    logger.error(f"DIAGNOSTIC: Non-string line at index {idx}: {type(line)}: {line}")
+                    logger.error(f"DIAGNOSTIC: Full content object: {repr(content)}")
+                    raise TypeError(f"DIAGNOSTIC: Non-string line in content.splitlines(): {type(line)} at index {idx}")
+            try:
+
+                for line in split_lines:
+                    if line.strip() == '':
+                        break
+                    if not isinstance(line, str):
+                        logger.error(f"Header line is not a string: {type(line)}: {line}")
+                        continue
+                    try:
+                        lower_line = line.lower()
+                    except Exception as e:
+                        logger.error(f"Failed to call lower() on line: {line} (type: {type(line)}) - {e}")
+                        import traceback
+                        logger.error(traceback.format_exc())
+                        logger.error(f"Full content.splitlines(): {split_lines}")
+                        continue
+                    if lower_line.startswith('to:'):
+                        to_address = line[3:].strip()
+                    elif lower_line.startswith('cc:'):
+                        cc_addresses = line[3:].strip()
+                    elif lower_line.startswith('subject:'):
+                        subject = line[8:].strip()
+            except Exception as e:
+                logger.error(f"Exception in header extraction loop: {e}")
+                import traceback
+                logger.error(traceback.format_exc())
+                logger.error(f"Full content.splitlines(): {split_lines}")
+
+            # Check if message content should be stored (sender or IP whitelist)
+            from email_server.models import get_sender_by_email, get_whitelisted_ip
+            store_message = False
+            sender_obj = get_sender_by_email(envelope.mail_from)
+            if sender_obj and getattr(sender_obj, 'store_message_content', False):
+                store_message = True
+            elif client_ip:
+                domain_name = sender_domain
+                ip_obj = get_whitelisted_ip(client_ip, domain_name)
+                if ip_obj and getattr(ip_obj, 'store_message_content', False):
+                    store_message = True
+
+            attachments_to_save = []
+            # Get attachments path from settings
+            attachments_path = settings['Attachments'].get('attachments_path', 'email_server/server_data/attachments')
+            saved_attachments = []
+            logger.debug(f"Using attachments base path: {attachments_path}")
+            email_log_id = None
+
+            if store_message:
+                # Parse the message for attachments using the email library
+                msg = BytesParser(policy=policy.default).parsebytes(raw_bytes)
+                if msg.is_multipart():
+                    # Get storage path for this sender
+                    storage_path = self.get_attachment_storage_path(
+                        attachments_base_path=attachments_path,
+                        sender_domain=sender_domain,
+                        username=username,
+                        client_ip=client_ip
+                    )
+                    ensure_folder_exists(storage_path)
+                    
+                    for part in msg.walk():
+                        content_disposition = part.get_content_disposition()
+                        if content_disposition == 'attachment':
+                            filename = part.get_filename()
+                            if not filename:
+                                continue
+                                
+                            # Get file data and validate
+                            file_data = part.get_payload(decode=True)
+                            if not file_data:
+                                continue
+                                
+                            # Get proper content type
+                            content_type = self.get_content_type(part, filename)
+                            size = len(file_data)
+                            
+                            # Build a unique file path
+                            safe_filename = f"{message_id}_{filename}"
+                            file_path = os.path.join(storage_path, safe_filename)
+                            
+                            try:
+                                # Save the file
+                                with open(file_path, 'wb') as f:
+                                    f.write(file_data)
+                                logger.debug(f"Saved attachment {filename} ({content_type}) to {file_path}")
+                                    
+                                attachments_to_save.append({
+                                    'filename': filename,
+                                    'content_type': content_type,
+                                    'file_path': file_path,
+                                    'size': size
+                                })
+                            except Exception as e:
+                                logger.error(f"Failed to save attachment {filename}: {str(e)}")
+                                continue
+
+            # Parse addresses to determine recipient types
+            def parse_addresses(addr_str):
+                if not isinstance(addr_str, str):
+                    logger.warning(f"Expected string for address header, got {type(addr_str)}: {addr_str}")
+                    return []
+                
+                return [addr.strip().lower() for addr in addr_str.split(',') if isinstance(addr, str) and addr.strip()]
+            
+            to_list = parse_addresses(to_address)
+            cc_list = parse_addresses(cc_addresses)
+            
+            # Map recipients to their types based on headers
+            recipient_type_map = {}
+            for rcpt in envelope.rcpt_tos:
+                if not isinstance(rcpt, str):
+                    logger.warning(f"Expected string for recipient, got {type(rcpt)}: {rcpt}")
+                    continue
+                rcpt_l = rcpt.lower()
+                if rcpt_l in to_list:
+                    recipient_type_map[rcpt] = 'to'
+                elif rcpt_l in cc_list:
+                    recipient_type_map[rcpt] = 'cc'
+                else:
+                    recipient_type_map[rcpt] = 'bcc'  # Any recipient not in To/Cc is a Bcc
+
+            # Build recipient results
+            recipient_results = []
+            recipient_types = []
+            for rcpt in envelope.rcpt_tos:
+                rtype = recipient_type_map[rcpt]
+                recipient_results.append({'recipient': rcpt, 'recipient_type': rtype, 'status': 'pending'})
+                recipient_types.append(rtype)
+
+            # Relay the email and get per-recipient results
+            relay_results = await self.email_relay.relay_email_async(
                 envelope.mail_from,
                 envelope.rcpt_tos,
-                signed_content
+                signed_content,
+                username=username,
+                cc_addresses=cc_addresses,
+                bcc_addresses=None,  # BCC addresses are handled through envelope.rcpt_tos
+                recipient_types=recipient_types
             )
 
-            # Log the email
+            # Update status in recipient_results
-            status = 'relayed' if success else 'failed'
+            for result in relay_results:
+                for r in recipient_results:
+                    if r['recipient'] == result['recipient'] and r['recipient_type'] == result.get('recipient_type', 'to'):
+                        r.update(result)
+                        break
+
+            # Determine overall status
+            status = 'relayed' if all(r['status'] == 'success' for r in recipient_results) else 'failed'
+
+            # Extract headers and parse message content
+            msg = BytesParser(policy=policy.default).parsebytes(raw_bytes)
+            
+            # Extract headers
+            email_headers = []
+            for name, value in msg.items():
+                email_headers.append(f"{name}: {value}")
+            email_headers = '\n'.join(email_headers)
+
+            # Extract only the text content, not attachments
+            message_body = ""
+            if msg.is_multipart():
+                for part in msg.walk():
+                    if part.get_content_maintype() == 'text' and part.get_content_disposition() is None:
+                        # This is likely the main message text
+                        charset = part.get_content_charset() or 'utf-8'
+                        try:
+                            part_content = part.get_payload(decode=True).decode(charset)
+                            message_body += part_content + "\n"
+                        except Exception as e:
+                            logger.warning(f"Failed to decode message part: {e}")
+            else:
+                # Not multipart - if it's text, use it as is
+                if msg.get_content_maintype() == 'text':
+                    charset = msg.get_content_charset() or 'utf-8'
+                    try:
+                        message_body = msg.get_payload(decode=True).decode(charset)
+                    except Exception as e:
+                        logger.warning(f"Failed to decode message: {e}")
+
+            # Trim any extra whitespace
+            message_body = message_body.strip()
+
+            # Get client IP without port
+            client_ip = getattr(session, 'peer', ['unknown'])[0].split(':')[0] if hasattr(session, 'peer') else 'unknown'
+
+            # Log the email with all details
             self.email_relay.log_email(
                 message_id=message_id,
-                peer=session.peer,
+                peer=client_ip,
                 mail_from=envelope.mail_from,
-                rcpt_tos=envelope.rcpt_tos,
+                to_address=to_address,
-                content=content,  # Log original content, not signed
+                cc_addresses=cc_addresses,
+                bcc_addresses=', '.join([r['recipient'] for r in recipient_results if r['recipient_type'] == 'bcc']),
+                subject=subject,
+                email_headers=email_headers,
+                message_body=message_body,
                 status=status,
-                dkim_signed=dkim_signed
+                dkim_signed=dkim_signed,
+                username=username,
+                recipient_results=recipient_results
             )
 
-            if success:
+            # Save attachments to DB, linked to the correct EmailLog
+            if attachments_to_save:
+                db_session = Session()
+                try:
+                    email_log = db_session.query(EmailLog).filter_by(message_id=message_id).first()
+                    if email_log:
+                        for att in attachments_to_save:
+                            attachment = EmailAttachment(
+                                email_log_id=email_log.id,
+                                filename=att['filename'],
+                                content_type=att['content_type'],
+                                file_path=att['file_path'],
+                                size=att['size']
+                            )
+                            db_session.add(attachment)
+                        db_session.commit()
+                except Exception as e:
+                    logger.error(f"Failed to save attachments to DB: {e}")
+                    db_session.rollback()
+                finally:
+                    db_session.close()
+
+            if status == 'relayed':
                 logger.debug(f'Email {message_id} successfully relayed')
                 return '250 Message accepted for delivery'
             else:
                 logger.error(f'Email {message_id} failed to relay')
                 return '550 Message relay failed'
-                
         except Exception as e:
+            import traceback
             logger.error(f'Error handling email: {e}')
+            logger.error(f'Traceback: {traceback.format_exc()}')
+            logger.error(f'Locals: {locals()}')
             return '550 Internal server error'
 
     async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
@@ -372,12 +568,73 @@ class EnhancedCustomSMTPHandler:
         logger.info(f'MAIL FROM accepted: {address} - {message}')
         return '250 OK'
 
+    def get_attachment_storage_path(self, attachments_base_path: str, sender_domain: str, username: str = None, client_ip: str = None) -> str:
+        """Generate the storage path for attachments based on sender domain and authentication.
+        
+        Args:
+            attachments_base_path: Base path for attachments storage
+            sender_domain: Domain of the sender
+            username: Authenticated username (if any)
+            client_ip: Client IP address (if IP-based authentication)
+            
+        Returns:
+            str: Full path where attachments should be stored
+        """
+        # Sanitize domain name for folder name
+        safe_domain = sender_domain.replace('/', '_').replace('\\', '_')
+        domain_path = os.path.join(attachments_base_path, safe_domain)
+        
+        # Determine subfolder based on authentication
+        if username:
+            # Sanitize username for folder name
+            safe_username = username.replace('/', '_').replace('\\', '_')
+            return os.path.join(domain_path, safe_username)
+        elif client_ip:
+            # Sanitize IP for folder name
+            safe_ip = client_ip.replace(':', '_')
+            return os.path.join(domain_path, safe_ip)
+        else:
+            # Fallback to domain-only path
+            return domain_path
+
+    def get_content_type(self, part, filename):
+        """Get the correct content type for a file, trying multiple methods."""
+        import mimetypes
+        
+        # First try the part's content type
+        content_type = part.get_content_type()
+        
+        # If it's octet-stream, try to guess from filename
+        if content_type == 'application/octet-stream':
+            guessed_type, _ = mimetypes.guess_type(filename)
+            if guessed_type:
+                content_type = guessed_type
+            else:
+                # Use specific types for common extensions
+                ext = filename.lower().split('.')[-1] if '.' in filename else ''
+                type_map = {
+                    'txt': 'text/plain',
+                    'csv': 'text/csv',
+                    'jpg': 'image/jpeg',
+                    'jpeg': 'image/jpeg',
+                    'png': 'image/png',
+                    'gif': 'image/gif',
+                    'pdf': 'application/pdf',
+                    'json': 'application/json',
+                    'xml': 'application/xml',
+                    'html': 'text/html',
+                    'htm': 'text/html',
+                }
+                content_type = type_map.get(ext, 'application/octet-stream')
+                
+        return content_type
+
 class TLSController(Controller):
     """
     Custom controller for direct TLS (SMTPS, port 465) support.
     """
     
-    def __init__(self, handler, ssl_context, hostname='localhost', port=40587):
+    def __init__(self, handler, ssl_context, hostname='localhost', port=40465):
         logger.debug(f"TLSController __init__: ssl_context={ssl_context is not None}")
         self._ssl_context = ssl_context  # Use private attribute to avoid conflicts
         self.smtp_hostname = hostname  # Store for HELO identification

email_server/tool_box.py

@@ -5,6 +5,8 @@ Utility functions for the email server.
 import os
 import logging
 from email_server.settings_loader import load_settings
+from datetime import datetime
+import pytz
 
 settings = load_settings()
 
@@ -57,3 +59,8 @@ def get_logger(name=None):
         else:
             name = '__main__'
     return logging.getLogger(name)
+
+def get_current_time():
+    """Get current time with timezone from settings."""
+    timezone = pytz.timezone(settings['Server'].get('time_zone', 'UTC'))
+    return datetime.now(timezone)

migrations/add_email_attachments_table.sql

@@ -0,0 +1,11 @@
+-- Migration: Add EmailAttachment table for storing email attachments on disk
+CREATE TABLE IF NOT EXISTS esrv_email_attachments (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    email_log_id INTEGER NOT NULL,
+    filename TEXT NOT NULL,
+    content_type TEXT,
+    file_path TEXT NOT NULL,
+    size INTEGER,
+    uploaded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(email_log_id) REFERENCES esrv_email_logs(id) ON DELETE CASCADE
+);

requirements.txt

@@ -12,6 +12,7 @@ bcrypt
 dnspython
 dkimpy
 cryptography
+aiosmtplib
 
 # Web Frontend Dependencies
 Flask
@@ -19,7 +20,7 @@ Flask-SQLAlchemy
 Jinja2
 Werkzeug
 requests
-Flask-Migrate
+pytz
 gunicorn
 
 # Additional utilities

tests/bash_send_email.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# apt-get install -y swaks
 receiver="info@example.com"
 EMAIL_SERVER="localhost"   #"pymta.example.com" "localhost"
 EMAIL_SERVER_auth="10.100.111.1"   # IP for authenticated server ( not localhost), use your main interface ip
@@ -14,17 +15,12 @@ cc_recipient="ccrecipient@example.com"
 bcc_recipient="bccrecipient@example.com"
 
 <<com
-python -m email_server.cli_tools add-domain $domain
-python -m email_server.cli_tools add-user $username $password $domain
-python -m email_server.cli_tools add-ip 127.0.0.1 $domain
-python -m email_server.cli_tools add-ip 213.249.224.235 $domain
-python -m email_server.cli_tools generate-dkim $domain
-python -m email_server.cli_tools add-custom-header $domain X-Auth-Token "abc123-example-auth"
-python -m email_server.cli_tools add-custom-header $domain X-Server-ID "mail01.example.com"
 
 # options to add CC and BCC recipients for swaks
-      --cc $cc_recipient
+      --cc $cc_recipient \
-      --bcc $bcc_recipient
+      --bcc $bcc_recipient \
+      --header "To: $receiver" \
+      --header "Cc: $cc_recipient" \
 
 swaks --to $receiver \
       --from $sender \

tests/send_email.py

@@ -4,7 +4,7 @@ import ssl
 import argparse
 parser = argparse.ArgumentParser()
 parser.add_argument('--port', type=int, default=4025)
-parser.add_argument('--porttls', type=int, default=40587)
+parser.add_argument('--porttls', type=int, default=40465)
 parser.add_argument('--recipient', type=str, default="test@target-email.com")
 args = parser.parse_args()
 

tests/test_debug_db.py

@@ -1,53 +0,0 @@
-"""
-Debug script to test database operations.
-"""
-
-import sys
-import os
-import sqlite3
-
-# Add current directory to path
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-print("Testing database operations...")
-
-# Test direct SQLite connection
-try:
-    conn = sqlite3.connect('smtp_server.db')
-    cursor = conn.cursor()
-    
-    # Check tables
-    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
-    tables = cursor.fetchall()
-    print(f"Tables in database: {[table[0] for table in tables]}")
-    
-    # Check domains
-    cursor.execute("SELECT * FROM domains;")
-    domains = cursor.fetchall()
-    print(f"Domains: {domains}")
-    
-    conn.close()
-    print("Direct SQLite test successful")
-    
-except Exception as e:
-    print(f"Direct SQLite test failed: {e}")
-
-# Test SQLAlchemy models
-try:
-    from email_server.models import Session, Domain, User, WhitelistedIP, create_tables
-    print("Models imported successfully")
-    
-    # Create session
-    session = Session()
-    
-    # Check domains
-    domains = session.query(Domain).all()
-    print(f"SQLAlchemy domains: {[(d.id, d.domain_name) for d in domains]}")
-    
-    session.close()
-    print("SQLAlchemy test successful")
-    
-except Exception as e:
-    print(f"SQLAlchemy test failed: {e}")
-    import traceback
-    traceback.print_exc()