improved helo_hostname for better spam score

email_server/email_relay.py

@@ -7,6 +7,7 @@ import smtplib
 import ssl
 from datetime import datetime
 from email_server.models import Session, EmailLog
+from email_server.settings_loader import load_settings
 from email_server.tool_box import get_logger
 
 logger = get_logger()
@@ -16,6 +17,11 @@ class EmailRelay:
     
     def __init__(self):
         self.timeout = 30  # Increased timeout for TLS negotiations
+        # Get the configured hostname for HELO/EHLO identification
+        settings = load_settings()
+        self.hostname = settings['Server'].get('helo_hostname', 
+                                             settings['Server'].get('hostname', 'localhost'))
+        logger.debug(f"EmailRelay initialized with hostname: {self.hostname}")
     
     def relay_email(self, mail_from, rcpt_tos, content):
         """Relay email to recipient's mail server with opportunistic TLS."""
@@ -53,8 +59,9 @@ class EmailRelay:
                     
                     # Try to enable TLS if the server supports it
                     try:
-                        # Check if server supports STARTTLS
+                        # Check if server supports STARTTLS - use proper hostname for EHLO
-                        relay_server.ehlo()
+                        logger.debug(f'Sending EHLO {self.hostname} to {mx_host}')
+                        relay_server.ehlo(self.hostname)
                         if relay_server.has_extn('starttls'):
                             logger.debug(f'Starting TLS connection to {mx_host}')
                             context = ssl.create_default_context()
@@ -62,7 +69,8 @@ class EmailRelay:
                             context.check_hostname = False
                             context.verify_mode = ssl.CERT_NONE
                             relay_server.starttls(context=context)
-                            relay_server.ehlo()  # Say hello again after STARTTLS
+                            logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to {mx_host}')
+                            relay_server.ehlo(self.hostname)  # Say hello again after STARTTLS with proper hostname
                             logger.debug(f'TLS connection established to {mx_host}')
                         else:
                             logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text')
@@ -94,13 +102,15 @@ class EmailRelay:
                                 
                                 # Try TLS with backup server too
                                 try:
-                                    backup_server.ehlo()
+                                    logger.debug(f'Sending EHLO {self.hostname} to backup {backup_mx}')
+                                    backup_server.ehlo(self.hostname)
                                     if backup_server.has_extn('starttls'):
                                         context = ssl.create_default_context()
                                         context.check_hostname = False
                                         context.verify_mode = ssl.CERT_NONE
                                         backup_server.starttls(context=context)
-                                        backup_server.ehlo()
+                                        logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to backup {backup_mx}')
+                                        backup_server.ehlo(self.hostname)
                                         logger.debug(f'TLS connection established to backup {backup_mx}')
                                 except Exception:
                                     logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text')

email_server/server_runner.py

@@ -9,7 +9,7 @@ from email_server.tool_box import get_logger
 
 # Import our modules
 from email_server.models import create_tables
-from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController
+from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController, TLSController
 from email_server.tls_utils import generate_self_signed_cert, create_ssl_context
 from email_server.dkim_manager import DKIMManager
 from aiosmtpd.controller import Controller
@@ -18,7 +18,7 @@ from aiosmtpd.smtp import SMTP as AIOSMTP
 settings = load_settings()
 SMTP_PORT = int(settings['Server']['SMTP_PORT'])
 SMTP_TLS_PORT = int(settings['Server']['SMTP_TLS_PORT'])
-HOSTNAME = settings['Server']['HOSTNAME']
+HOSTNAME = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost'))
 LOG_LEVEL = settings['Logging']['LOG_LEVEL']
 BIND_IP = settings['Server']['BIND_IP']
 
@@ -94,37 +94,25 @@ async def start_server():
         logger.error("Failed to create SSL context")
         return
     
+    logger.debug(f"SSL context created: {ssl_context}")
+    logger.debug(f"SSL context type: {type(ssl_context)}")
+    
     # Start plain SMTP server (with IP whitelist fallback)
     handler_plain = EnhancedCustomSMTPHandler()
     controller_plain = PlainController(
         handler_plain,
-        hostname=BIND_IP,
+        hostname=HOSTNAME,  # Use proper hostname for HELO identification
-        server_hostname="TestEnvironment",
         port=SMTP_PORT
     )
     controller_plain.start()
     logger.debug(f'Starting plain SMTP server on {HOSTNAME}:{SMTP_PORT}...')
     
-    # Start TLS SMTP server using closure pattern like the original
+    # Start TLS SMTP server using the updated TLSController
     handler_tls = EnhancedCustomSMTPHandler()
-    
-    # Define TLS controller class with ssl_context in closure (like original)
-    class TLSController(Controller):
-        def factory(self):
-            return AIOSMTP(
-                self.handler,
-                tls_context=ssl_context,  # Use ssl_context from closure
-                require_starttls=False,  # Don't force STARTTLS, but make it available
-                auth_require_tls=True,   # If auth is used, require TLS
-                authenticator=self.handler.combined_authenticator,
-                decode_data=True,
-                hostname=self.hostname
-            )
-    
     controller_tls = TLSController(
         handler_tls,
-        hostname=BIND_IP,
+        ssl_context=ssl_context,
-        server_hostname="TestEnvironment",
+        hostname=HOSTNAME,  # Use proper hostname for HELO identification  
         port=SMTP_TLS_PORT
     )
     controller_tls.start()

email_server/settings_loader.py

@@ -13,7 +13,8 @@ DEFAULTS = {
     'Server': {
         'SMTP_PORT': '4025',
         'SMTP_TLS_PORT': '40587',
-        'HOSTNAME': 'localhost',
+        'HOSTNAME': 'mail.example.com',
+        'helo_hostname': 'mail.example.com',
         'BIND_IP': '0.0.0.0',
     },
     'Database': {

email_server/smtp_handler.py

@@ -63,6 +63,9 @@ class EnhancedCustomSMTPHandler:
     def _ensure_required_headers(self, content: str, envelope, message_id: str, custom_headers: list = None) -> str:
         """Ensure all required email headers are present and properly formatted.
         
+        Following RFC 5322 header order and best practices for spam score reduction.
+        Optimized based on Gmail's header structure for better deliverability.
+        
         Args:
             content (str): Email content.
             envelope: SMTP envelope.
@@ -73,8 +76,12 @@ class EnhancedCustomSMTPHandler:
             str: Email content with all required headers properly formatted.
         """
         import email.utils
+        from email_server.settings_loader import load_settings
         
         try:
+            settings = load_settings()
+            server_hostname = settings.get('Server', 'helo_hostname', fallback='mail.netbro.uk')
+            
             logger.debug(f"Processing headers for message {message_id}")
             
             # Parse the message properly
@@ -87,6 +94,7 @@ class EnhancedCustomSMTPHandler:
             # Find header/body boundary and collect existing headers
             body_start = 0
             existing_headers = {}
+            original_header_order = []
             
             for i, line in enumerate(lines):
                 if line.strip() == '':
@@ -104,81 +112,106 @@ class EnhancedCustomSMTPHandler:
                         j += 1
                     
                     existing_headers[header_name_lower] = header_value
+                    original_header_order.append((header_name.strip(), header_value))
                     logger.debug(f"Found existing header: {header_name_lower} = {header_value}")
             
-            # Extract body
+            # Extract body and clean it
             body_lines = lines[body_start:] if body_start < len(lines) else []
             while body_lines and body_lines[-1].strip() == '':
                 body_lines.pop()
             body = '\n'.join(body_lines)
             
-            # Build required headers list
+            # Build headers in optimized order based on Gmail's structure
             required_headers = []
             
-            # Add custom headers first
+            # 1. Message-ID (critical for spam filters)
-            if custom_headers:
-                for header_name, header_value in custom_headers:
-                    required_headers.append(f"{header_name}: {header_value}")
-                    logger.debug(f"Added custom header: {header_name}: {header_value}")
-            
-            # Message-ID (always add to ensure it's present)
             if 'message-id' in existing_headers:
                 required_headers.append(f"Message-ID: {existing_headers['message-id']}")
             else:
-                domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else 'localhost'
+                domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else server_hostname.replace('mail.', '')
                 required_headers.append(f"Message-ID: <{message_id}@{domain}>")
             
-            # Date
+            # 2. Date (critical for spam filters)
             if 'date' in existing_headers:
                 required_headers.append(f"Date: {existing_headers['date']}")
             else:
                 date_str = email.utils.formatdate(localtime=True)
                 required_headers.append(f"Date: {date_str}")
             
-            # From (required)
+            # 3. MIME-Version (declare MIME compliance early)
-            if 'from' in existing_headers:
+            if 'mime-version' in existing_headers:
-                required_headers.append(f"From: {existing_headers['from']}")
+                required_headers.append(f"MIME-Version: {existing_headers['mime-version']}")
             else:
-                required_headers.append(f"From: {envelope.mail_from}")
+                required_headers.append("MIME-Version: 1.0")
             
-            # To (required)
+            # 4. User-Agent (if present, helps with reputation)
+            if 'user-agent' in existing_headers:
+                required_headers.append(f"User-Agent: {existing_headers['user-agent']}")
+            
+            # 5. Content-Language (if present)
+            if 'content-language' in existing_headers:
+                required_headers.append(f"Content-Language: {existing_headers['content-language']}")
+            
+            # 6. To (primary recipients - critical)
             if 'to' in existing_headers:
                 required_headers.append(f"To: {existing_headers['to']}")
             else:
                 to_list = ', '.join(envelope.rcpt_tos)
                 required_headers.append(f"To: {to_list}")
             
-            # Subject (preserve existing or add empty)
+            # 7. From (sender identification - critical)
+            if 'from' in existing_headers:
+                required_headers.append(f"From: {existing_headers['from']}")
+            else:
+                required_headers.append(f"From: {envelope.mail_from}")
+            
+            # 8. Subject (message topic - critical)
             if 'subject' in existing_headers:
                 required_headers.append(f"Subject: {existing_headers['subject']}")
             else:
                 required_headers.append("Subject: ")
             
-            # MIME headers
+            # 9. Content-Type (media type information)
-            if 'mime-version' in existing_headers:
-                required_headers.append(f"MIME-Version: {existing_headers['mime-version']}")
-            else:
-                required_headers.append("MIME-Version: 1.0")
-            
             if 'content-type' in existing_headers:
                 required_headers.append(f"Content-Type: {existing_headers['content-type']}")
             else:
-                required_headers.append("Content-Type: text/plain; charset=utf-8")
+                required_headers.append("Content-Type: text/plain; charset=UTF-8; format=flowed")
             
+            # 10. Content-Transfer-Encoding
             if 'content-transfer-encoding' in existing_headers:
                 required_headers.append(f"Content-Transfer-Encoding: {existing_headers['content-transfer-encoding']}")
             else:
                 required_headers.append("Content-Transfer-Encoding: 7bit")
             
-            # Add any other existing headers
+            # Add custom headers after essential headers but before misc headers
+            if custom_headers:
+                for header_name, header_value in custom_headers:
+                    # Skip if already added in essential headers
+                    if header_name.lower() not in ['message-id', 'date', 'mime-version', 'user-agent',
+                                                  'content-language', 'to', 'from', 'subject', 
+                                                  'content-type', 'content-transfer-encoding']:
+                        required_headers.append(f"{header_name}: {header_value}")
+                        logger.debug(f"Added custom header: {header_name}: {header_value}")
+            
+            # Add any other existing headers that weren't handled above
             essential_headers = {
                 'message-id', 'date', 'from', 'to', 'subject', 
-                'mime-version', 'content-type', 'content-transfer-encoding'
+                'mime-version', 'content-type', 'content-transfer-encoding',
+                'user-agent', 'content-language'
             }
             
-            for header_name, header_value in existing_headers.items():
+            # Preserve original header names and values for non-essential headers
-                if header_name not in essential_headers:
+            for header_name, header_value in original_header_order:
-                    required_headers.append(f"{header_name.title()}: {header_value}")
+                if header_name.lower() not in essential_headers:
+                    # Skip custom headers we already added
+                    skip = False
+                    if custom_headers:
+                        for custom_name, _ in custom_headers:
+                            if header_name.lower() == custom_name.lower():
+                                skip = True
+                                break
+                    if not skip:
+                        required_headers.append(f"{header_name}: {header_value}")
             
             # Build final message
             final_content = '\r\n'.join(required_headers)
@@ -201,7 +234,7 @@ class EnhancedCustomSMTPHandler:
             return content
 
     async def handle_DATA(self, server, session, envelope):
-        """Handle incoming email data."""
+        """Handle incoming email data with improved header management."""
         try:
             message_id = str(uuid.uuid4())
             logger.debug(f'Received email {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}')
@@ -220,6 +253,18 @@ class EnhancedCustomSMTPHandler:
             if sender_domain:
                 custom_headers = self.dkim_manager.get_active_custom_headers(sender_domain)
             
+            # Add beneficial headers for spam score improvement
+            client_ip = getattr(session, 'peer', ['unknown'])[0] if hasattr(session, 'peer') else None
+            if client_ip:
+                # Add X-Originating-IP header (helps with reputation)
+                custom_headers.append(('X-Originating-IP', f'[{client_ip}]'))
+            
+            # Add X-Mailer header for identification
+            custom_headers.append(('X-Mailer', 'NetBro Mail Server 1.0'))
+            
+            # Add X-Priority header (normal priority)
+            custom_headers.append(('X-Priority', '3'))
+            
             # Ensure required headers are present (including custom headers)
             content = self._ensure_required_headers(content, envelope, message_id, custom_headers)
             
@@ -294,28 +339,39 @@ class TLSController(Controller):
     """Custom controller with TLS support - modeled after the working original."""
     
     def __init__(self, handler, ssl_context, hostname='localhost', port=40587):
-        self.ssl_context = ssl_context
+        logger.debug(f"TLSController __init__: ssl_context={ssl_context is not None}")
-        super().__init__(handler, hostname=hostname, port=port)
+        self._ssl_context = ssl_context  # Use private attribute to avoid conflicts
+        self.smtp_hostname = hostname  # Store for HELO identification
+        super().__init__(handler, hostname='0.0.0.0', port=port)  # Bind to all interfaces
     
     def factory(self):
-        return AIOSMTP(
+        logger.debug(f"TLSController factory: ssl_context={self._ssl_context is not None}")
+        logger.debug(f"TLSController factory: ssl_context object={self._ssl_context}")
+        logger.debug(f"TLSController factory: hostname={self.smtp_hostname}")
+        smtp_instance = AIOSMTP(
             self.handler,
-            tls_context=self.ssl_context,
+            tls_context=self._ssl_context,
-            require_starttls=True,  # Don't force STARTTLS, but make it available
+            require_starttls=False,  # Don't require STARTTLS immediately, but make it available
             auth_require_tls=True,   # If auth is used, require TLS
             authenticator=self.handler.combined_authenticator,
             decode_data=True,
-            hostname=self.hostname
+            hostname=self.smtp_hostname  # Use proper hostname for HELO
         )
+        logger.debug(f"TLSController AIOSMTP instance created with TLS: {hasattr(smtp_instance, 'tls_context')}")
+        return smtp_instance
 
 class PlainController(Controller):
     """Controller for plain SMTP with username/password and IP-based authentication."""
     
+    def __init__(self, handler, hostname='localhost', port=4025):
+        self.smtp_hostname = hostname  # Store for HELO identification
+        super().__init__(handler, hostname='0.0.0.0', port=port)  # Bind to all interfaces
+    
     def factory(self):
         return AIOSMTP(
             self.handler,
             authenticator=self.handler.combined_authenticator,
             auth_require_tls=False,  # Allow AUTH over plain text (not recommended for production)
             decode_data=True,
-            hostname=self.hostname
+            hostname=self.smtp_hostname  # Use proper hostname for HELO
         )