From d0fb2eafd231c1a588b1f077c754889ab64fb82a Mon Sep 17 00:00:00 2001 From: nahakubuilde Date: Sat, 7 Jun 2025 07:36:46 +0100 Subject: [PATCH] improved helo_hostname for better spam score --- email_server/email_relay.py | 20 +++-- email_server/server_runner.py | 30 +++----- email_server/settings_loader.py | 3 +- email_server/smtp_config.py | 0 email_server/smtp_handler.py | 130 +++++++++++++++++++++++--------- 5 files changed, 119 insertions(+), 64 deletions(-) create mode 100644 email_server/smtp_config.py diff --git a/email_server/email_relay.py b/email_server/email_relay.py index 95bcfaa..3831380 100644 --- a/email_server/email_relay.py +++ b/email_server/email_relay.py @@ -7,6 +7,7 @@ import smtplib import ssl from datetime import datetime from email_server.models import Session, EmailLog +from email_server.settings_loader import load_settings from email_server.tool_box import get_logger logger = get_logger() @@ -16,6 +17,11 @@ class EmailRelay: def __init__(self): self.timeout = 30 # Increased timeout for TLS negotiations + # Get the configured hostname for HELO/EHLO identification + settings = load_settings() + self.hostname = settings['Server'].get('helo_hostname', + settings['Server'].get('hostname', 'localhost')) + logger.debug(f"EmailRelay initialized with hostname: {self.hostname}") def relay_email(self, mail_from, rcpt_tos, content): """Relay email to recipient's mail server with opportunistic TLS.""" @@ -53,8 +59,9 @@ class EmailRelay: # Try to enable TLS if the server supports it try: - # Check if server supports STARTTLS - relay_server.ehlo() + # Check if server supports STARTTLS - use proper hostname for EHLO + logger.debug(f'Sending EHLO {self.hostname} to {mx_host}') + relay_server.ehlo(self.hostname) if relay_server.has_extn('starttls'): logger.debug(f'Starting TLS connection to {mx_host}') context = ssl.create_default_context() @@ -62,7 +69,8 @@ class EmailRelay: context.check_hostname = False context.verify_mode = ssl.CERT_NONE relay_server.starttls(context=context) - relay_server.ehlo() # Say hello again after STARTTLS + logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to {mx_host}') + relay_server.ehlo(self.hostname) # Say hello again after STARTTLS with proper hostname logger.debug(f'TLS connection established to {mx_host}') else: logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text') @@ -94,13 +102,15 @@ class EmailRelay: # Try TLS with backup server too try: - backup_server.ehlo() + logger.debug(f'Sending EHLO {self.hostname} to backup {backup_mx}') + backup_server.ehlo(self.hostname) if backup_server.has_extn('starttls'): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE backup_server.starttls(context=context) - backup_server.ehlo() + logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to backup {backup_mx}') + backup_server.ehlo(self.hostname) logger.debug(f'TLS connection established to backup {backup_mx}') except Exception: logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text') diff --git a/email_server/server_runner.py b/email_server/server_runner.py index 5e7ad7e..1b6f51b 100644 --- a/email_server/server_runner.py +++ b/email_server/server_runner.py @@ -9,7 +9,7 @@ from email_server.tool_box import get_logger # Import our modules from email_server.models import create_tables -from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController +from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController, TLSController from email_server.tls_utils import generate_self_signed_cert, create_ssl_context from email_server.dkim_manager import DKIMManager from aiosmtpd.controller import Controller @@ -18,7 +18,7 @@ from aiosmtpd.smtp import SMTP as AIOSMTP settings = load_settings() SMTP_PORT = int(settings['Server']['SMTP_PORT']) SMTP_TLS_PORT = int(settings['Server']['SMTP_TLS_PORT']) -HOSTNAME = settings['Server']['HOSTNAME'] +HOSTNAME = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost')) LOG_LEVEL = settings['Logging']['LOG_LEVEL'] BIND_IP = settings['Server']['BIND_IP'] @@ -94,37 +94,25 @@ async def start_server(): logger.error("Failed to create SSL context") return + logger.debug(f"SSL context created: {ssl_context}") + logger.debug(f"SSL context type: {type(ssl_context)}") + # Start plain SMTP server (with IP whitelist fallback) handler_plain = EnhancedCustomSMTPHandler() controller_plain = PlainController( handler_plain, - hostname=BIND_IP, - server_hostname="TestEnvironment", + hostname=HOSTNAME, # Use proper hostname for HELO identification port=SMTP_PORT ) controller_plain.start() logger.debug(f'Starting plain SMTP server on {HOSTNAME}:{SMTP_PORT}...') - # Start TLS SMTP server using closure pattern like the original + # Start TLS SMTP server using the updated TLSController handler_tls = EnhancedCustomSMTPHandler() - - # Define TLS controller class with ssl_context in closure (like original) - class TLSController(Controller): - def factory(self): - return AIOSMTP( - self.handler, - tls_context=ssl_context, # Use ssl_context from closure - require_starttls=False, # Don't force STARTTLS, but make it available - auth_require_tls=True, # If auth is used, require TLS - authenticator=self.handler.combined_authenticator, - decode_data=True, - hostname=self.hostname - ) - controller_tls = TLSController( handler_tls, - hostname=BIND_IP, - server_hostname="TestEnvironment", + ssl_context=ssl_context, + hostname=HOSTNAME, # Use proper hostname for HELO identification port=SMTP_TLS_PORT ) controller_tls.start() diff --git a/email_server/settings_loader.py b/email_server/settings_loader.py index 3b5811f..95a4317 100644 --- a/email_server/settings_loader.py +++ b/email_server/settings_loader.py @@ -13,7 +13,8 @@ DEFAULTS = { 'Server': { 'SMTP_PORT': '4025', 'SMTP_TLS_PORT': '40587', - 'HOSTNAME': 'localhost', + 'HOSTNAME': 'mail.example.com', + 'helo_hostname': 'mail.example.com', 'BIND_IP': '0.0.0.0', }, 'Database': { diff --git a/email_server/smtp_config.py b/email_server/smtp_config.py new file mode 100644 index 0000000..e69de29 diff --git a/email_server/smtp_handler.py b/email_server/smtp_handler.py index 7c8dc15..b38b118 100644 --- a/email_server/smtp_handler.py +++ b/email_server/smtp_handler.py @@ -63,6 +63,9 @@ class EnhancedCustomSMTPHandler: def _ensure_required_headers(self, content: str, envelope, message_id: str, custom_headers: list = None) -> str: """Ensure all required email headers are present and properly formatted. + Following RFC 5322 header order and best practices for spam score reduction. + Optimized based on Gmail's header structure for better deliverability. + Args: content (str): Email content. envelope: SMTP envelope. @@ -73,8 +76,12 @@ class EnhancedCustomSMTPHandler: str: Email content with all required headers properly formatted. """ import email.utils + from email_server.settings_loader import load_settings try: + settings = load_settings() + server_hostname = settings.get('Server', 'helo_hostname', fallback='mail.netbro.uk') + logger.debug(f"Processing headers for message {message_id}") # Parse the message properly @@ -87,6 +94,7 @@ class EnhancedCustomSMTPHandler: # Find header/body boundary and collect existing headers body_start = 0 existing_headers = {} + original_header_order = [] for i, line in enumerate(lines): if line.strip() == '': @@ -104,81 +112,106 @@ class EnhancedCustomSMTPHandler: j += 1 existing_headers[header_name_lower] = header_value + original_header_order.append((header_name.strip(), header_value)) logger.debug(f"Found existing header: {header_name_lower} = {header_value}") - # Extract body + # Extract body and clean it body_lines = lines[body_start:] if body_start < len(lines) else [] while body_lines and body_lines[-1].strip() == '': body_lines.pop() body = '\n'.join(body_lines) - # Build required headers list + # Build headers in optimized order based on Gmail's structure required_headers = [] - # Add custom headers first - if custom_headers: - for header_name, header_value in custom_headers: - required_headers.append(f"{header_name}: {header_value}") - logger.debug(f"Added custom header: {header_name}: {header_value}") - - # Message-ID (always add to ensure it's present) + # 1. Message-ID (critical for spam filters) if 'message-id' in existing_headers: required_headers.append(f"Message-ID: {existing_headers['message-id']}") else: - domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else 'localhost' + domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else server_hostname.replace('mail.', '') required_headers.append(f"Message-ID: <{message_id}@{domain}>") - # Date + # 2. Date (critical for spam filters) if 'date' in existing_headers: required_headers.append(f"Date: {existing_headers['date']}") else: date_str = email.utils.formatdate(localtime=True) required_headers.append(f"Date: {date_str}") - # From (required) - if 'from' in existing_headers: - required_headers.append(f"From: {existing_headers['from']}") + # 3. MIME-Version (declare MIME compliance early) + if 'mime-version' in existing_headers: + required_headers.append(f"MIME-Version: {existing_headers['mime-version']}") else: - required_headers.append(f"From: {envelope.mail_from}") + required_headers.append("MIME-Version: 1.0") - # To (required) + # 4. User-Agent (if present, helps with reputation) + if 'user-agent' in existing_headers: + required_headers.append(f"User-Agent: {existing_headers['user-agent']}") + + # 5. Content-Language (if present) + if 'content-language' in existing_headers: + required_headers.append(f"Content-Language: {existing_headers['content-language']}") + + # 6. To (primary recipients - critical) if 'to' in existing_headers: required_headers.append(f"To: {existing_headers['to']}") else: to_list = ', '.join(envelope.rcpt_tos) required_headers.append(f"To: {to_list}") - # Subject (preserve existing or add empty) + # 7. From (sender identification - critical) + if 'from' in existing_headers: + required_headers.append(f"From: {existing_headers['from']}") + else: + required_headers.append(f"From: {envelope.mail_from}") + + # 8. Subject (message topic - critical) if 'subject' in existing_headers: required_headers.append(f"Subject: {existing_headers['subject']}") else: required_headers.append("Subject: ") - # MIME headers - if 'mime-version' in existing_headers: - required_headers.append(f"MIME-Version: {existing_headers['mime-version']}") - else: - required_headers.append("MIME-Version: 1.0") - + # 9. Content-Type (media type information) if 'content-type' in existing_headers: required_headers.append(f"Content-Type: {existing_headers['content-type']}") else: - required_headers.append("Content-Type: text/plain; charset=utf-8") + required_headers.append("Content-Type: text/plain; charset=UTF-8; format=flowed") + # 10. Content-Transfer-Encoding if 'content-transfer-encoding' in existing_headers: required_headers.append(f"Content-Transfer-Encoding: {existing_headers['content-transfer-encoding']}") else: required_headers.append("Content-Transfer-Encoding: 7bit") - # Add any other existing headers + # Add custom headers after essential headers but before misc headers + if custom_headers: + for header_name, header_value in custom_headers: + # Skip if already added in essential headers + if header_name.lower() not in ['message-id', 'date', 'mime-version', 'user-agent', + 'content-language', 'to', 'from', 'subject', + 'content-type', 'content-transfer-encoding']: + required_headers.append(f"{header_name}: {header_value}") + logger.debug(f"Added custom header: {header_name}: {header_value}") + + # Add any other existing headers that weren't handled above essential_headers = { 'message-id', 'date', 'from', 'to', 'subject', - 'mime-version', 'content-type', 'content-transfer-encoding' + 'mime-version', 'content-type', 'content-transfer-encoding', + 'user-agent', 'content-language' } - for header_name, header_value in existing_headers.items(): - if header_name not in essential_headers: - required_headers.append(f"{header_name.title()}: {header_value}") + # Preserve original header names and values for non-essential headers + for header_name, header_value in original_header_order: + if header_name.lower() not in essential_headers: + # Skip custom headers we already added + skip = False + if custom_headers: + for custom_name, _ in custom_headers: + if header_name.lower() == custom_name.lower(): + skip = True + break + if not skip: + required_headers.append(f"{header_name}: {header_value}") # Build final message final_content = '\r\n'.join(required_headers) @@ -201,7 +234,7 @@ class EnhancedCustomSMTPHandler: return content async def handle_DATA(self, server, session, envelope): - """Handle incoming email data.""" + """Handle incoming email data with improved header management.""" try: message_id = str(uuid.uuid4()) logger.debug(f'Received email {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}') @@ -220,6 +253,18 @@ class EnhancedCustomSMTPHandler: if sender_domain: custom_headers = self.dkim_manager.get_active_custom_headers(sender_domain) + # Add beneficial headers for spam score improvement + client_ip = getattr(session, 'peer', ['unknown'])[0] if hasattr(session, 'peer') else None + if client_ip: + # Add X-Originating-IP header (helps with reputation) + custom_headers.append(('X-Originating-IP', f'[{client_ip}]')) + + # Add X-Mailer header for identification + custom_headers.append(('X-Mailer', 'NetBro Mail Server 1.0')) + + # Add X-Priority header (normal priority) + custom_headers.append(('X-Priority', '3')) + # Ensure required headers are present (including custom headers) content = self._ensure_required_headers(content, envelope, message_id, custom_headers) @@ -294,28 +339,39 @@ class TLSController(Controller): """Custom controller with TLS support - modeled after the working original.""" def __init__(self, handler, ssl_context, hostname='localhost', port=40587): - self.ssl_context = ssl_context - super().__init__(handler, hostname=hostname, port=port) + logger.debug(f"TLSController __init__: ssl_context={ssl_context is not None}") + self._ssl_context = ssl_context # Use private attribute to avoid conflicts + self.smtp_hostname = hostname # Store for HELO identification + super().__init__(handler, hostname='0.0.0.0', port=port) # Bind to all interfaces def factory(self): - return AIOSMTP( + logger.debug(f"TLSController factory: ssl_context={self._ssl_context is not None}") + logger.debug(f"TLSController factory: ssl_context object={self._ssl_context}") + logger.debug(f"TLSController factory: hostname={self.smtp_hostname}") + smtp_instance = AIOSMTP( self.handler, - tls_context=self.ssl_context, - require_starttls=True, # Don't force STARTTLS, but make it available + tls_context=self._ssl_context, + require_starttls=False, # Don't require STARTTLS immediately, but make it available auth_require_tls=True, # If auth is used, require TLS authenticator=self.handler.combined_authenticator, decode_data=True, - hostname=self.hostname + hostname=self.smtp_hostname # Use proper hostname for HELO ) + logger.debug(f"TLSController AIOSMTP instance created with TLS: {hasattr(smtp_instance, 'tls_context')}") + return smtp_instance class PlainController(Controller): """Controller for plain SMTP with username/password and IP-based authentication.""" + def __init__(self, handler, hostname='localhost', port=4025): + self.smtp_hostname = hostname # Store for HELO identification + super().__init__(handler, hostname='0.0.0.0', port=port) # Bind to all interfaces + def factory(self): return AIOSMTP( self.handler, authenticator=self.handler.combined_authenticator, auth_require_tls=False, # Allow AUTH over plain text (not recommended for production) decode_data=True, - hostname=self.hostname + hostname=self.smtp_hostname # Use proper hostname for HELO )