Merge pull request #2 from ghostersk/testing

Fixed email headers order and DKIM signing

email_server/cli_tools.py

@@ -233,7 +233,7 @@ def main():
         print("Database tables created successfully")
     
     elif args.command == 'add-domain':
-        add_domain(args.domain, not args.no_auth)
+        add_domain(args.domain)
     
     elif args.command == 'add-user':
         add_user(args.email, args.password, args.domain)

email_server/email_relay.py

@@ -7,6 +7,7 @@ import smtplib
 import ssl
 from datetime import datetime
 from email_server.models import Session, EmailLog
+from email_server.settings_loader import load_settings
 from email_server.tool_box import get_logger
 
 logger = get_logger()
@@ -16,6 +17,11 @@ class EmailRelay:
     
     def __init__(self):
         self.timeout = 30  # Increased timeout for TLS negotiations
+        # Get the configured hostname for HELO/EHLO identification
+        settings = load_settings()
+        self.hostname = settings['Server'].get('helo_hostname', 
+                                             settings['Server'].get('hostname', 'localhost'))
+        logger.debug(f"EmailRelay initialized with hostname: {self.hostname}")
     
     def relay_email(self, mail_from, rcpt_tos, content):
         """Relay email to recipient's mail server with opportunistic TLS."""
@@ -53,8 +59,9 @@ class EmailRelay:
                     
                     # Try to enable TLS if the server supports it
                     try:
-                        # Check if server supports STARTTLS
+                        # Check if server supports STARTTLS - use proper hostname for EHLO
-                        relay_server.ehlo()
+                        logger.debug(f'Sending EHLO {self.hostname} to {mx_host}')
+                        relay_server.ehlo(self.hostname)
                         if relay_server.has_extn('starttls'):
                             logger.debug(f'Starting TLS connection to {mx_host}')
                             context = ssl.create_default_context()
@@ -62,7 +69,8 @@ class EmailRelay:
                             context.check_hostname = False
                             context.verify_mode = ssl.CERT_NONE
                             relay_server.starttls(context=context)
-                            relay_server.ehlo()  # Say hello again after STARTTLS
+                            logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to {mx_host}')
+                            relay_server.ehlo(self.hostname)  # Say hello again after STARTTLS with proper hostname
                             logger.debug(f'TLS connection established to {mx_host}')
                         else:
                             logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text')
@@ -94,13 +102,15 @@ class EmailRelay:
                                 
                                 # Try TLS with backup server too
                                 try:
-                                    backup_server.ehlo()
+                                    logger.debug(f'Sending EHLO {self.hostname} to backup {backup_mx}')
+                                    backup_server.ehlo(self.hostname)
                                     if backup_server.has_extn('starttls'):
                                         context = ssl.create_default_context()
                                         context.check_hostname = False
                                         context.verify_mode = ssl.CERT_NONE
                                         backup_server.starttls(context=context)
-                                        backup_server.ehlo()
+                                        logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to backup {backup_mx}')
+                                        backup_server.ehlo(self.hostname)
                                         logger.debug(f'TLS connection established to backup {backup_mx}')
                                 except Exception:
                                     logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text')

email_server/server_runner.py

@@ -9,7 +9,7 @@ from email_server.tool_box import get_logger
 
 # Import our modules
 from email_server.models import create_tables
-from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController
+from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController, TLSController
 from email_server.tls_utils import generate_self_signed_cert, create_ssl_context
 from email_server.dkim_manager import DKIMManager
 from aiosmtpd.controller import Controller
@@ -18,7 +18,7 @@ from aiosmtpd.smtp import SMTP as AIOSMTP
 settings = load_settings()
 SMTP_PORT = int(settings['Server']['SMTP_PORT'])
 SMTP_TLS_PORT = int(settings['Server']['SMTP_TLS_PORT'])
-HOSTNAME = settings['Server']['HOSTNAME']
+HOSTNAME = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost'))
 LOG_LEVEL = settings['Logging']['LOG_LEVEL']
 BIND_IP = settings['Server']['BIND_IP']
 
@@ -94,37 +94,25 @@ async def start_server():
         logger.error("Failed to create SSL context")
         return
     
+    logger.debug(f"SSL context created: {ssl_context}")
+    logger.debug(f"SSL context type: {type(ssl_context)}")
+    
     # Start plain SMTP server (with IP whitelist fallback)
     handler_plain = EnhancedCustomSMTPHandler()
     controller_plain = PlainController(
         handler_plain,
-        hostname=BIND_IP,
+        hostname=HOSTNAME,  # Use proper hostname for HELO identification
-        server_hostname="TestEnvironment",
         port=SMTP_PORT
     )
     controller_plain.start()
     logger.debug(f'Starting plain SMTP server on {HOSTNAME}:{SMTP_PORT}...')
     
-    # Start TLS SMTP server using closure pattern like the original
+    # Start TLS SMTP server using the updated TLSController
     handler_tls = EnhancedCustomSMTPHandler()
-    
-    # Define TLS controller class with ssl_context in closure (like original)
-    class TLSController(Controller):
-        def factory(self):
-            return AIOSMTP(
-                self.handler,
-                tls_context=ssl_context,  # Use ssl_context from closure
-                require_starttls=False,  # Don't force STARTTLS, but make it available
-                auth_require_tls=True,   # If auth is used, require TLS
-                authenticator=self.handler.combined_authenticator,
-                decode_data=True,
-                hostname=self.hostname
-            )
-    
     controller_tls = TLSController(
         handler_tls,
-        hostname=BIND_IP,
+        ssl_context=ssl_context,
-        server_hostname="TestEnvironment",
+        hostname=HOSTNAME,  # Use proper hostname for HELO identification  
         port=SMTP_TLS_PORT
     )
     controller_tls.start()

email_server/settings_loader.py

@@ -8,40 +8,64 @@ from pathlib import Path
 
 SETTINGS_PATH = Path(__file__).parent.parent / 'settings.ini'
 
-# Default values for settings.ini
+# Default values and comments for settings.ini
 DEFAULTS = {
     'Server': {
+        '; Server configuration for SMTP ports and hostname': None,
+        '; Plain SMTP port for internal/whitelisted IPs': None,
         'SMTP_PORT': '4025',
+        '; STARTTLS SMTP port for authenticated users': None,
         'SMTP_TLS_PORT': '40587',
-        'HOSTNAME': 'localhost',
+        '; Server hostname for HELO/EHLO identification': None,
+        'HOSTNAME': 'mail.example.com',
+        '; Override HELO hostname': None,
+        'helo_hostname': 'mail.example.com',
+        '; IP address to bind to (0.0.0.0 = all interfaces), on Windows must use specific IP': None,
         'BIND_IP': '0.0.0.0',
+        '; Custom server banner (to make it empty use "" must be double quotes)': None,
+        'server_banner': "",
     },
     'Database': {
+        '; Database configuration': None,
         'DATABASE_URL': 'sqlite:///email_server/server_data/smtp_server.db',
     },
     'Logging': {
+        '; Logging configuration': None,
+        '; Log level: DEBUG, INFO, WARNING, ERROR, CRITICAL': None,
         'LOG_LEVEL': 'INFO',
+        '; Hide verbose aiosmtpd INFO messages when LOG_LEVEL = INFO': None,
         'hide_info_aiosmtpd': 'true',
     },
     'Relay': {
+        '; Timeout in seconds for external SMTP connections': None,
         'RELAY_TIMEOUT': '10',
     },
     'TLS': {
+        '; TLS/SSL certificate configuration': None,
         'TLS_CERT_FILE': 'email_server/ssl_certs/server.crt',
         'TLS_KEY_FILE': 'email_server/ssl_certs/server.key',
     },
     'DKIM': {
+        '; DKIM signing configuration': None,
+        '; RSA key size for DKIM keys (1024, 2048, 4096)': None,
         'DKIM_KEY_SIZE': '2048',
     },
 }
 
 def generate_settings_ini(settings_path: Path = SETTINGS_PATH) -> None:
-    """Generate settings.ini with default values if it does not exist."""
+    """Generate settings.ini with default values and comments if it does not exist."""
     if settings_path.exists():
         return
-    config_parser = configparser.ConfigParser()
+    config_parser = configparser.ConfigParser(allow_no_value=True)
     for section, values in DEFAULTS.items():
-        config_parser[section] = values
+        config_parser.add_section(section)
+        for key, value in values.items():
+            if key.startswith(';'):
+                # This is a comment line
+                config_parser.set(section, key)
+            else:
+                # This is a setting with value
+                config_parser.set(section, key, value)
     with open(settings_path, 'w') as f:
         config_parser.write(f)
 

email_server/smtp_handler.py

@@ -9,17 +9,32 @@ Security Features:
 """
 
 import uuid
-import email.utils
 from datetime import datetime
 from aiosmtpd.smtp import SMTP as AIOSMTP, AuthResult
 from aiosmtpd.controller import Controller
 from email_server.auth import EnhancedAuthenticator, EnhancedIPAuthenticator, validate_sender_authorization, get_authenticated_domain_id
 from email_server.email_relay import EmailRelay
 from email_server.dkim_manager import DKIMManager
+from email_server.settings_loader import load_settings
 from email_server.tool_box import get_logger
 
 logger = get_logger()
 
+class CustomSMTP(AIOSMTP):
+    """Custom SMTP class with configurable banner."""
+    
+    def __init__(self, *args, **kwargs):
+        # Sets Custom SMTP banner from settings
+        settings = load_settings()
+        _banner_message = settings['Server'].get('server_banner', '')
+        if _banner_message == '""':
+            _banner_message = ''
+        self.custom_banner = _banner_message
+            
+        super().__init__(*args, **kwargs)
+        # Override the __ident__ to use our custom banner
+        self.__ident__ = self.custom_banner
+
 class EnhancedCombinedAuthenticator:
     """
     Enhanced combined authenticator with sender validation support.
@@ -61,51 +76,182 @@ class EnhancedCustomSMTPHandler:
         self.auth_require_tls = False
         self.auth_methods = ['LOGIN', 'PLAIN']
 
-    def _ensure_required_headers(self, content: str, envelope, message_id: str) -> str:
+    def _ensure_required_headers(self, content: str, envelope, message_id: str, custom_headers: list = None) -> str:
         """Ensure all required email headers are present and properly formatted.
         
+        Following RFC 5322 header order and best practices for spam score reduction.
+        Optimized based on Gmail's header structure for better deliverability.
+        
         Args:
             content (str): Email content.
             envelope: SMTP envelope.
             message_id (str): Generated message ID.
+            custom_headers (list): List of (name, value) tuples for custom headers.
             
         Returns:
-            str: Email content with all required headers.
+            str: Email content with all required headers properly formatted.
         """
-        import email
+        import email.utils
-        from email.parser import Parser
+        from email_server.settings_loader import load_settings
-        from email.policy import default
         
-        # Parse the message using the email library
+        try:
-        msg = Parser(policy=default).parsestr(content)
+            settings = load_settings()
+            fallback_hostname = settings.get('Server', 'HOSTNAME', fallback='localhost')
+            server_hostname = settings.get('Server', 'helo_hostname', fallback=fallback_hostname)
             
-        # Set or add required headers if missing
+            logger.debug(f"Processing headers for message {message_id}")
-        if not msg.get('Message-ID'):
-            msg['Message-ID'] = f"<{message_id}@{envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else 'localhost'}>"
-        if not msg.get('Date'):
-            msg['Date'] = email.utils.formatdate(localtime=True)
-        if not msg.get('From'):
-            msg['From'] = envelope.mail_from
-        if not msg.get('To'):
-            msg['To'] = ', '.join(envelope.rcpt_tos)
-        if not msg.get('MIME-Version'):
-            msg['MIME-Version'] = '1.0'
-        if not msg.get('Content-Type'):
-            msg['Content-Type'] = 'text/plain; charset=utf-8'
-        if not msg.get('Subject'):
-            msg['Subject'] = '(No Subject)'
-        if not msg.get('Content-Transfer-Encoding'):
-            msg['Content-Transfer-Encoding'] = '7bit'
             
-        # Ensure exactly one blank line between headers and body
+            # Parse the message properly
-        # The email library will handle this when flattening
+            if isinstance(content, bytes):
-        from io import StringIO
+                content = content.decode('utf-8', errors='replace')
-        out = StringIO()
+            
-        out.write(msg.as_string())
+            # Split content into lines and normalize line endings
-        return out.getvalue()
+            lines = content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
+            
+            # Find header/body boundary and collect existing headers
+            body_start = 0
+            existing_headers = {}
+            original_header_order = []
+            
+            for i, line in enumerate(lines):
+                if line.strip() == '':
+                    body_start = i + 1
+                    break
+                if ':' in line and not line.startswith((' ', '\t')):
+                    header_name, header_value = line.split(':', 1)
+                    header_name_lower = header_name.strip().lower()
+                    header_value = header_value.strip()
+                    
+                    # Handle continuation lines
+                    j = i + 1
+                    while j < len(lines) and lines[j].startswith((' ', '\t')):
+                        header_value += ' ' + lines[j].strip()
+                        j += 1
+                    
+                    existing_headers[header_name_lower] = header_value
+                    original_header_order.append((header_name.strip(), header_value))
+                    logger.debug(f"Found existing header: {header_name_lower} = {header_value}")
+            
+            # Extract body and clean it
+            body_lines = lines[body_start:] if body_start < len(lines) else []
+            while body_lines and body_lines[-1].strip() == '':
+                body_lines.pop()
+            body = '\n'.join(body_lines)
+            
+            # Build headers in optimized order based on Gmail's structure
+            required_headers = []
+            
+            # 1. Message-ID (critical for spam filters)
+            if 'message-id' in existing_headers:
+                required_headers.append(f"Message-ID: {existing_headers['message-id']}")
+            else:
+                domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else server_hostname.replace('mail.', '')
+                required_headers.append(f"Message-ID: <{message_id}@{domain}>")
+            
+            # 2. Date (critical for spam filters)
+            if 'date' in existing_headers:
+                required_headers.append(f"Date: {existing_headers['date']}")
+            else:
+                date_str = email.utils.formatdate(localtime=True)
+                required_headers.append(f"Date: {date_str}")
+            
+            # 3. MIME-Version (declare MIME compliance early)
+            if 'mime-version' in existing_headers:
+                required_headers.append(f"MIME-Version: {existing_headers['mime-version']}")
+            else:
+                required_headers.append("MIME-Version: 1.0")
+            
+            # 4. User-Agent (if present, helps with reputation)
+            if 'user-agent' in existing_headers:
+                required_headers.append(f"User-Agent: {existing_headers['user-agent']}")
+            
+            # 5. Content-Language (if present)
+            if 'content-language' in existing_headers:
+                required_headers.append(f"Content-Language: {existing_headers['content-language']}")
+            
+            # 6. To (primary recipients - critical)
+            if 'to' in existing_headers:
+                required_headers.append(f"To: {existing_headers['to']}")
+            else:
+                to_list = ', '.join(envelope.rcpt_tos)
+                required_headers.append(f"To: {to_list}")
+            
+            # 7. From (sender identification - critical)
+            if 'from' in existing_headers:
+                required_headers.append(f"From: {existing_headers['from']}")
+            else:
+                required_headers.append(f"From: {envelope.mail_from}")
+            
+            # 8. Subject (message topic - critical)
+            if 'subject' in existing_headers:
+                required_headers.append(f"Subject: {existing_headers['subject']}")
+            else:
+                required_headers.append("Subject: ")
+            
+            # 9. Content-Type (media type information)
+            if 'content-type' in existing_headers:
+                required_headers.append(f"Content-Type: {existing_headers['content-type']}")
+            else:
+                required_headers.append("Content-Type: text/plain; charset=UTF-8; format=flowed")
+            
+            # 10. Content-Transfer-Encoding
+            if 'content-transfer-encoding' in existing_headers:
+                required_headers.append(f"Content-Transfer-Encoding: {existing_headers['content-transfer-encoding']}")
+            else:
+                required_headers.append("Content-Transfer-Encoding: 7bit")
+            
+            # Add custom headers after essential headers but before misc headers
+            if custom_headers:
+                for header_name, header_value in custom_headers:
+                    # Skip if already added in essential headers
+                    if header_name.lower() not in ['message-id', 'date', 'mime-version', 'user-agent',
+                                                  'content-language', 'to', 'from', 'subject', 
+                                                  'content-type', 'content-transfer-encoding']:
+                        required_headers.append(f"{header_name}: {header_value}")
+                        logger.debug(f"Added custom header: {header_name}: {header_value}")
+            
+            # Add any other existing headers that weren't handled above
+            essential_headers = {
+                'message-id', 'date', 'from', 'to', 'subject', 
+                'mime-version', 'content-type', 'content-transfer-encoding',
+                'user-agent', 'content-language'
+            }
+            
+            # Preserve original header names and values for non-essential headers
+            for header_name, header_value in original_header_order:
+                if header_name.lower() not in essential_headers:
+                    # Skip custom headers we already added
+                    skip = False
+                    if custom_headers:
+                        for custom_name, _ in custom_headers:
+                            if header_name.lower() == custom_name.lower():
+                                skip = True
+                                break
+                    if not skip:
+                        required_headers.append(f"{header_name}: {header_value}")
+            
+            # Build final message
+            final_content = '\r\n'.join(required_headers)
+            if body.strip():
+                final_content += '\r\n\r\n' + body
+            else:
+                final_content += '\r\n\r\n'
+            
+            logger.debug(f"Final headers for message {message_id}:")
+            for header in required_headers:
+                logger.debug(f"  {header}")
+            
+            return final_content
+            
+        except Exception as e:
+            logger.error(f"Error ensuring headers: {e}")
+            import traceback
+            logger.error(f"Traceback: {traceback.format_exc()}")
+            # Fallback to original content if parsing fails
+            return content
 
     async def handle_DATA(self, server, session, envelope):
-        """Handle incoming email data."""
+        """Handle incoming email data with improved header management."""
         try:
             message_id = str(uuid.uuid4())
             logger.debug(f'Received email {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}')
@@ -119,26 +265,36 @@ class EnhancedCustomSMTPHandler:
             # Extract domain from sender for DKIM signing
             sender_domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else None
             
-            # Ensure required headers are present
+            # Get custom headers before processing
-            content = self._ensure_required_headers(content, envelope, message_id)
+            custom_headers = []
-            
-            # Add custom headers before DKIM signing
             if sender_domain:
                 custom_headers = self.dkim_manager.get_active_custom_headers(sender_domain)
-                for header_name, header_value in custom_headers:
-                    # Insert header at the top of the message
-                    content = f"{header_name}: {header_value}\r\n" + content
             
-            # Relay the email (all modifications done)
+            # Add beneficial headers for spam score improvement
+            client_ip = getattr(session, 'peer', ['unknown'])[0] if hasattr(session, 'peer') else None
+            if client_ip:
+                # Add X-Originating-IP header (helps with reputation)
+                custom_headers.append(('X-Originating-IP', f'[{client_ip}]'))
+            
+            # Add X-Mailer header for identification
+            custom_headers.append(('X-Mailer', 'NetBro Mail Server 1.0'))
+            
+            # Add X-Priority header (normal priority)
+            custom_headers.append(('X-Priority', '3'))
+            
+            # Ensure required headers are present (including custom headers)
+            content = self._ensure_required_headers(content, envelope, message_id, custom_headers)
+            
+            # DKIM-sign the final version of the message (only once, after all modifications)
             signed_content = content
             dkim_signed = False
             if sender_domain:
-                # DKIM-sign the final version of the message
                 signed_content = self.dkim_manager.sign_email(content, sender_domain)
                 dkim_signed = signed_content != content
                 if dkim_signed:
                     logger.debug(f'Email {message_id} signed with DKIM for domain {sender_domain}')
             
+            # Relay the email (no further modifications allowed)
             success = self.email_relay.relay_email(
                 envelope.mail_from, 
                 envelope.rcpt_tos, 
@@ -200,28 +356,39 @@ class TLSController(Controller):
     """Custom controller with TLS support - modeled after the working original."""
     
     def __init__(self, handler, ssl_context, hostname='localhost', port=40587):
-        self.ssl_context = ssl_context
+        logger.debug(f"TLSController __init__: ssl_context={ssl_context is not None}")
-        super().__init__(handler, hostname=hostname, port=port)
+        self._ssl_context = ssl_context  # Use private attribute to avoid conflicts
+        self.smtp_hostname = hostname  # Store for HELO identification
+        super().__init__(handler, hostname='0.0.0.0', port=port)  # Bind to all interfaces
     
     def factory(self):
-        return AIOSMTP(
+        logger.debug(f"TLSController factory: ssl_context={self._ssl_context is not None}")
+        logger.debug(f"TLSController factory: ssl_context object={self._ssl_context}")
+        logger.debug(f"TLSController factory: hostname={self.smtp_hostname}")
+        smtp_instance = CustomSMTP(
             self.handler,
-            tls_context=self.ssl_context,
+            tls_context=self._ssl_context,
-            require_starttls=True,  # Don't force STARTTLS, but make it available
+            require_starttls=False,  # Don't require STARTTLS immediately, but make it available
             auth_require_tls=True,   # If auth is used, require TLS
             authenticator=self.handler.combined_authenticator,
             decode_data=True,
-            hostname=self.hostname
+            hostname=self.smtp_hostname  # Use proper hostname for HELO
         )
+        logger.debug(f"TLSController CustomSMTP instance created with TLS: {hasattr(smtp_instance, 'tls_context')}")
+        return smtp_instance
 
 class PlainController(Controller):
     """Controller for plain SMTP with username/password and IP-based authentication."""
     
+    def __init__(self, handler, hostname='localhost', port=4025):
+        self.smtp_hostname = hostname  # Store for HELO identification
+        super().__init__(handler, hostname='0.0.0.0', port=port)  # Bind to all interfaces
+    
     def factory(self):
-        return AIOSMTP(
+        return CustomSMTP(
             self.handler,
             authenticator=self.handler.combined_authenticator,
             auth_require_tls=False,  # Allow AUTH over plain text (not recommended for production)
             decode_data=True,
-            hostname=self.hostname
+            hostname=self.smtp_hostname  # Use proper hostname for HELO
         )

pymta-server.service

@@ -0,0 +1,32 @@
+[Unit]
+Description=PyMTA Email Server
+After=network.target
+StartLimitIntervalSec=0
+# check any errors when using this service:
+# journalctl -u pymta-server.service -b -f
+
+[Service]
+Type=simple
+User=appuser
+Group=appuser
+WorkingDirectory=/opt/PyMTA-server
+Environment=PYTHONUNBUFFERED=1
+ExecStart=/opt/PyMTA-server/.venv/bin/python main.py
+Restart=always
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+# Security settings
+# Capabilities for low ports < 1024 following 2 lines:
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+# if using port < 1024 comment out line bellow:
+# NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ReadWritePaths=/opt/PyMTA-server
+ProtectHome=true
+
+[Install]
+WantedBy=multi-user.target

requirements.txt

@@ -1,4 +1,8 @@
-# SMTP MTA server:
+# SMTP MTA server
+# then you can allow run ports < 1024 with 
+# create env with `python -m venv .venv --copies` (This will copy the Python binary)
+# for f in /opt/PyMTA-server/.venv/bin/python*; do sudo setcap 'cap_net_bind_service=+ep' "$f"; done
+
 aiosmtpd
 sqlalchemy
 pyOpenSSL