diff --git a/email_server/cli_tools.py b/email_server/cli_tools.py index 0f797f9..6e98954 100644 --- a/email_server/cli_tools.py +++ b/email_server/cli_tools.py @@ -233,7 +233,7 @@ def main(): print("Database tables created successfully") elif args.command == 'add-domain': - add_domain(args.domain, not args.no_auth) + add_domain(args.domain) elif args.command == 'add-user': add_user(args.email, args.password, args.domain) diff --git a/email_server/email_relay.py b/email_server/email_relay.py index 95bcfaa..3831380 100644 --- a/email_server/email_relay.py +++ b/email_server/email_relay.py @@ -7,6 +7,7 @@ import smtplib import ssl from datetime import datetime from email_server.models import Session, EmailLog +from email_server.settings_loader import load_settings from email_server.tool_box import get_logger logger = get_logger() @@ -16,6 +17,11 @@ class EmailRelay: def __init__(self): self.timeout = 30 # Increased timeout for TLS negotiations + # Get the configured hostname for HELO/EHLO identification + settings = load_settings() + self.hostname = settings['Server'].get('helo_hostname', + settings['Server'].get('hostname', 'localhost')) + logger.debug(f"EmailRelay initialized with hostname: {self.hostname}") def relay_email(self, mail_from, rcpt_tos, content): """Relay email to recipient's mail server with opportunistic TLS.""" @@ -53,8 +59,9 @@ class EmailRelay: # Try to enable TLS if the server supports it try: - # Check if server supports STARTTLS - relay_server.ehlo() + # Check if server supports STARTTLS - use proper hostname for EHLO + logger.debug(f'Sending EHLO {self.hostname} to {mx_host}') + relay_server.ehlo(self.hostname) if relay_server.has_extn('starttls'): logger.debug(f'Starting TLS connection to {mx_host}') context = ssl.create_default_context() @@ -62,7 +69,8 @@ class EmailRelay: context.check_hostname = False context.verify_mode = ssl.CERT_NONE relay_server.starttls(context=context) - relay_server.ehlo() # Say hello again after STARTTLS + logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to {mx_host}') + relay_server.ehlo(self.hostname) # Say hello again after STARTTLS with proper hostname logger.debug(f'TLS connection established to {mx_host}') else: logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text') @@ -94,13 +102,15 @@ class EmailRelay: # Try TLS with backup server too try: - backup_server.ehlo() + logger.debug(f'Sending EHLO {self.hostname} to backup {backup_mx}') + backup_server.ehlo(self.hostname) if backup_server.has_extn('starttls'): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE backup_server.starttls(context=context) - backup_server.ehlo() + logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to backup {backup_mx}') + backup_server.ehlo(self.hostname) logger.debug(f'TLS connection established to backup {backup_mx}') except Exception: logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text') diff --git a/email_server/server_runner.py b/email_server/server_runner.py index 5e7ad7e..1b6f51b 100644 --- a/email_server/server_runner.py +++ b/email_server/server_runner.py @@ -9,7 +9,7 @@ from email_server.tool_box import get_logger # Import our modules from email_server.models import create_tables -from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController +from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController, TLSController from email_server.tls_utils import generate_self_signed_cert, create_ssl_context from email_server.dkim_manager import DKIMManager from aiosmtpd.controller import Controller @@ -18,7 +18,7 @@ from aiosmtpd.smtp import SMTP as AIOSMTP settings = load_settings() SMTP_PORT = int(settings['Server']['SMTP_PORT']) SMTP_TLS_PORT = int(settings['Server']['SMTP_TLS_PORT']) -HOSTNAME = settings['Server']['HOSTNAME'] +HOSTNAME = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost')) LOG_LEVEL = settings['Logging']['LOG_LEVEL'] BIND_IP = settings['Server']['BIND_IP'] @@ -94,37 +94,25 @@ async def start_server(): logger.error("Failed to create SSL context") return + logger.debug(f"SSL context created: {ssl_context}") + logger.debug(f"SSL context type: {type(ssl_context)}") + # Start plain SMTP server (with IP whitelist fallback) handler_plain = EnhancedCustomSMTPHandler() controller_plain = PlainController( handler_plain, - hostname=BIND_IP, - server_hostname="TestEnvironment", + hostname=HOSTNAME, # Use proper hostname for HELO identification port=SMTP_PORT ) controller_plain.start() logger.debug(f'Starting plain SMTP server on {HOSTNAME}:{SMTP_PORT}...') - # Start TLS SMTP server using closure pattern like the original + # Start TLS SMTP server using the updated TLSController handler_tls = EnhancedCustomSMTPHandler() - - # Define TLS controller class with ssl_context in closure (like original) - class TLSController(Controller): - def factory(self): - return AIOSMTP( - self.handler, - tls_context=ssl_context, # Use ssl_context from closure - require_starttls=False, # Don't force STARTTLS, but make it available - auth_require_tls=True, # If auth is used, require TLS - authenticator=self.handler.combined_authenticator, - decode_data=True, - hostname=self.hostname - ) - controller_tls = TLSController( handler_tls, - hostname=BIND_IP, - server_hostname="TestEnvironment", + ssl_context=ssl_context, + hostname=HOSTNAME, # Use proper hostname for HELO identification port=SMTP_TLS_PORT ) controller_tls.start() diff --git a/email_server/settings_loader.py b/email_server/settings_loader.py index 3b5811f..7cf2dd8 100644 --- a/email_server/settings_loader.py +++ b/email_server/settings_loader.py @@ -8,40 +8,64 @@ from pathlib import Path SETTINGS_PATH = Path(__file__).parent.parent / 'settings.ini' -# Default values for settings.ini +# Default values and comments for settings.ini DEFAULTS = { 'Server': { + '; Server configuration for SMTP ports and hostname': None, + '; Plain SMTP port for internal/whitelisted IPs': None, 'SMTP_PORT': '4025', + '; STARTTLS SMTP port for authenticated users': None, 'SMTP_TLS_PORT': '40587', - 'HOSTNAME': 'localhost', + '; Server hostname for HELO/EHLO identification': None, + 'HOSTNAME': 'mail.example.com', + '; Override HELO hostname': None, + 'helo_hostname': 'mail.example.com', + '; IP address to bind to (0.0.0.0 = all interfaces), on Windows must use specific IP': None, 'BIND_IP': '0.0.0.0', + '; Custom server banner (to make it empty use "" must be double quotes)': None, + 'server_banner': "", }, 'Database': { + '; Database configuration': None, 'DATABASE_URL': 'sqlite:///email_server/server_data/smtp_server.db', }, 'Logging': { + '; Logging configuration': None, + '; Log level: DEBUG, INFO, WARNING, ERROR, CRITICAL': None, 'LOG_LEVEL': 'INFO', + '; Hide verbose aiosmtpd INFO messages when LOG_LEVEL = INFO': None, 'hide_info_aiosmtpd': 'true', }, 'Relay': { + '; Timeout in seconds for external SMTP connections': None, 'RELAY_TIMEOUT': '10', }, 'TLS': { + '; TLS/SSL certificate configuration': None, 'TLS_CERT_FILE': 'email_server/ssl_certs/server.crt', 'TLS_KEY_FILE': 'email_server/ssl_certs/server.key', }, 'DKIM': { + '; DKIM signing configuration': None, + '; RSA key size for DKIM keys (1024, 2048, 4096)': None, 'DKIM_KEY_SIZE': '2048', }, } def generate_settings_ini(settings_path: Path = SETTINGS_PATH) -> None: - """Generate settings.ini with default values if it does not exist.""" + """Generate settings.ini with default values and comments if it does not exist.""" if settings_path.exists(): return - config_parser = configparser.ConfigParser() + config_parser = configparser.ConfigParser(allow_no_value=True) for section, values in DEFAULTS.items(): - config_parser[section] = values + config_parser.add_section(section) + for key, value in values.items(): + if key.startswith(';'): + # This is a comment line + config_parser.set(section, key) + else: + # This is a setting with value + config_parser.set(section, key, value) with open(settings_path, 'w') as f: config_parser.write(f) diff --git a/email_server/smtp_handler.py b/email_server/smtp_handler.py index 5da2b3c..db100a1 100644 --- a/email_server/smtp_handler.py +++ b/email_server/smtp_handler.py @@ -9,17 +9,32 @@ Security Features: """ import uuid -import email.utils from datetime import datetime from aiosmtpd.smtp import SMTP as AIOSMTP, AuthResult from aiosmtpd.controller import Controller from email_server.auth import EnhancedAuthenticator, EnhancedIPAuthenticator, validate_sender_authorization, get_authenticated_domain_id from email_server.email_relay import EmailRelay from email_server.dkim_manager import DKIMManager +from email_server.settings_loader import load_settings from email_server.tool_box import get_logger logger = get_logger() +class CustomSMTP(AIOSMTP): + """Custom SMTP class with configurable banner.""" + + def __init__(self, *args, **kwargs): + # Sets Custom SMTP banner from settings + settings = load_settings() + _banner_message = settings['Server'].get('server_banner', '') + if _banner_message == '""': + _banner_message = '' + self.custom_banner = _banner_message + + super().__init__(*args, **kwargs) + # Override the __ident__ to use our custom banner + self.__ident__ = self.custom_banner + class EnhancedCombinedAuthenticator: """ Enhanced combined authenticator with sender validation support. @@ -61,51 +76,182 @@ class EnhancedCustomSMTPHandler: self.auth_require_tls = False self.auth_methods = ['LOGIN', 'PLAIN'] - def _ensure_required_headers(self, content: str, envelope, message_id: str) -> str: + def _ensure_required_headers(self, content: str, envelope, message_id: str, custom_headers: list = None) -> str: """Ensure all required email headers are present and properly formatted. - + + Following RFC 5322 header order and best practices for spam score reduction. + Optimized based on Gmail's header structure for better deliverability. + Args: content (str): Email content. envelope: SMTP envelope. message_id (str): Generated message ID. - + custom_headers (list): List of (name, value) tuples for custom headers. + Returns: - str: Email content with all required headers. + str: Email content with all required headers properly formatted. """ - import email - from email.parser import Parser - from email.policy import default - - # Parse the message using the email library - msg = Parser(policy=default).parsestr(content) - - # Set or add required headers if missing - if not msg.get('Message-ID'): - msg['Message-ID'] = f"<{message_id}@{envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else 'localhost'}>" - if not msg.get('Date'): - msg['Date'] = email.utils.formatdate(localtime=True) - if not msg.get('From'): - msg['From'] = envelope.mail_from - if not msg.get('To'): - msg['To'] = ', '.join(envelope.rcpt_tos) - if not msg.get('MIME-Version'): - msg['MIME-Version'] = '1.0' - if not msg.get('Content-Type'): - msg['Content-Type'] = 'text/plain; charset=utf-8' - if not msg.get('Subject'): - msg['Subject'] = '(No Subject)' - if not msg.get('Content-Transfer-Encoding'): - msg['Content-Transfer-Encoding'] = '7bit' - - # Ensure exactly one blank line between headers and body - # The email library will handle this when flattening - from io import StringIO - out = StringIO() - out.write(msg.as_string()) - return out.getvalue() + import email.utils + from email_server.settings_loader import load_settings + + try: + settings = load_settings() + fallback_hostname = settings.get('Server', 'HOSTNAME', fallback='localhost') + server_hostname = settings.get('Server', 'helo_hostname', fallback=fallback_hostname) + + logger.debug(f"Processing headers for message {message_id}") + + # Parse the message properly + if isinstance(content, bytes): + content = content.decode('utf-8', errors='replace') + + # Split content into lines and normalize line endings + lines = content.replace('\r\n', '\n').replace('\r', '\n').split('\n') + + # Find header/body boundary and collect existing headers + body_start = 0 + existing_headers = {} + original_header_order = [] + + for i, line in enumerate(lines): + if line.strip() == '': + body_start = i + 1 + break + if ':' in line and not line.startswith((' ', '\t')): + header_name, header_value = line.split(':', 1) + header_name_lower = header_name.strip().lower() + header_value = header_value.strip() + + # Handle continuation lines + j = i + 1 + while j < len(lines) and lines[j].startswith((' ', '\t')): + header_value += ' ' + lines[j].strip() + j += 1 + + existing_headers[header_name_lower] = header_value + original_header_order.append((header_name.strip(), header_value)) + logger.debug(f"Found existing header: {header_name_lower} = {header_value}") + + # Extract body and clean it + body_lines = lines[body_start:] if body_start < len(lines) else [] + while body_lines and body_lines[-1].strip() == '': + body_lines.pop() + body = '\n'.join(body_lines) + + # Build headers in optimized order based on Gmail's structure + required_headers = [] + + # 1. Message-ID (critical for spam filters) + if 'message-id' in existing_headers: + required_headers.append(f"Message-ID: {existing_headers['message-id']}") + else: + domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else server_hostname.replace('mail.', '') + required_headers.append(f"Message-ID: <{message_id}@{domain}>") + + # 2. Date (critical for spam filters) + if 'date' in existing_headers: + required_headers.append(f"Date: {existing_headers['date']}") + else: + date_str = email.utils.formatdate(localtime=True) + required_headers.append(f"Date: {date_str}") + + # 3. MIME-Version (declare MIME compliance early) + if 'mime-version' in existing_headers: + required_headers.append(f"MIME-Version: {existing_headers['mime-version']}") + else: + required_headers.append("MIME-Version: 1.0") + + # 4. User-Agent (if present, helps with reputation) + if 'user-agent' in existing_headers: + required_headers.append(f"User-Agent: {existing_headers['user-agent']}") + + # 5. Content-Language (if present) + if 'content-language' in existing_headers: + required_headers.append(f"Content-Language: {existing_headers['content-language']}") + + # 6. To (primary recipients - critical) + if 'to' in existing_headers: + required_headers.append(f"To: {existing_headers['to']}") + else: + to_list = ', '.join(envelope.rcpt_tos) + required_headers.append(f"To: {to_list}") + + # 7. From (sender identification - critical) + if 'from' in existing_headers: + required_headers.append(f"From: {existing_headers['from']}") + else: + required_headers.append(f"From: {envelope.mail_from}") + + # 8. Subject (message topic - critical) + if 'subject' in existing_headers: + required_headers.append(f"Subject: {existing_headers['subject']}") + else: + required_headers.append("Subject: ") + + # 9. Content-Type (media type information) + if 'content-type' in existing_headers: + required_headers.append(f"Content-Type: {existing_headers['content-type']}") + else: + required_headers.append("Content-Type: text/plain; charset=UTF-8; format=flowed") + + # 10. Content-Transfer-Encoding + if 'content-transfer-encoding' in existing_headers: + required_headers.append(f"Content-Transfer-Encoding: {existing_headers['content-transfer-encoding']}") + else: + required_headers.append("Content-Transfer-Encoding: 7bit") + + # Add custom headers after essential headers but before misc headers + if custom_headers: + for header_name, header_value in custom_headers: + # Skip if already added in essential headers + if header_name.lower() not in ['message-id', 'date', 'mime-version', 'user-agent', + 'content-language', 'to', 'from', 'subject', + 'content-type', 'content-transfer-encoding']: + required_headers.append(f"{header_name}: {header_value}") + logger.debug(f"Added custom header: {header_name}: {header_value}") + + # Add any other existing headers that weren't handled above + essential_headers = { + 'message-id', 'date', 'from', 'to', 'subject', + 'mime-version', 'content-type', 'content-transfer-encoding', + 'user-agent', 'content-language' + } + + # Preserve original header names and values for non-essential headers + for header_name, header_value in original_header_order: + if header_name.lower() not in essential_headers: + # Skip custom headers we already added + skip = False + if custom_headers: + for custom_name, _ in custom_headers: + if header_name.lower() == custom_name.lower(): + skip = True + break + if not skip: + required_headers.append(f"{header_name}: {header_value}") + + # Build final message + final_content = '\r\n'.join(required_headers) + if body.strip(): + final_content += '\r\n\r\n' + body + else: + final_content += '\r\n\r\n' + + logger.debug(f"Final headers for message {message_id}:") + for header in required_headers: + logger.debug(f" {header}") + + return final_content + + except Exception as e: + logger.error(f"Error ensuring headers: {e}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + # Fallback to original content if parsing fails + return content async def handle_DATA(self, server, session, envelope): - """Handle incoming email data.""" + """Handle incoming email data with improved header management.""" try: message_id = str(uuid.uuid4()) logger.debug(f'Received email {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}') @@ -119,26 +265,36 @@ class EnhancedCustomSMTPHandler: # Extract domain from sender for DKIM signing sender_domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else None - # Ensure required headers are present - content = self._ensure_required_headers(content, envelope, message_id) - - # Add custom headers before DKIM signing + # Get custom headers before processing + custom_headers = [] if sender_domain: custom_headers = self.dkim_manager.get_active_custom_headers(sender_domain) - for header_name, header_value in custom_headers: - # Insert header at the top of the message - content = f"{header_name}: {header_value}\r\n" + content - # Relay the email (all modifications done) + # Add beneficial headers for spam score improvement + client_ip = getattr(session, 'peer', ['unknown'])[0] if hasattr(session, 'peer') else None + if client_ip: + # Add X-Originating-IP header (helps with reputation) + custom_headers.append(('X-Originating-IP', f'[{client_ip}]')) + + # Add X-Mailer header for identification + custom_headers.append(('X-Mailer', 'NetBro Mail Server 1.0')) + + # Add X-Priority header (normal priority) + custom_headers.append(('X-Priority', '3')) + + # Ensure required headers are present (including custom headers) + content = self._ensure_required_headers(content, envelope, message_id, custom_headers) + + # DKIM-sign the final version of the message (only once, after all modifications) signed_content = content dkim_signed = False if sender_domain: - # DKIM-sign the final version of the message signed_content = self.dkim_manager.sign_email(content, sender_domain) dkim_signed = signed_content != content if dkim_signed: logger.debug(f'Email {message_id} signed with DKIM for domain {sender_domain}') + # Relay the email (no further modifications allowed) success = self.email_relay.relay_email( envelope.mail_from, envelope.rcpt_tos, @@ -200,28 +356,39 @@ class TLSController(Controller): """Custom controller with TLS support - modeled after the working original.""" def __init__(self, handler, ssl_context, hostname='localhost', port=40587): - self.ssl_context = ssl_context - super().__init__(handler, hostname=hostname, port=port) + logger.debug(f"TLSController __init__: ssl_context={ssl_context is not None}") + self._ssl_context = ssl_context # Use private attribute to avoid conflicts + self.smtp_hostname = hostname # Store for HELO identification + super().__init__(handler, hostname='0.0.0.0', port=port) # Bind to all interfaces def factory(self): - return AIOSMTP( + logger.debug(f"TLSController factory: ssl_context={self._ssl_context is not None}") + logger.debug(f"TLSController factory: ssl_context object={self._ssl_context}") + logger.debug(f"TLSController factory: hostname={self.smtp_hostname}") + smtp_instance = CustomSMTP( self.handler, - tls_context=self.ssl_context, - require_starttls=True, # Don't force STARTTLS, but make it available + tls_context=self._ssl_context, + require_starttls=False, # Don't require STARTTLS immediately, but make it available auth_require_tls=True, # If auth is used, require TLS authenticator=self.handler.combined_authenticator, decode_data=True, - hostname=self.hostname + hostname=self.smtp_hostname # Use proper hostname for HELO ) + logger.debug(f"TLSController CustomSMTP instance created with TLS: {hasattr(smtp_instance, 'tls_context')}") + return smtp_instance class PlainController(Controller): """Controller for plain SMTP with username/password and IP-based authentication.""" + def __init__(self, handler, hostname='localhost', port=4025): + self.smtp_hostname = hostname # Store for HELO identification + super().__init__(handler, hostname='0.0.0.0', port=port) # Bind to all interfaces + def factory(self): - return AIOSMTP( + return CustomSMTP( self.handler, authenticator=self.handler.combined_authenticator, auth_require_tls=False, # Allow AUTH over plain text (not recommended for production) decode_data=True, - hostname=self.hostname + hostname=self.smtp_hostname # Use proper hostname for HELO ) diff --git a/pymta-server.service b/pymta-server.service new file mode 100644 index 0000000..3908070 --- /dev/null +++ b/pymta-server.service @@ -0,0 +1,32 @@ +[Unit] +Description=PyMTA Email Server +After=network.target +StartLimitIntervalSec=0 +# check any errors when using this service: +# journalctl -u pymta-server.service -b -f + +[Service] +Type=simple +User=appuser +Group=appuser +WorkingDirectory=/opt/PyMTA-server +Environment=PYTHONUNBUFFERED=1 +ExecStart=/opt/PyMTA-server/.venv/bin/python main.py +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal + +# Security settings +# Capabilities for low ports < 1024 following 2 lines: +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +# if using port < 1024 comment out line bellow: +# NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ReadWritePaths=/opt/PyMTA-server +ProtectHome=true + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cc73bef..64954fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,8 @@ -# SMTP MTA server: +# SMTP MTA server +# then you can allow run ports < 1024 with +# create env with `python -m venv .venv --copies` (This will copy the Python binary) +# for f in /opt/PyMTA-server/.venv/bin/python*; do sudo setcap 'cap_net_bind_service=+ep' "$f"; done + aiosmtpd sqlalchemy pyOpenSSL