Merge pull request #2 from ghostersk/testing

Fixed email headers order and DKIM signing
This commit is contained in:
ghostersk
2025-06-07 09:00:27 +01:00
committed by GitHub
7 changed files with 312 additions and 87 deletions

View File

@@ -233,7 +233,7 @@ def main():
print("Database tables created successfully")
elif args.command == 'add-domain':
add_domain(args.domain, not args.no_auth)
add_domain(args.domain)
elif args.command == 'add-user':
add_user(args.email, args.password, args.domain)

View File

@@ -7,6 +7,7 @@ import smtplib
import ssl
from datetime import datetime
from email_server.models import Session, EmailLog
from email_server.settings_loader import load_settings
from email_server.tool_box import get_logger
logger = get_logger()
@@ -16,6 +17,11 @@ class EmailRelay:
def __init__(self):
self.timeout = 30 # Increased timeout for TLS negotiations
# Get the configured hostname for HELO/EHLO identification
settings = load_settings()
self.hostname = settings['Server'].get('helo_hostname',
settings['Server'].get('hostname', 'localhost'))
logger.debug(f"EmailRelay initialized with hostname: {self.hostname}")
def relay_email(self, mail_from, rcpt_tos, content):
"""Relay email to recipient's mail server with opportunistic TLS."""
@@ -53,8 +59,9 @@ class EmailRelay:
# Try to enable TLS if the server supports it
try:
# Check if server supports STARTTLS
relay_server.ehlo()
# Check if server supports STARTTLS - use proper hostname for EHLO
logger.debug(f'Sending EHLO {self.hostname} to {mx_host}')
relay_server.ehlo(self.hostname)
if relay_server.has_extn('starttls'):
logger.debug(f'Starting TLS connection to {mx_host}')
context = ssl.create_default_context()
@@ -62,7 +69,8 @@ class EmailRelay:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
relay_server.starttls(context=context)
relay_server.ehlo() # Say hello again after STARTTLS
logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to {mx_host}')
relay_server.ehlo(self.hostname) # Say hello again after STARTTLS with proper hostname
logger.debug(f'TLS connection established to {mx_host}')
else:
logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text')
@@ -94,13 +102,15 @@ class EmailRelay:
# Try TLS with backup server too
try:
backup_server.ehlo()
logger.debug(f'Sending EHLO {self.hostname} to backup {backup_mx}')
backup_server.ehlo(self.hostname)
if backup_server.has_extn('starttls'):
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
backup_server.starttls(context=context)
backup_server.ehlo()
logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to backup {backup_mx}')
backup_server.ehlo(self.hostname)
logger.debug(f'TLS connection established to backup {backup_mx}')
except Exception:
logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text')

View File

@@ -9,7 +9,7 @@ from email_server.tool_box import get_logger
# Import our modules
from email_server.models import create_tables
from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController
from email_server.smtp_handler import EnhancedCustomSMTPHandler, PlainController, TLSController
from email_server.tls_utils import generate_self_signed_cert, create_ssl_context
from email_server.dkim_manager import DKIMManager
from aiosmtpd.controller import Controller
@@ -18,7 +18,7 @@ from aiosmtpd.smtp import SMTP as AIOSMTP
settings = load_settings()
SMTP_PORT = int(settings['Server']['SMTP_PORT'])
SMTP_TLS_PORT = int(settings['Server']['SMTP_TLS_PORT'])
HOSTNAME = settings['Server']['HOSTNAME']
HOSTNAME = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost'))
LOG_LEVEL = settings['Logging']['LOG_LEVEL']
BIND_IP = settings['Server']['BIND_IP']
@@ -94,37 +94,25 @@ async def start_server():
logger.error("Failed to create SSL context")
return
logger.debug(f"SSL context created: {ssl_context}")
logger.debug(f"SSL context type: {type(ssl_context)}")
# Start plain SMTP server (with IP whitelist fallback)
handler_plain = EnhancedCustomSMTPHandler()
controller_plain = PlainController(
handler_plain,
hostname=BIND_IP,
server_hostname="TestEnvironment",
hostname=HOSTNAME, # Use proper hostname for HELO identification
port=SMTP_PORT
)
controller_plain.start()
logger.debug(f'Starting plain SMTP server on {HOSTNAME}:{SMTP_PORT}...')
# Start TLS SMTP server using closure pattern like the original
# Start TLS SMTP server using the updated TLSController
handler_tls = EnhancedCustomSMTPHandler()
# Define TLS controller class with ssl_context in closure (like original)
class TLSController(Controller):
def factory(self):
return AIOSMTP(
self.handler,
tls_context=ssl_context, # Use ssl_context from closure
require_starttls=False, # Don't force STARTTLS, but make it available
auth_require_tls=True, # If auth is used, require TLS
authenticator=self.handler.combined_authenticator,
decode_data=True,
hostname=self.hostname
)
controller_tls = TLSController(
handler_tls,
hostname=BIND_IP,
server_hostname="TestEnvironment",
ssl_context=ssl_context,
hostname=HOSTNAME, # Use proper hostname for HELO identification
port=SMTP_TLS_PORT
)
controller_tls.start()

View File

@@ -8,40 +8,64 @@ from pathlib import Path
SETTINGS_PATH = Path(__file__).parent.parent / 'settings.ini'
# Default values for settings.ini
# Default values and comments for settings.ini
DEFAULTS = {
'Server': {
'; Server configuration for SMTP ports and hostname': None,
'; Plain SMTP port for internal/whitelisted IPs': None,
'SMTP_PORT': '4025',
'; STARTTLS SMTP port for authenticated users': None,
'SMTP_TLS_PORT': '40587',
'HOSTNAME': 'localhost',
'; Server hostname for HELO/EHLO identification': None,
'HOSTNAME': 'mail.example.com',
'; Override HELO hostname': None,
'helo_hostname': 'mail.example.com',
'; IP address to bind to (0.0.0.0 = all interfaces), on Windows must use specific IP': None,
'BIND_IP': '0.0.0.0',
'; Custom server banner (to make it empty use "" must be double quotes)': None,
'server_banner': "",
},
'Database': {
'; Database configuration': None,
'DATABASE_URL': 'sqlite:///email_server/server_data/smtp_server.db',
},
'Logging': {
'; Logging configuration': None,
'; Log level: DEBUG, INFO, WARNING, ERROR, CRITICAL': None,
'LOG_LEVEL': 'INFO',
'; Hide verbose aiosmtpd INFO messages when LOG_LEVEL = INFO': None,
'hide_info_aiosmtpd': 'true',
},
'Relay': {
'; Timeout in seconds for external SMTP connections': None,
'RELAY_TIMEOUT': '10',
},
'TLS': {
'; TLS/SSL certificate configuration': None,
'TLS_CERT_FILE': 'email_server/ssl_certs/server.crt',
'TLS_KEY_FILE': 'email_server/ssl_certs/server.key',
},
'DKIM': {
'; DKIM signing configuration': None,
'; RSA key size for DKIM keys (1024, 2048, 4096)': None,
'DKIM_KEY_SIZE': '2048',
},
}
def generate_settings_ini(settings_path: Path = SETTINGS_PATH) -> None:
"""Generate settings.ini with default values if it does not exist."""
"""Generate settings.ini with default values and comments if it does not exist."""
if settings_path.exists():
return
config_parser = configparser.ConfigParser()
config_parser = configparser.ConfigParser(allow_no_value=True)
for section, values in DEFAULTS.items():
config_parser[section] = values
config_parser.add_section(section)
for key, value in values.items():
if key.startswith(';'):
# This is a comment line
config_parser.set(section, key)
else:
# This is a setting with value
config_parser.set(section, key, value)
with open(settings_path, 'w') as f:
config_parser.write(f)

View File

@@ -9,17 +9,32 @@ Security Features:
"""
import uuid
import email.utils
from datetime import datetime
from aiosmtpd.smtp import SMTP as AIOSMTP, AuthResult
from aiosmtpd.controller import Controller
from email_server.auth import EnhancedAuthenticator, EnhancedIPAuthenticator, validate_sender_authorization, get_authenticated_domain_id
from email_server.email_relay import EmailRelay
from email_server.dkim_manager import DKIMManager
from email_server.settings_loader import load_settings
from email_server.tool_box import get_logger
logger = get_logger()
class CustomSMTP(AIOSMTP):
"""Custom SMTP class with configurable banner."""
def __init__(self, *args, **kwargs):
# Sets Custom SMTP banner from settings
settings = load_settings()
_banner_message = settings['Server'].get('server_banner', '')
if _banner_message == '""':
_banner_message = ''
self.custom_banner = _banner_message
super().__init__(*args, **kwargs)
# Override the __ident__ to use our custom banner
self.__ident__ = self.custom_banner
class EnhancedCombinedAuthenticator:
"""
Enhanced combined authenticator with sender validation support.
@@ -61,51 +76,182 @@ class EnhancedCustomSMTPHandler:
self.auth_require_tls = False
self.auth_methods = ['LOGIN', 'PLAIN']
def _ensure_required_headers(self, content: str, envelope, message_id: str) -> str:
def _ensure_required_headers(self, content: str, envelope, message_id: str, custom_headers: list = None) -> str:
"""Ensure all required email headers are present and properly formatted.
Following RFC 5322 header order and best practices for spam score reduction.
Optimized based on Gmail's header structure for better deliverability.
Args:
content (str): Email content.
envelope: SMTP envelope.
message_id (str): Generated message ID.
custom_headers (list): List of (name, value) tuples for custom headers.
Returns:
str: Email content with all required headers.
str: Email content with all required headers properly formatted.
"""
import email
from email.parser import Parser
from email.policy import default
# Parse the message using the email library
msg = Parser(policy=default).parsestr(content)
# Set or add required headers if missing
if not msg.get('Message-ID'):
msg['Message-ID'] = f"<{message_id}@{envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else 'localhost'}>"
if not msg.get('Date'):
msg['Date'] = email.utils.formatdate(localtime=True)
if not msg.get('From'):
msg['From'] = envelope.mail_from
if not msg.get('To'):
msg['To'] = ', '.join(envelope.rcpt_tos)
if not msg.get('MIME-Version'):
msg['MIME-Version'] = '1.0'
if not msg.get('Content-Type'):
msg['Content-Type'] = 'text/plain; charset=utf-8'
if not msg.get('Subject'):
msg['Subject'] = '(No Subject)'
if not msg.get('Content-Transfer-Encoding'):
msg['Content-Transfer-Encoding'] = '7bit'
# Ensure exactly one blank line between headers and body
# The email library will handle this when flattening
from io import StringIO
out = StringIO()
out.write(msg.as_string())
return out.getvalue()
import email.utils
from email_server.settings_loader import load_settings
try:
settings = load_settings()
fallback_hostname = settings.get('Server', 'HOSTNAME', fallback='localhost')
server_hostname = settings.get('Server', 'helo_hostname', fallback=fallback_hostname)
logger.debug(f"Processing headers for message {message_id}")
# Parse the message properly
if isinstance(content, bytes):
content = content.decode('utf-8', errors='replace')
# Split content into lines and normalize line endings
lines = content.replace('\r\n', '\n').replace('\r', '\n').split('\n')
# Find header/body boundary and collect existing headers
body_start = 0
existing_headers = {}
original_header_order = []
for i, line in enumerate(lines):
if line.strip() == '':
body_start = i + 1
break
if ':' in line and not line.startswith((' ', '\t')):
header_name, header_value = line.split(':', 1)
header_name_lower = header_name.strip().lower()
header_value = header_value.strip()
# Handle continuation lines
j = i + 1
while j < len(lines) and lines[j].startswith((' ', '\t')):
header_value += ' ' + lines[j].strip()
j += 1
existing_headers[header_name_lower] = header_value
original_header_order.append((header_name.strip(), header_value))
logger.debug(f"Found existing header: {header_name_lower} = {header_value}")
# Extract body and clean it
body_lines = lines[body_start:] if body_start < len(lines) else []
while body_lines and body_lines[-1].strip() == '':
body_lines.pop()
body = '\n'.join(body_lines)
# Build headers in optimized order based on Gmail's structure
required_headers = []
# 1. Message-ID (critical for spam filters)
if 'message-id' in existing_headers:
required_headers.append(f"Message-ID: {existing_headers['message-id']}")
else:
domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else server_hostname.replace('mail.', '')
required_headers.append(f"Message-ID: <{message_id}@{domain}>")
# 2. Date (critical for spam filters)
if 'date' in existing_headers:
required_headers.append(f"Date: {existing_headers['date']}")
else:
date_str = email.utils.formatdate(localtime=True)
required_headers.append(f"Date: {date_str}")
# 3. MIME-Version (declare MIME compliance early)
if 'mime-version' in existing_headers:
required_headers.append(f"MIME-Version: {existing_headers['mime-version']}")
else:
required_headers.append("MIME-Version: 1.0")
# 4. User-Agent (if present, helps with reputation)
if 'user-agent' in existing_headers:
required_headers.append(f"User-Agent: {existing_headers['user-agent']}")
# 5. Content-Language (if present)
if 'content-language' in existing_headers:
required_headers.append(f"Content-Language: {existing_headers['content-language']}")
# 6. To (primary recipients - critical)
if 'to' in existing_headers:
required_headers.append(f"To: {existing_headers['to']}")
else:
to_list = ', '.join(envelope.rcpt_tos)
required_headers.append(f"To: {to_list}")
# 7. From (sender identification - critical)
if 'from' in existing_headers:
required_headers.append(f"From: {existing_headers['from']}")
else:
required_headers.append(f"From: {envelope.mail_from}")
# 8. Subject (message topic - critical)
if 'subject' in existing_headers:
required_headers.append(f"Subject: {existing_headers['subject']}")
else:
required_headers.append("Subject: ")
# 9. Content-Type (media type information)
if 'content-type' in existing_headers:
required_headers.append(f"Content-Type: {existing_headers['content-type']}")
else:
required_headers.append("Content-Type: text/plain; charset=UTF-8; format=flowed")
# 10. Content-Transfer-Encoding
if 'content-transfer-encoding' in existing_headers:
required_headers.append(f"Content-Transfer-Encoding: {existing_headers['content-transfer-encoding']}")
else:
required_headers.append("Content-Transfer-Encoding: 7bit")
# Add custom headers after essential headers but before misc headers
if custom_headers:
for header_name, header_value in custom_headers:
# Skip if already added in essential headers
if header_name.lower() not in ['message-id', 'date', 'mime-version', 'user-agent',
'content-language', 'to', 'from', 'subject',
'content-type', 'content-transfer-encoding']:
required_headers.append(f"{header_name}: {header_value}")
logger.debug(f"Added custom header: {header_name}: {header_value}")
# Add any other existing headers that weren't handled above
essential_headers = {
'message-id', 'date', 'from', 'to', 'subject',
'mime-version', 'content-type', 'content-transfer-encoding',
'user-agent', 'content-language'
}
# Preserve original header names and values for non-essential headers
for header_name, header_value in original_header_order:
if header_name.lower() not in essential_headers:
# Skip custom headers we already added
skip = False
if custom_headers:
for custom_name, _ in custom_headers:
if header_name.lower() == custom_name.lower():
skip = True
break
if not skip:
required_headers.append(f"{header_name}: {header_value}")
# Build final message
final_content = '\r\n'.join(required_headers)
if body.strip():
final_content += '\r\n\r\n' + body
else:
final_content += '\r\n\r\n'
logger.debug(f"Final headers for message {message_id}:")
for header in required_headers:
logger.debug(f" {header}")
return final_content
except Exception as e:
logger.error(f"Error ensuring headers: {e}")
import traceback
logger.error(f"Traceback: {traceback.format_exc()}")
# Fallback to original content if parsing fails
return content
async def handle_DATA(self, server, session, envelope):
"""Handle incoming email data."""
"""Handle incoming email data with improved header management."""
try:
message_id = str(uuid.uuid4())
logger.debug(f'Received email {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}')
@@ -119,26 +265,36 @@ class EnhancedCustomSMTPHandler:
# Extract domain from sender for DKIM signing
sender_domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else None
# Ensure required headers are present
content = self._ensure_required_headers(content, envelope, message_id)
# Add custom headers before DKIM signing
# Get custom headers before processing
custom_headers = []
if sender_domain:
custom_headers = self.dkim_manager.get_active_custom_headers(sender_domain)
for header_name, header_value in custom_headers:
# Insert header at the top of the message
content = f"{header_name}: {header_value}\r\n" + content
# Relay the email (all modifications done)
# Add beneficial headers for spam score improvement
client_ip = getattr(session, 'peer', ['unknown'])[0] if hasattr(session, 'peer') else None
if client_ip:
# Add X-Originating-IP header (helps with reputation)
custom_headers.append(('X-Originating-IP', f'[{client_ip}]'))
# Add X-Mailer header for identification
custom_headers.append(('X-Mailer', 'NetBro Mail Server 1.0'))
# Add X-Priority header (normal priority)
custom_headers.append(('X-Priority', '3'))
# Ensure required headers are present (including custom headers)
content = self._ensure_required_headers(content, envelope, message_id, custom_headers)
# DKIM-sign the final version of the message (only once, after all modifications)
signed_content = content
dkim_signed = False
if sender_domain:
# DKIM-sign the final version of the message
signed_content = self.dkim_manager.sign_email(content, sender_domain)
dkim_signed = signed_content != content
if dkim_signed:
logger.debug(f'Email {message_id} signed with DKIM for domain {sender_domain}')
# Relay the email (no further modifications allowed)
success = self.email_relay.relay_email(
envelope.mail_from,
envelope.rcpt_tos,
@@ -200,28 +356,39 @@ class TLSController(Controller):
"""Custom controller with TLS support - modeled after the working original."""
def __init__(self, handler, ssl_context, hostname='localhost', port=40587):
self.ssl_context = ssl_context
super().__init__(handler, hostname=hostname, port=port)
logger.debug(f"TLSController __init__: ssl_context={ssl_context is not None}")
self._ssl_context = ssl_context # Use private attribute to avoid conflicts
self.smtp_hostname = hostname # Store for HELO identification
super().__init__(handler, hostname='0.0.0.0', port=port) # Bind to all interfaces
def factory(self):
return AIOSMTP(
logger.debug(f"TLSController factory: ssl_context={self._ssl_context is not None}")
logger.debug(f"TLSController factory: ssl_context object={self._ssl_context}")
logger.debug(f"TLSController factory: hostname={self.smtp_hostname}")
smtp_instance = CustomSMTP(
self.handler,
tls_context=self.ssl_context,
require_starttls=True, # Don't force STARTTLS, but make it available
tls_context=self._ssl_context,
require_starttls=False, # Don't require STARTTLS immediately, but make it available
auth_require_tls=True, # If auth is used, require TLS
authenticator=self.handler.combined_authenticator,
decode_data=True,
hostname=self.hostname
hostname=self.smtp_hostname # Use proper hostname for HELO
)
logger.debug(f"TLSController CustomSMTP instance created with TLS: {hasattr(smtp_instance, 'tls_context')}")
return smtp_instance
class PlainController(Controller):
"""Controller for plain SMTP with username/password and IP-based authentication."""
def __init__(self, handler, hostname='localhost', port=4025):
self.smtp_hostname = hostname # Store for HELO identification
super().__init__(handler, hostname='0.0.0.0', port=port) # Bind to all interfaces
def factory(self):
return AIOSMTP(
return CustomSMTP(
self.handler,
authenticator=self.handler.combined_authenticator,
auth_require_tls=False, # Allow AUTH over plain text (not recommended for production)
decode_data=True,
hostname=self.hostname
hostname=self.smtp_hostname # Use proper hostname for HELO
)

32
pymta-server.service Normal file
View File

@@ -0,0 +1,32 @@
[Unit]
Description=PyMTA Email Server
After=network.target
StartLimitIntervalSec=0
# check any errors when using this service:
# journalctl -u pymta-server.service -b -f
[Service]
Type=simple
User=appuser
Group=appuser
WorkingDirectory=/opt/PyMTA-server
Environment=PYTHONUNBUFFERED=1
ExecStart=/opt/PyMTA-server/.venv/bin/python main.py
Restart=always
RestartSec=5
StandardOutput=journal
StandardError=journal
# Security settings
# Capabilities for low ports < 1024 following 2 lines:
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
# if using port < 1024 comment out line bellow:
# NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/opt/PyMTA-server
ProtectHome=true
[Install]
WantedBy=multi-user.target

View File

@@ -1,4 +1,8 @@
# SMTP MTA server:
# SMTP MTA server
# then you can allow run ports < 1024 with
# create env with `python -m venv .venv --copies` (This will copy the Python binary)
# for f in /opt/PyMTA-server/.venv/bin/python*; do sudo setcap 'cap_net_bind_service=+ep' "$f"; done
aiosmtpd
sqlalchemy
pyOpenSSL