diff --git a/.flaskenv b/.flaskenv index 72b17fb..6537a6b 100644 --- a/.flaskenv +++ b/.flaskenv @@ -1,2 +1,2 @@ -FLASK_APP=app:create_app +FLASK_APP=app:flask_app FLASK_ENV=development \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0e5f122..d4170e8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ *$py.class # Certs, db, private test files +attachments/ settings.ini *.crt *.key diff --git a/app.py b/app.py index d368d70..d332825 100644 --- a/app.py +++ b/app.py @@ -31,7 +31,7 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__))) # Import SMTP server components from email_server.server_runner import start_server -from email_server.models import create_tables, Session, Domain, User, WhitelistedIP, DKIMKey, hash_password +from email_server.models import create_tables, Session, Domain, Sender, WhitelistedIP, DKIMKey, hash_password from email_server.settings_loader import load_settings from email_server.tool_box import get_logger from email_server.dkim_manager import DKIMManager @@ -91,7 +91,7 @@ class SMTPServerApp: db = SQLAlchemy(app) # Import existing models and register them with Flask-SQLAlchemy - from email_server.models import Base, Domain, User, WhitelistedIP, DKIMKey, EmailLog, AuthLog, CustomHeader + from email_server.models import Base, Domain, Sender, WhitelistedIP, DKIMKey, EmailLog, AuthLog, CustomHeader # Set the metadata for Flask-Migrate to use existing models db.Model.metadata = Base.metadata @@ -109,71 +109,7 @@ class SMTPServerApp: def index(): """Redirect root to email dashboard""" return redirect(url_for('email.dashboard')) - - @app.route('/health') - def health_check(): - """Health check endpoint""" - return jsonify({ - 'status': 'healthy', - 'timestamp': datetime.now(ZoneInfo('Europe/London')).isoformat(), - 'services': { - 'smtp_server': 'running' if self.smtp_task and not self.smtp_task.done() else 'stopped', - 'web_frontend': 'running' - }, - 'version': '1.0.0' - }) - - @app.route('/api/server/status') - def server_status(): - """Get detailed server status""" - session = Session() - try: - status = { - 'smtp_server': { - 'running': self.smtp_task and not self.smtp_task.done(), - 'port': int(self.settings.get('Server', 'SMTP_PORT', fallback=25)), - 'tls_port': int(self.settings.get('Server', 'SMTP_TLS_PORT', fallback=587)), - 'hostname': self.settings.get('Server', 'hostname', fallback='localhost') - }, - 'database': { - 'domains': session.query(Domain).filter_by(is_active=True).count(), - 'users': session.query(User).filter_by(is_active=True).count(), - 'dkim_keys': session.query(DKIMKey).filter_by(is_active=True).count(), - 'whitelisted_ips': session.query(WhitelistedIP).filter_by(is_active=True).count() - }, - 'settings': { - 'relay_enabled': self.settings.getboolean('Relay', 'enable_relay', fallback=False), - 'tls_enabled': self.settings.getboolean('TLS', 'enable_tls', fallback=True), - 'dkim_enabled': self.settings.getboolean('DKIM', 'enable_dkim', fallback=True) - } - } - return jsonify(status) - finally: - session.close() - - @app.route('/api/server/restart', methods=['POST']) - def restart_server(): - """Restart the SMTP server (API endpoint) via systemd.""" - try: - # Only allow from localhost for security - if request.remote_addr not in ('127.0.0.1', '::1'): - return jsonify({'status': 'error', 'message': 'Unauthorized'}), 403 - # Restart the systemd service for SMTP (update service name as needed) - result = subprocess.run( - ['systemctl', '--user', 'restart', 'pymta-smtp.service'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - if result.returncode == 0: - return jsonify({'status': 'success', 'message': 'SMTP server restart requested.'}) - else: - return jsonify({'status': 'error', 'message': f'Failed to restart: {result.stderr}'}), 500 - except Exception as e: - logger.error(f"Error restarting server: {e}") - return jsonify({'status': 'error', 'message': str(e)}), 500 - # Error handlers @app.errorhandler(404) def not_found_error(error): @@ -192,6 +128,11 @@ class SMTPServerApp: error_message="Internal server error", error_details=str(error)), 500 + @app.route('/health') + def health_check(): + """Health check endpoint""" + return jsonify(self.check_health()) + # Context processors for templates @app.context_processor def utility_processor(): @@ -203,6 +144,7 @@ class SMTPServerApp: 'zip': zip, 'str': str, 'int': int, + 'check_health': self.check_health } self.flask_app = app @@ -235,24 +177,24 @@ class SMTPServerApp: for domain_name in sample_domains: dkim_manager.generate_dkim_keypair(domain_name) - # Add sample users - sample_users = [ + # Add sample senders + sample_senders = [ ('admin@example.com', 'example.com', 'admin123', False), ] - for email, domain_name, password, can_send_as_domain in sample_users: - existing = session.query(User).filter_by(email=email).first() + for email, domain_name, password, can_send_as_domain in sample_senders: + existing = session.query(Sender).filter_by(email=email).first() if not existing: domain = session.query(Domain).filter_by(domain_name=domain_name).first() if domain: - user = User( + sender = Sender( email=email, password_hash=hash_password(password), domain_id=domain.id, can_send_as_domain=can_send_as_domain ) - session.add(user) - logger.info(f"Added sample user: {email}") + session.add(sender) + logger.info(f"Added sample sender: {email}") # Add sample whitelisted IPs sample_ips = [ @@ -374,6 +316,42 @@ class SMTPServerApp: finally: self.shutdown_requested = True + def check_health(self): + """Check the health of all services""" + status = { + 'status': 'healthy', + 'timestamp': datetime.now(ZoneInfo('Europe/London')).isoformat(), + 'services': { + 'smtp_server': 'running' if self.smtp_task and not self.smtp_task.done() else 'stopped', + 'web_frontend': 'running', + 'database': 'ok' + }, + 'version': '1.0.0' + } + + # Check database connection + try: + session = Session() + # Try to query a simple table to verify connection + session.query(Domain).first() + session.close() + except Exception as e: + status['services']['database'] = 'error' + status['status'] = 'degraded' + logger.error(f"Database health check failed: {e}") + # Try to reconnect to database + try: + create_tables() + logger.info("Database reconnection attempted") + except Exception as reconnect_error: + logger.error(f"Database reconnection failed: {reconnect_error}") + + # If any service is not running, set overall status to degraded + if status['services']['smtp_server'] == 'stopped' or status['services']['database'] == 'error': + status['status'] = 'degraded' + + return status + def main(): """Main function""" diff --git a/email_server/auth.py b/email_server/auth.py index ea03da0..fa43b63 100644 --- a/email_server/auth.py +++ b/email_server/auth.py @@ -11,8 +11,8 @@ Security Features: from datetime import datetime from aiosmtpd.smtp import AuthResult, LoginPassword from email_server.models import ( - Session, User, Domain, WhitelistedIP, - check_password, log_auth_attempt, get_user_by_email, + Session, Sender, Domain, WhitelistedIP, + check_password, log_auth_attempt, get_sender_by_email, get_whitelisted_ip, get_domain_by_name ) from email_server.tool_box import get_logger @@ -32,7 +32,7 @@ class EnhancedAuthenticator: def __call__(self, server, session, envelope, mechanism, auth_data): if not isinstance(auth_data, LoginPassword): logger.warning(f'Invalid auth data format: {type(auth_data)}') - return AuthResult(success=False, handled=True, message='535 Authentication failed') + return AuthResult(success=False, handled=False, message='535 Authentication failed') # Decode bytes to string if necessary username = auth_data.login @@ -47,48 +47,45 @@ class EnhancedAuthenticator: logger.debug(f'Authentication attempt: {username} from {peer_ip}') try: - # Look up user in database - user = get_user_by_email(username) - - if user and check_password(password, user.password_hash): - # Store authenticated user info in session for later validation - session.authenticated_user = user - session.auth_type = 'user' - + # Look up sender in database + sender = get_sender_by_email(username) + if sender and check_password(password, sender.password_hash): + # Store authenticated sender info in session for later validation + session.authenticated_sender = sender + session.auth_type = 'sender' + session.username = username # Store username in session # Log successful authentication log_auth_attempt( - auth_type='user', + auth_type='sender', identifier=username, ip_address=peer_ip, success=True, - message=f'Successful user authentication' + message=f'Successful sender authentication' ) - - logger.info(f'Authenticated user: {username} (ID: {user.id}, can_send_as_domain: {user.can_send_as_domain})') + logger.info(f'Authenticated sender: {username} (ID: {sender.id}, can_send_as_domain: {sender.can_send_as_domain})') return AuthResult(success=True, handled=True) else: # Log failed authentication log_auth_attempt( - auth_type='user', + auth_type='sender', identifier=username, ip_address=peer_ip, success=False, message=f'Invalid credentials for {username}' ) - logger.warning(f'Authentication failed for {username}: invalid credentials') - return AuthResult(success=False, handled=True, message='535 Authentication failed') + return AuthResult(success=False, handled=False, message='535 Authentication failed') except Exception as e: logger.error(f'Authentication error for {username}: {e}') log_auth_attempt( - auth_type='user', + auth_type='sender', identifier=username, ip_address=peer_ip, success=False, message=f'Authentication error: {str(e)}' ) - return AuthResult(success=False, handled=True, message='451 Internal server error') + return AuthResult(success=False, handled=False, message='451 Internal server error') class EnhancedIPAuthenticator: """ @@ -145,19 +142,18 @@ def validate_sender_authorization(session, mail_from: str) -> tuple[bool, str]: peer_ip = session.peer[0] - # Check user authentication - if hasattr(session, 'authenticated_user') and session.authenticated_user: - user = session.authenticated_user - - if user.can_send_as(mail_from): - logger.info(f"User {user.email} authorized to send as {mail_from}") - return True, f"User authorized to send as {mail_from}" + # Check sender authentication + if hasattr(session, 'authenticated_sender') and session.authenticated_sender: + sender = session.authenticated_sender + if sender.can_send_as(mail_from): + logger.info(f"Sender {sender.email} authorized to send as {mail_from}") + return True, f"Sender authorized to send as {mail_from}" else: - message = f"User {user.email} not authorized to send as {mail_from}" + message = f"Sender {sender.email} not authorized to send as {mail_from}" logger.warning(message) log_auth_attempt( auth_type='sender_validation', - identifier=f"{user.email} -> {mail_from}", + identifier=f"{sender.email} -> {mail_from}", ip_address=peer_ip, success=False, message=message @@ -172,6 +168,7 @@ def validate_sender_authorization(session, mail_from: str) -> tuple[bool, str]: # Store IP auth info in session session.auth_type = 'ip' session.authorized_domain = from_domain + session.username = f"IP:{peer_ip}" # Store IP as username for IP authentication log_auth_attempt( auth_type='ip', @@ -205,8 +202,8 @@ def get_authenticated_domain_id(session) -> int: Returns: Domain ID or None if not authenticated """ - if hasattr(session, 'authenticated_user') and session.authenticated_user: - return session.authenticated_user.domain_id + if hasattr(session, 'authenticated_sender') and session.authenticated_sender: + return session.authenticated_sender.domain_id if hasattr(session, 'authorized_domain') and session.authorized_domain: domain = get_domain_by_name(session.authorized_domain) diff --git a/email_server/dkim_manager.py b/email_server/dkim_manager.py index 0b3274c..74b1f0b 100644 --- a/email_server/dkim_manager.py +++ b/email_server/dkim_manager.py @@ -8,7 +8,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from datetime import datetime from email_server.models import Session, Domain, DKIMKey, CustomHeader from email_server.settings_loader import load_settings -from email_server.tool_box import get_logger +from email_server.tool_box import get_logger, get_current_time import random import string @@ -55,7 +55,7 @@ class DKIMManager: existing_active_keys = session.query(DKIMKey).filter_by(domain_id=domain.id, is_active=True).all() for existing_key in existing_active_keys: existing_key.is_active = False - existing_key.replaced_at = datetime.now() + existing_key.replaced_at = get_current_time() logger.debug(f"Marked DKIM key as replaced for domain {domain_name} selector {existing_key.selector}") # Check if we're reusing an existing selector - if so, reactivate instead of creating new @@ -75,7 +75,7 @@ class DKIMManager: ).all() for key in other_active_keys: key.is_active = False - key.replaced_at = datetime.now() + key.replaced_at = get_current_time() logger.debug(f"Deactivated other active DKIM key for domain {domain_name} selector {key.selector}") # Reactivate existing key with same selector, clear replaced_at timestamp existing_key_with_selector.is_active = True @@ -114,7 +114,7 @@ class DKIMManager: selector=use_selector, private_key=private_pem, public_key=public_pem, - created_at=datetime.now(), + created_at=get_current_time(), is_active=True ) session.add(dkim_key) diff --git a/email_server/email_relay.py b/email_server/email_relay.py index 3831380..71da166 100644 --- a/email_server/email_relay.py +++ b/email_server/email_relay.py @@ -2,156 +2,346 @@ Email relay functionality for the SMTP server. """ +import asyncio import dns.resolver -import smtplib -import ssl -from datetime import datetime -from email_server.models import Session, EmailLog +from email_server.models import Session, EmailLog, EmailRecipientLog from email_server.settings_loader import load_settings -from email_server.tool_box import get_logger +from email_server.tool_box import get_logger, get_current_time +import aiosmtplib logger = get_logger() +settings = load_settings() +_relay_tls_timeout = settings['Server'].get('relay_timeout', 30) + +port = 25 # Default MX SMTP port for relaying emails + class EmailRelay: """Handles relaying emails to recipient mail servers.""" - - def __init__(self): - self.timeout = 30 # Increased timeout for TLS negotiations - # Get the configured hostname for HELO/EHLO identification - settings = load_settings() - self.hostname = settings['Server'].get('helo_hostname', - settings['Server'].get('hostname', 'localhost')) - logger.debug(f"EmailRelay initialized with hostname: {self.hostname}") - - def relay_email(self, mail_from, rcpt_tos, content): - """Relay email to recipient's mail server with opportunistic TLS.""" - try: - for rcpt in rcpt_tos: - domain = rcpt.split('@')[1] - - # Resolve MX record for the domain - try: - mx_records = dns.resolver.resolve(domain, 'MX') - # Sort by priority (lower number = higher priority) - mx_records = sorted(mx_records, key=lambda x: x.preference) - mx_host = mx_records[0].exchange.to_text().rstrip('.') - logger.debug(f'Found MX record for {domain}: {mx_host}') - except Exception as e: - logger.error(f'Failed to resolve MX for {domain}: {e}') - return False - # Try to relay with opportunistic TLS - if not self._relay_with_opportunistic_tls(mail_from, rcpt, content, mx_host): - return False - - return True - except Exception as e: - logger.error(f'General relay error: {e}') - return False - - def _relay_with_opportunistic_tls(self, mail_from, rcpt, content, mx_host): - """Relay email with opportunistic TLS (like Gmail does).""" - try: - # First, try with STARTTLS (encrypted) + def __init__(self): + + self.timeout = _relay_tls_timeout # Increased timeout for TLS negotiations + # Get the configured hostname for HELO/EHLO identification + self.hostname = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost')) + logger.debug(f"EmailRelay initialized with hostname: {self.hostname}") + + def _modify_headers_for_recipients(self, content, to_addresses, cc_addresses=None): + """Modify email headers to set To and Cc fields, preserving original structure for DKIM. + + Args: + content: Raw email content + to_addresses: List of TO recipients + cc_addresses: List of CC recipients (optional) + """ + lines = content.splitlines() + new_headers = [] + body_start = 0 + has_to = False + has_cc = False + + # First pass: find header/body boundary and examine existing headers + for i, line in enumerate(lines): + if line.strip() == '': + body_start = i + break + # Skip BCC headers but preserve TO and CC + if line.lower().startswith('bcc:'): + continue + # Track if we have TO/CC headers + if line.lower().startswith('to:'): + has_to = True + elif line.lower().startswith('cc:'): + has_cc = True + new_headers.append(line) + + # Only add headers if they don't exist + if not has_to and to_addresses: + new_headers.append(f"To: {', '.join(to_addresses)}") + if not has_cc and cc_addresses: + new_headers.append(f"Cc: {', '.join(cc_addresses)}") + + # Reconstruct the message + body = '\n'.join(lines[body_start:]) if body_start < len(lines) else '' + return '\r\n'.join(new_headers) + '\r\n\r\n' + body + + def _prepare_email_for_recipient(self, content: str, bcc_recipient: str = None) -> str: + """Prepare a copy of the email for a specific recipient without modifying original content. + + Args: + content: The original signed email content + bcc_recipient: If specified, prepare content for this BCC recipient + + Returns: + str: Email content ready for the specific recipient + """ + lines = content.splitlines() + new_lines = [] + headers_done = False + empty_line_added = False + + for line in lines: + if not headers_done: + if line.strip() == '': + headers_done = True + empty_line_added = True + new_lines.append(line) # Keep the empty line separator + # Skip BCC headers + elif not line.lower().startswith('bcc:'): + new_lines.append(line) + else: + new_lines.append(line) + + # Ensure there's a blank line between headers and body if not already present + if not empty_line_added: + new_lines.append('') + + return '\r\n'.join(new_lines) + + async def relay_email_async( + self, + mail_from: str, + rcpt_tos: list[str], + content: str, + username: str = None, + cc_addresses: list[str] = None, + bcc_addresses: list[str] = None, + recipient_types: list[str] = None + ) -> list[dict]: + """Relay email to recipients' mail servers asynchronously with encryption. + Preserves DKIM signatures by not modifying the signed content.""" + results = [] + recipient_type_map = {} + if recipient_types and len(recipient_types) == len(rcpt_tos): + for addr, rtype in zip(rcpt_tos, recipient_types): + recipient_type_map[addr] = rtype + else: + for addr in rcpt_tos: + recipient_type_map[addr] = 'to' + + # Separate visible recipients (TO/CC) and BCC recipients + visible_recipients = [] + bcc_list = [] + + for rcpt in rcpt_tos: + if recipient_type_map.get(rcpt) in ['to', 'cc']: + visible_recipients.append(rcpt) + elif recipient_type_map.get(rcpt) == 'bcc': + bcc_list.append(rcpt) + + # Group recipients by domain for efficient delivery + domain_groups = {} + for rcpt in visible_recipients: + domain = rcpt.split('@')[1].lower() + rtype = recipient_type_map.get(rcpt, 'to') + if domain not in domain_groups: + domain_groups[domain] = {'to': [], 'cc': [], 'bcc': []} + domain_groups[domain][rtype].append(rcpt) + + # Handle TO/CC recipients - use original signed content + for domain, recipients in domain_groups.items(): + to_recipients = recipients['to'] + cc_recipients = recipients['cc'] + if not to_recipients and not cc_recipients: + continue + + # Prepare content for TO/CC recipients without modifying headers + prepared_content = self._prepare_email_for_recipient(content) + try: - with smtplib.SMTP(mx_host, 25, timeout=self.timeout) as relay_server: - relay_server.set_debuglevel(1) - - # Try to enable TLS if the server supports it - try: - # Check if server supports STARTTLS - use proper hostname for EHLO - logger.debug(f'Sending EHLO {self.hostname} to {mx_host}') - relay_server.ehlo(self.hostname) - if relay_server.has_extn('starttls'): - logger.debug(f'Starting TLS connection to {mx_host}') - context = ssl.create_default_context() - # Allow self-signed certificates for mail servers - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - relay_server.starttls(context=context) - logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to {mx_host}') - relay_server.ehlo(self.hostname) # Say hello again after STARTTLS with proper hostname - logger.debug(f'TLS connection established to {mx_host}') - else: - logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text') - except Exception as tls_e: - logger.warning(f'STARTTLS failed with {mx_host}, continuing with plain text: {tls_e}') - - # Send the email - relay_server.sendmail(mail_from, rcpt, content) - logger.debug(f'Successfully relayed email to {rcpt} via {mx_host}') - return True - + mx_records = dns.resolver.resolve(domain, 'MX') + mx_records = sorted(mx_records, key=lambda x: x.preference) + mx_hosts = [mx.exchange.to_text().rstrip('.') for mx in mx_records] + logger.debug(f'Found MX records for {domain}: {mx_hosts}') except Exception as e: - logger.error(f'Failed to relay email to {rcpt} via {mx_host}: {e}') - - # Fallback: try alternative MX records if available + logger.error(f'Failed to resolve MX for {domain}: {e}') + for rcpt in to_recipients + cc_recipients: + results.append({ + 'recipient': rcpt, + 'status': 'failed', + 'error_code': 'MX', + 'error_message': str(e), + 'server_response': None, + 'recipient_type': recipient_type_map.get(rcpt, 'to') + }) + continue + + delivered = False + last_error = None + for mx_host in mx_hosts: try: - domain = rcpt.split('@')[1] - mx_records = dns.resolver.resolve(domain, 'MX') - mx_records = sorted(mx_records, key=lambda x: x.preference) - - # Try other MX records - for mx_record in mx_records[1:3]: # Try up to 2 backup MX records - backup_mx = mx_record.exchange.to_text().rstrip('.') - logger.debug(f'Trying backup MX record: {backup_mx}') - - try: - with smtplib.SMTP(backup_mx, 25, timeout=self.timeout) as backup_server: - backup_server.set_debuglevel(1) - - # Try TLS with backup server too - try: - logger.debug(f'Sending EHLO {self.hostname} to backup {backup_mx}') - backup_server.ehlo(self.hostname) - if backup_server.has_extn('starttls'): - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - backup_server.starttls(context=context) - logger.debug(f'Sending EHLO {self.hostname} again after STARTTLS to backup {backup_mx}') - backup_server.ehlo(self.hostname) - logger.debug(f'TLS connection established to backup {backup_mx}') - except Exception: - logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text') - - backup_server.sendmail(mail_from, rcpt, content) - logger.debug(f'Successfully relayed email to {rcpt} via backup {backup_mx}') - return True - except Exception as backup_e: - logger.warning(f'Backup MX {backup_mx} also failed: {backup_e}') - continue - - except Exception as fallback_e: - logger.error(f'All MX records failed for {rcpt}: {fallback_e}') + smtp = aiosmtplib.SMTP(hostname=mx_host, port=port, timeout=self.timeout, local_hostname=self.hostname) + await smtp.connect() + ext = getattr(smtp, 'extensions', None) + if ext is None: + ext = getattr(smtp, 'esmtp_extensions', None) + if ext is None: + logger.error(f"SMTP object has no 'extensions' or 'esmtp_extensions'. Available attributes: {dir(smtp)}") + ext = {} + if 'starttls' in ext: + logger.debug(f'STARTTLS supported by {mx_host}:{port}, upgrading to TLS') + await smtp.starttls() + else: + logger.warning(f'STARTTLS not supported by {mx_host}:{port}, sending in plain text!') + response = await smtp.sendmail(mail_from, to_recipients + cc_recipients, prepared_content) + logger.debug(f'Successfully relayed email to {to_recipients + cc_recipients} via {mx_host}:{port}') + for rcpt in to_recipients + cc_recipients: + results.append({ + 'recipient': rcpt, + 'status': 'success', + 'error_code': None, + 'error_message': None, + 'server_response': str(response), + 'recipient_type': recipient_type_map.get(rcpt, 'to') + }) + await smtp.quit() + delivered = True + break + except Exception as e: + logger.error(f'Failed to relay email to {to_recipients + cc_recipients} via {mx_host}:{port}: {e}') + last_error = { + 'status': 'failed', + 'error_code': 'RELAY', + 'error_message': str(e), + 'server_response': None + } + continue + + if not delivered and last_error: + for rcpt in to_recipients + cc_recipients: + results.append({ + 'recipient': rcpt, + 'status': last_error['status'], + 'error_code': last_error['error_code'], + 'error_message': last_error['error_message'], + 'server_response': last_error['server_response'], + 'recipient_type': recipient_type_map.get(rcpt, 'to') + }) + + # Handle BCC recipients - each gets their own copy with original headers + for bcc in bcc_list: + domain = bcc.split('@')[1].lower() + # Prepare content for BCC recipient - remove BCC headers but keep everything else + prepared_content = self._prepare_email_for_recipient(content, bcc) + + try: + mx_records = dns.resolver.resolve(domain, 'MX') + mx_records = sorted(mx_records, key=lambda x: x.preference) + mx_hosts = [mx.exchange.to_text().rstrip('.') for mx in mx_records] + logger.debug(f'Found MX records for {domain}: {mx_hosts}') + except Exception as e: + logger.error(f'Failed to resolve MX for {domain}: {e}') + results.append({ + 'recipient': bcc, + 'status': 'failed', + 'error_code': 'MX', + 'error_message': str(e), + 'server_response': None, + 'recipient_type': 'bcc' + }) + continue + + delivered = False + last_error = None + for mx_host in mx_hosts: + try: + smtp = aiosmtplib.SMTP(hostname=mx_host, port=port, timeout=self.timeout, local_hostname=self.hostname) + await smtp.connect() + ext = getattr(smtp, 'extensions', None) + if ext is None: + ext = getattr(smtp, 'esmtp_extensions', None) + if ext is None: + logger.error(f"SMTP object has no 'extensions' or 'esmtp_extensions'. Available attributes: {dir(smtp)}") + ext = {} + if 'starttls' in ext: + logger.debug(f'STARTTLS supported by {mx_host}:{port} for BCC, upgrading to TLS') + await smtp.starttls() + else: + logger.warning(f'STARTTLS not supported by {mx_host}:{port} for BCC, sending in plain text!') + response = await smtp.sendmail(mail_from, [bcc], prepared_content) + logger.debug(f'Successfully relayed BCC email to {bcc} via {mx_host}:{port}') + results.append({ + 'recipient': bcc, + 'status': 'success', + 'error_code': None, + 'error_message': None, + 'server_response': str(response), + 'recipient_type': 'bcc' + }) + await smtp.quit() + delivered = True + break + except Exception as e: + logger.error(f'Failed to relay BCC email to {bcc} via {mx_host}:{port}: {e}') + last_error = { + 'status': 'failed', + 'error_code': 'RELAY', + 'error_message': str(e), + 'server_response': None + } + continue + + if not delivered and last_error: + results.append({ + 'recipient': bcc, + 'status': last_error['status'], + 'error_code': last_error['error_code'], + 'error_message': last_error['error_message'], + 'server_response': last_error['server_response'], + 'recipient_type': 'bcc' + }) - return False - - except Exception as e: - logger.error(f'Unexpected error in TLS relay: {e}') - return False - - def log_email(self, message_id, peer, mail_from, rcpt_tos, content, status, dkim_signed=False): - """Log email activity to database.""" + return results + + def relay_email(self, *args, **kwargs): + """Synchronous wrapper for relay_email_async for compatibility.""" + return asyncio.run(self.relay_email_async(*args, **kwargs)) + + def log_email(self, message_id, peer, mail_from, to_address, cc_addresses, bcc_addresses, subject, email_headers, message_body, status, dkim_signed=False, username=None, recipient_results=None): + """Log email activity to database, including per-recipient results.""" session_db = Session() try: - # Convert content to string if it's bytes - if isinstance(content, bytes): - content_str = content.decode('utf-8', errors='replace') + # Determine status: relayed, partial, failed + delivered = [r for r in (recipient_results or []) if r['status'] == 'success'] + failed = [r for r in (recipient_results or []) if r['status'] != 'success'] + if delivered and failed: + overall_status = 'partial' + elif delivered: + overall_status = 'relayed' else: - content_str = content + overall_status = 'failed' email_log = EmailLog( message_id=message_id, - timestamp=datetime.now(), - peer=str(peer), + timestamp=get_current_time(), + peer_ip=peer, mail_from=mail_from, - rcpt_tos=', '.join(rcpt_tos), - content=content_str, - status=status, - dkim_signed=dkim_signed + to_address=to_address or '', + cc_addresses=cc_addresses or '', + bcc_addresses=bcc_addresses or '', + subject=subject, + email_headers=email_headers, + message_body=message_body, + status=overall_status, + dkim_signed=dkim_signed, + username=username ) session_db.add(email_log) + session_db.flush() + + # Log per-recipient results + if recipient_results: + for r in recipient_results: + recipient_log = EmailRecipientLog( + email_log_id=email_log.id, + recipient=r['recipient'], + recipient_type=r.get('recipient_type', 'to'), + status=r['status'], + error_code=r.get('error_code'), + error_message=r.get('error_message'), + server_response=r.get('server_response') + ) + session_db.add(recipient_log) session_db.commit() logger.debug(f'Logged email: {message_id}') except Exception as e: diff --git a/email_server/models.py b/email_server/models.py index a2c4fed..bd2415d 100644 --- a/email_server/models.py +++ b/email_server/models.py @@ -38,7 +38,7 @@ class Domain(Base): created_at = Column(DateTime, default=func.now()) # Add relationships with proper foreign key references - users = relationship("User", backref="domain", lazy="joined") + senders = relationship("Sender", backref="domain", lazy="joined") dkim_keys = relationship("DKIMKey", backref="domain", lazy="joined") whitelisted_ips = relationship("WhitelistedIP", backref="domain", lazy="joined") custom_headers = relationship("CustomHeader", backref="domain", lazy="joined") @@ -46,15 +46,15 @@ class Domain(Base): def __repr__(self): return f"" -class User(Base): +class Sender(Base): """ - User model with enhanced authentication controls. + Sender model with enhanced authentication controls. Security features: - - can_send_as_domain: If True, user can send as any email from their domain - - If False, user can only send as their own email address + - can_send_as_domain: If True, sender can send as any email from their domain + - If False, sender can only send as their own email address """ - __tablename__ = 'esrv_users' + __tablename__ = 'esrv_senders' id = Column(Integer, primary_key=True) email = Column(String, unique=True, nullable=False) @@ -63,31 +63,29 @@ class User(Base): can_send_as_domain = Column(Boolean, default=False) is_active = Column(Boolean, default=True) created_at = Column(DateTime, default=func.now()) + store_message_content = Column(Boolean, default=False) # Store message body/attachments def can_send_as(self, from_address: str) -> bool: """ - Check if this user can send emails as the given from_address. + Check if this sender can send emails as the given from_address. Args: - from_address: The email address the user wants to send from - + from_address: The email address the sender wants to send from Returns: - True if user is allowed to send as this address + True if sender is allowed to send as this address """ - # User can always send as their own email + # Sender can always send as their own email if from_address.lower() == self.email.lower(): return True - - # If user has domain privileges, check if from_address is from same domain + # If sender has domain privileges, check if from_address is from same domain if self.can_send_as_domain: - user_domain = self.email.split('@')[1].lower() + sender_domain = self.email.split('@')[1].lower() from_domain = from_address.split('@')[1].lower() if '@' in from_address else '' - return user_domain == from_domain - + return sender_domain == from_domain return False def __repr__(self): - return f"" + return f"" class WhitelistedIP(Base): """ @@ -103,6 +101,7 @@ class WhitelistedIP(Base): domain_id = Column(Integer, ForeignKey('esrv_domains.id'), nullable=False) is_active = Column(Boolean, default=True) created_at = Column(DateTime, default=func.now()) + store_message_content = Column(Boolean, default=False) # Store message body/attachments def can_send_for_domain(self, domain_name: str) -> bool: """ @@ -136,26 +135,44 @@ class EmailLog(Base): __tablename__ = 'esrv_email_logs' id = Column(Integer, primary_key=True) - - # Legacy columns (from original schema) message_id = Column(String, unique=True, nullable=False) timestamp = Column(DateTime, nullable=False) - peer = Column(String, nullable=False) + peer_ip = Column(String, nullable=False) # Store only IP address mail_from = Column(String, nullable=False) - rcpt_tos = Column(String, nullable=False) - content = Column(Text, nullable=False) + to_address = Column(String, nullable=False, server_default='') + cc_addresses = Column(String, nullable=True, server_default='') # Comma-separated CC + bcc_addresses = Column(String, nullable=True, server_default='') # Comma-separated BCC + subject = Column(Text, nullable=True) + email_headers = Column(Text, nullable=False) # Store only email headers + message_body = Column(Text, nullable=True) # Store actual message content status = Column(String, nullable=False) dkim_signed = Column(Boolean, default=False) - - # New columns (added later) - from_address = Column(String, nullable=False, server_default='unknown') - to_address = Column(String, nullable=False, server_default='unknown') - subject = Column(Text, nullable=True) - message = Column(Text, nullable=True) + username = Column(String, nullable=True) # Authenticated username created_at = Column(DateTime, default=func.now()) - + + recipients = relationship("EmailRecipientLog", back_populates="email_log", cascade="all, delete-orphan") + attachments = relationship("EmailAttachment", back_populates="email_log", cascade="all, delete-orphan") + def __repr__(self): - return f"" + return f"" + +class EmailRecipientLog(Base): + """Log for each recipient of an email, including status and error details.""" + __tablename__ = 'esrv_email_recipient_logs' + + id = Column(Integer, primary_key=True) + email_log_id = Column(Integer, ForeignKey('esrv_email_logs.id'), nullable=False) + recipient = Column(String, nullable=False) + recipient_type = Column(String, nullable=False) # 'to', 'cc', 'bcc' + status = Column(String, nullable=False) # 'success', 'failed', etc. + error_code = Column(String, nullable=True) + error_message = Column(Text, nullable=True) + server_response = Column(Text, nullable=True) + + email_log = relationship("EmailLog", back_populates="recipients") + + def __repr__(self): + return f"" class AuthLog(Base): """Authentication log model for security auditing.""" @@ -202,6 +219,23 @@ class CustomHeader(Base): def __repr__(self): return f"" +class EmailAttachment(Base): + """Attachment metadata and file path, linked to EmailLog.""" + __tablename__ = 'esrv_email_attachments' + + id = Column(Integer, primary_key=True) + email_log_id = Column(Integer, ForeignKey('esrv_email_logs.id'), nullable=False) + filename = Column(String, nullable=False) + content_type = Column(String, nullable=True) + file_path = Column(String, nullable=False) # Path on disk + size = Column(Integer, nullable=True) + uploaded_at = Column(DateTime, default=func.now()) + + email_log = relationship("EmailLog", back_populates="attachments") + + def __repr__(self): + return f"" + def create_tables(): """Create all database tables using ESRV schema.""" @@ -276,11 +310,11 @@ def log_email(from_address: str, to_address: str, subject: str, finally: session.close() -def get_user_by_email(email: str): - """Get user by email address.""" +def get_sender_by_email(email: str): + """Get sender by email address.""" session = Session() try: - return session.query(User).filter_by(email=email.lower(), is_active=True).first() + return session.query(Sender).filter_by(email=email.lower(), is_active=True).first() finally: session.close() diff --git a/email_server/server_runner.py b/email_server/server_runner.py index e5093cf..d2a065c 100644 --- a/email_server/server_runner.py +++ b/email_server/server_runner.py @@ -46,7 +46,7 @@ async def start_server(shutdown_event=None): # dkim_manager.initialize_default_keys() # Removed: do not auto-generate DKIM keys for all domains # Add test data if needed - from .models import Session, Domain, User, WhitelistedIP, hash_password + from .models import Session, Domain, Sender, WhitelistedIP, hash_password session = Session() try: # Add example.com domain if not exists @@ -57,17 +57,17 @@ async def start_server(shutdown_event=None): session.commit() logger.debug("Added example.com domain") - # Add test user if not exists - user = session.query(User).filter_by(email='test@example.com').first() - if not user: - user = User( + # Add test sender if not exists + sender = session.query(Sender).filter_by(email='test@example.com').first() + if not sender: + sender = Sender( email='test@example.com', password_hash=hash_password('testpass123'), domain_id=domain.id ) - session.add(user) + session.add(sender) session.commit() - logger.debug("Added test user: test@example.com") + logger.debug("Added test sender: test@example.com") # Add whitelisted IP if not exists whitelist = session.query(WhitelistedIP).filter_by(ip_address='127.0.0.1').first() @@ -117,7 +117,7 @@ async def start_server(shutdown_event=None): ) controller_tls.start() logger.debug(f' - Plain SMTP (IP whitelist): {BIND_IP}:{SMTP_PORT}') - logger.debug(f' - STARTTLS SMTP (auth required): {BIND_IP}:{SMTP_TLS_PORT}') + logger.debug(f' - Direct TLS SMTP (SMTPS, auth required): {BIND_IP}:{SMTP_TLS_PORT}') logger.debug('Management available via web interface at: http://localhost:5000/email') try: diff --git a/email_server/server_web_ui/__init__.py b/email_server/server_web_ui/__init__.py index 55e06f1..3dece42 100644 --- a/email_server/server_web_ui/__init__.py +++ b/email_server/server_web_ui/__init__.py @@ -15,3 +15,4 @@ from .ip_whitelist import * from .dkim import * from .settings import * from .logs import * +from .view_message import * diff --git a/email_server/server_web_ui/dashboard.py b/email_server/server_web_ui/dashboard.py index ad7e1e6..d6f0569 100644 --- a/email_server/server_web_ui/dashboard.py +++ b/email_server/server_web_ui/dashboard.py @@ -5,7 +5,7 @@ This module provides the main dashboard view and overview functionality. """ from flask import render_template -from email_server.models import Session, Domain, User, DKIMKey, EmailLog, AuthLog +from email_server.models import Session, Domain, Sender, DKIMKey, EmailLog, AuthLog, EmailRecipientLog from email_server.tool_box import get_logger from .routes import email_bp @@ -19,20 +19,23 @@ def dashboard(): try: # Get counts domain_count = session.query(Domain).filter_by(is_active=True).count() - user_count = session.query(User).filter_by(is_active=True).count() + sender_count = session.query(Sender).filter_by(is_active=True).count() dkim_count = session.query(DKIMKey).filter_by(is_active=True).count() # Get recent email logs recent_emails = session.query(EmailLog).order_by(EmailLog.created_at.desc()).limit(10).all() + # Get recipient logs for each recent email + recipient_logs_map = {email.id: session.query(EmailRecipientLog).filter_by(email_log_id=email.id).all() for email in recent_emails} # Get recent auth logs recent_auths = session.query(AuthLog).order_by(AuthLog.created_at.desc()).limit(10).all() return render_template('dashboard.html', domain_count=domain_count, - user_count=user_count, + sender_count=sender_count, dkim_count=dkim_count, recent_emails=recent_emails, - recent_auths=recent_auths) + recent_auths=recent_auths, + recipient_logs_map=recipient_logs_map) finally: session.close() \ No newline at end of file diff --git a/email_server/server_web_ui/dkim.py b/email_server/server_web_ui/dkim.py index 6f7e16b..468757f 100644 --- a/email_server/server_web_ui/dkim.py +++ b/email_server/server_web_ui/dkim.py @@ -9,12 +9,12 @@ This module provides DKIM key management functionality including: - DKIM DNS verification """ -from flask import render_template, request, redirect, url_for, flash, jsonify +from flask import render_template, request, redirect, url_for, flash, jsonify, current_app from datetime import datetime import re from email_server.models import Session, Domain, DKIMKey from email_server.dkim_manager import DKIMManager -from email_server.tool_box import get_logger +from email_server.tool_box import get_logger, get_current_time from .utils import get_public_ip, check_dns_record, generate_spf_record from .routes import email_bp @@ -107,7 +107,7 @@ def create_dkim(): active_keys = session.query(DKIMKey).filter_by(domain_id=domain.id, is_active=True).all() for key in active_keys: key.is_active = False - key.replaced_at = datetime.now() + key.replaced_at = get_current_time() # Create new DKIM key dkim_manager = DKIMManager() created = dkim_manager.generate_dkim_keypair(domain_name, selector=selector, force_new_key=True) @@ -146,7 +146,7 @@ def regenerate_dkim(domain_id: int): # Mark existing keys as replaced for key in existing_keys: key.is_active = False - key.replaced_at = datetime.now() # Mark when this key was replaced + key.replaced_at = get_current_time() # Mark when this key was replaced # Generate new DKIM key preserving the existing selector dkim_manager = DKIMManager() @@ -331,7 +331,7 @@ def toggle_dkim(dkim_id: int): ).all() for key in other_active_keys: key.is_active = False - key.replaced_at = datetime.now() + key.replaced_at = get_current_time() dkim_key.is_active = not old_status if dkim_key.is_active: @@ -432,11 +432,22 @@ def check_spf_dns(): spf_record = record break + spf_valid_for_server = False + spf_check_message = '' + public_ip = get_public_ip() + ip_mechanism = f'ip4:{public_ip}' if spf_record: result['spf_record'] = spf_record + if ip_mechanism in spf_record: + spf_valid_for_server = True + spf_check_message = f'SPF is valid for this server (contains {ip_mechanism})' + else: + spf_check_message = f'SPF is missing this server\'s IP ({ip_mechanism})' result['message'] = 'SPF record found' else: result['success'] = False result['message'] = 'No SPF record found' - + result['spf_valid_for_server'] = spf_valid_for_server + result['spf_check_message'] = spf_check_message + result['public_ip'] = public_ip return jsonify(result) \ No newline at end of file diff --git a/email_server/server_web_ui/domains.py b/email_server/server_web_ui/domains.py index e5af0f8..ee2eca4 100644 --- a/email_server/server_web_ui/domains.py +++ b/email_server/server_web_ui/domains.py @@ -10,7 +10,7 @@ This module provides domain management functionality including: """ from flask import render_template, request, redirect, url_for, flash -from email_server.models import Session, Domain, User, WhitelistedIP, DKIMKey, CustomHeader +from email_server.models import Session, Domain, Sender, WhitelistedIP, DKIMKey, CustomHeader from email_server.dkim_manager import DKIMManager from email_server.tool_box import get_logger from sqlalchemy.orm import joinedload @@ -188,12 +188,12 @@ def remove_domain(domain_id: int): domain_name = domain.domain_name # Count associated records - user_count = session.query(User).filter_by(domain_id=domain_id).count() + sender_count = session.query(Sender).filter_by(domain_id=domain_id).count() ip_count = session.query(WhitelistedIP).filter_by(domain_id=domain_id).count() dkim_count = session.query(DKIMKey).filter_by(domain_id=domain_id).count() # Delete associated records - session.query(User).filter_by(domain_id=domain_id).delete() + session.query(Sender).filter_by(domain_id=domain_id).delete() session.query(WhitelistedIP).filter_by(domain_id=domain_id).delete() session.query(DKIMKey).filter_by(domain_id=domain_id).delete() session.query(CustomHeader).filter_by(domain_id=domain_id).delete() @@ -202,7 +202,7 @@ def remove_domain(domain_id: int): session.delete(domain) session.commit() - flash(f'Domain {domain_name} and all associated data permanently removed ({user_count} users, {ip_count} IPs, {dkim_count} DKIM keys)', 'success') + flash(f'Domain {domain_name} and all associated data permanently removed ({sender_count} senders, {ip_count} IPs, {dkim_count} DKIM keys)', 'success') return redirect(url_for('email.domains_list')) except Exception as e: diff --git a/email_server/server_web_ui/ip_whitelist.py b/email_server/server_web_ui/ip_whitelist.py index a9ed0ae..c4372d6 100644 --- a/email_server/server_web_ui/ip_whitelist.py +++ b/email_server/server_web_ui/ip_whitelist.py @@ -37,6 +37,7 @@ def add_ip(): if request.method == 'POST': ip_address = request.form.get('ip_address', '').strip() domain_id = request.form.get('domain_id', type=int) + store_message_content = bool(request.form.get('store_message_content')) if not all([ip_address, domain_id]): flash('All fields are required', 'error') @@ -58,7 +59,8 @@ def add_ip(): # Create whitelisted IP whitelist = WhitelistedIP( ip_address=ip_address, - domain_id=domain_id + domain_id=domain_id, + store_message_content=store_message_content ) session.add(whitelist) session.commit() @@ -166,6 +168,7 @@ def edit_ip(ip_id: int): if request.method == 'POST': ip_address = request.form.get('ip_address', '').strip() domain_id = request.form.get('domain_id', type=int) + store_message_content = bool(request.form.get('store_message_content')) if not all([ip_address, domain_id]): flash('All fields are required', 'error') @@ -191,6 +194,7 @@ def edit_ip(ip_id: int): # Update IP record ip_record.ip_address = ip_address ip_record.domain_id = domain_id + ip_record.store_message_content = store_message_content session.commit() flash(f'IP whitelist record updated', 'success') diff --git a/email_server/server_web_ui/logs.py b/email_server/server_web_ui/logs.py index ba0dc48..753c7c3 100644 --- a/email_server/server_web_ui/logs.py +++ b/email_server/server_web_ui/logs.py @@ -4,12 +4,11 @@ Logs blueprint for the SMTP server web UI. This module provides email and authentication log viewing functionality. """ -from flask import render_template, request, jsonify -from email_server.models import Session, EmailLog, AuthLog, Domain +from flask import render_template, request, send_file, redirect, url_for, flash, Response +from email_server.models import Session, EmailLog, AuthLog, EmailRecipientLog, EmailAttachment from email_server.tool_box import get_logger -from sqlalchemy import desc -from datetime import datetime, timedelta from .routes import email_bp +import os logger = get_logger() @@ -40,10 +39,15 @@ def logs(): # Convert to unified format combined_logs = [] for log in email_logs: + # Fetch recipient logs and attachments for each email log + recipient_logs = session.query(EmailRecipientLog).filter_by(email_log_id=log.id).all() + attachments = session.query(EmailAttachment).filter_by(email_log_id=log.id).all() combined_logs.append({ 'type': 'email', 'timestamp': log.created_at, - 'data': log + 'data': log, + 'recipients': recipient_logs, + 'attachments': attachments }) for log in auth_logs: combined_logs.append({ @@ -69,12 +73,20 @@ def logs(): has_next = offset + per_page < total has_prev = page > 1 - + # Fetch recipient logs and attachments for each email log if emails + recipient_logs_map = {} + attachments_map = {} + if filter_type == 'emails': + for log in logs: + recipient_logs_map[log.id] = session.query(EmailRecipientLog).filter_by(email_log_id=log.id).all() + attachments_map[log.id] = session.query(EmailAttachment).filter_by(email_log_id=log.id).all() return render_template('logs.html', logs=logs, filter_type=filter_type, page=page, has_next=has_next, - has_prev=has_prev) + has_prev=has_prev, + recipient_logs_map=recipient_logs_map, + attachments_map=attachments_map) finally: - session.close() \ No newline at end of file + session.close() diff --git a/email_server/server_web_ui/routes.py b/email_server/server_web_ui/routes.py index be00dc7..f24ce4f 100644 --- a/email_server/server_web_ui/routes.py +++ b/email_server/server_web_ui/routes.py @@ -1,9 +1,12 @@ """ Main routes and blueprint definition for the SMTP server web UI. """ -from flask import Blueprint, render_template -from email_server.tool_box import get_logger +from flask import Blueprint, render_template, request, jsonify, current_app +from email_server.models import Session, EmailLog, AuthLog +from email_server.tool_box import get_logger, get_current_time +from email_server.settings_loader import load_settings from datetime import datetime +import pytz # Create the main email blueprint @@ -15,14 +18,35 @@ email_bp = Blueprint('email', __name__, logger = get_logger() +# Get timezone from settings +settings = load_settings() +timezone = pytz.timezone(settings['Server'].get('time_zone', 'UTC')) + +@email_bp.app_template_filter('format_datetime') +def format_datetime(value, timezone=None): + """Format datetime with the correct timezone from settings or argument.""" + if value is None: + return '' + import pytz + if timezone is None: + settings = load_settings() + timezone = settings['Server'].get('time_zone', 'UTC') + tz = pytz.timezone(timezone) + if value.tzinfo is None: + value = pytz.UTC.localize(value) + local_dt = value.astimezone(tz) + return local_dt.strftime('%Y-%m-%d %H:%M:%S') + +from .view_message import * # Import view_message routes + # Error handlers @email_bp.errorhandler(404) def not_found(error): """Handle 404 errors.""" return render_template('error.html', error_code=404, - error_message='Page not found', - current_time=datetime.now()), 404 + error_message="Page not found", + current_time=get_current_time()), 404 @email_bp.errorhandler(500) def internal_error(error): @@ -30,5 +54,5 @@ def internal_error(error): logger.error(f"Internal error: {error}") return render_template('error.html', error_code=500, - error_message='Internal server error', - current_time=datetime.now()), 500 \ No newline at end of file + error_message=str(error), + current_time=get_current_time()), 500 \ No newline at end of file diff --git a/email_server/server_web_ui/senders.py b/email_server/server_web_ui/senders.py index 434ed13..8eedb5d 100644 --- a/email_server/server_web_ui/senders.py +++ b/email_server/server_web_ui/senders.py @@ -10,27 +10,27 @@ This module provides sender management functionality including: """ from flask import render_template, request, redirect, url_for, flash -from email_server.models import Session, Domain, User +from email_server.models import Session, Domain, Sender from email_server.tool_box import get_logger import bcrypt from .routes import email_bp -from email_server.models import Session, Domain, User, hash_password +from email_server.models import Session, Domain, Sender, hash_password logger = get_logger() @email_bp.route('/senders') def senders_list(): - """List all users.""" + """List all senders.""" session = Session() try: - users = session.query(User, Domain).join(Domain, User.domain_id == Domain.id).order_by(User.email).all() - return render_template('senders.html', users=users) + senders = session.query(Sender, Domain).join(Domain, Sender.domain_id == Domain.id).order_by(Sender.email).all() + return render_template('senders.html', senders=senders) finally: session.close() @email_bp.route('/senders/add', methods=['GET', 'POST']) def add_sender(): - """Add new user.""" + """Add new sender.""" session = Session() try: domains = session.query(Domain).filter_by(is_active=True).order_by(Domain.domain_name).all() @@ -40,6 +40,7 @@ def add_sender(): password = request.form.get('password', '').strip() domain_id = request.form.get('domain_id', type=int) can_send_as_domain = request.form.get('can_send_as_domain') == 'on' + store_message_content = request.form.get('store_message_content') == 'on' if not all([email, password, domain_id]): flash('All fields are required', 'error') @@ -50,118 +51,119 @@ def add_sender(): flash('Invalid email format', 'error') return redirect(url_for('email.add_sender')) - # Check if user already exists - existing = session.query(User).filter_by(email=email).first() + # Check if sender already exists + existing = session.query(Sender).filter_by(email=email).first() if existing: - flash(f'User {email} already exists', 'error') + flash(f'Sender {email} already exists', 'error') return redirect(url_for('email.senders_list')) - # Create user - user = User( + # Create sender + sender = Sender( email=email, password_hash=hash_password(password), domain_id=domain_id, - can_send_as_domain=can_send_as_domain + can_send_as_domain=can_send_as_domain, + store_message_content=store_message_content ) - session.add(user) + session.add(sender) session.commit() - flash(f'User {email} added successfully', 'success') + flash(f'Sender {email} added successfully', 'success') return redirect(url_for('email.senders_list')) return render_template('add_sender.html', domains=domains) except Exception as e: session.rollback() - logger.error(f"Error adding user: {e}") - flash(f'Error adding user: {str(e)}', 'error') + logger.error(f"Error adding sender: {e}") + flash(f'Error adding sender: {str(e)}', 'error') return redirect(url_for('email.add_sender')) finally: session.close() @email_bp.route('/senders//delete', methods=['POST']) def delete_sender(user_id: int): - """Disable user (soft delete).""" + """Disable sender (soft delete).""" session = Session() try: - user = session.query(User).get(user_id) - if not user: - flash('User not found', 'error') + sender = session.query(Sender).get(user_id) + if not sender: + flash('Sender not found', 'error') return redirect(url_for('email.senders_list')) - user_email = user.email - user.is_active = False + sender_email = sender.email + sender.is_active = False session.commit() - flash(f'User {user_email} disabled', 'success') + flash(f'Sender {sender_email} disabled', 'success') return redirect(url_for('email.senders_list')) except Exception as e: session.rollback() - logger.error(f"Error disabling user: {e}") - flash(f'Error disabling user: {str(e)}', 'error') + logger.error(f"Error disabling sender: {e}") + flash(f'Error disabling sender: {str(e)}', 'error') return redirect(url_for('email.senders_list')) finally: session.close() @email_bp.route('/senders//enable', methods=['POST']) def enable_sender(user_id: int): - """Enable user.""" + """Enable sender.""" session = Session() try: - user = session.query(User).get(user_id) - if not user: - flash('User not found', 'error') + sender = session.query(Sender).get(user_id) + if not sender: + flash('Sender not found', 'error') return redirect(url_for('email.senders_list')) - user_email = user.email - user.is_active = True + sender_email = sender.email + sender.is_active = True session.commit() - flash(f'User {user_email} enabled', 'success') + flash(f'Sender {sender_email} enabled', 'success') return redirect(url_for('email.senders_list')) except Exception as e: session.rollback() - logger.error(f"Error enabling user: {e}") - flash(f'Error enabling user: {str(e)}', 'error') + logger.error(f"Error enabling sender: {e}") + flash(f'Error enabling sender: {str(e)}', 'error') return redirect(url_for('email.senders_list')) finally: session.close() @email_bp.route('/senders//remove', methods=['POST']) def remove_sender(user_id: int): - """Permanently remove user.""" + """Permanently remove sender.""" session = Session() try: - user = session.query(User).get(user_id) - if not user: - flash('User not found', 'error') + sender = session.query(Sender).get(user_id) + if not sender: + flash('Sender not found', 'error') return redirect(url_for('email.senders_list')) - user_email = user.email - session.delete(user) + sender_email = sender.email + session.delete(sender) session.commit() - flash(f'User {user_email} permanently removed', 'success') + flash(f'Sender {sender_email} permanently removed', 'success') return redirect(url_for('email.senders_list')) except Exception as e: session.rollback() - logger.error(f"Error removing user: {e}") - flash(f'Error removing user: {str(e)}', 'error') + logger.error(f"Error removing sender: {e}") + flash(f'Error removing sender: {str(e)}', 'error') return redirect(url_for('email.senders_list')) finally: session.close() @email_bp.route('/senders//edit', methods=['GET', 'POST']) def edit_sender(user_id: int): - """Edit user.""" + """Edit sender.""" session = Session() try: - user = session.query(User).get(user_id) - if not user: - flash('User not found', 'error') + sender = session.query(Sender).get(user_id) + if not sender: + flash('Sender not found', 'error') return redirect(url_for('email.senders_list')) domains = session.query(Domain).filter_by(is_active=True).order_by(Domain.domain_name).all() @@ -171,6 +173,7 @@ def edit_sender(user_id: int): password = request.form.get('password', '').strip() domain_id = request.form.get('domain_id', type=int) can_send_as_domain = request.form.get('can_send_as_domain') == 'on' + store_message_content = request.form.get('store_message_content') == 'on' if not all([email, domain_id]): flash('Email and domain are required', 'error') @@ -181,35 +184,36 @@ def edit_sender(user_id: int): flash('Invalid email format', 'error') return redirect(url_for('email.edit_sender', user_id=user_id)) - # Check if email already exists (excluding current user) - existing = session.query(User).filter( - User.email == email, - User.id != user_id + # Check if email already exists (excluding current sender) + existing = session.query(Sender).filter( + Sender.email == email, + Sender.id != user_id ).first() if existing: flash(f'Email {email} already exists', 'error') return redirect(url_for('email.edit_sender', user_id=user_id)) - # Update user - user.email = email - user.domain_id = domain_id - user.can_send_as_domain = can_send_as_domain + # Update sender + sender.email = email + sender.domain_id = domain_id + sender.can_send_as_domain = can_send_as_domain + sender.store_message_content = store_message_content # Update password if provided if password: - user.password_hash = hash_password(password) + sender.password_hash = hash_password(password) session.commit() - flash(f'User {email} updated successfully', 'success') + flash(f'Sender {email} updated successfully', 'success') return redirect(url_for('email.senders_list')) - return render_template('edit_sender.html', user=user, domains=domains) + return render_template('edit_sender.html', sender=sender, domains=domains) except Exception as e: session.rollback() - logger.error(f"Error editing user: {e}") - flash(f'Error editing user: {str(e)}', 'error') + logger.error(f"Error editing sender: {e}") + flash(f'Error editing sender: {str(e)}', 'error') return redirect(url_for('email.senders_list')) finally: session.close() \ No newline at end of file diff --git a/email_server/server_web_ui/settings.py b/email_server/server_web_ui/settings.py index f593de7..bce6bf7 100644 --- a/email_server/server_web_ui/settings.py +++ b/email_server/server_web_ui/settings.py @@ -12,6 +12,7 @@ This module provides server settings management functionality including: import os import time from pathlib import Path +import zoneinfo from flask import render_template, request, redirect, url_for, flash, jsonify from werkzeug.utils import secure_filename from email_server.settings_loader import load_settings, SETTINGS_PATH @@ -29,11 +30,21 @@ ALLOWED_EXTENSIONS = {'crt', 'key', 'pem'} def allowed_file(filename): return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS +def get_template_context(): + """Get template context with CSRF token and common data.""" + context = { + 'settings': load_settings(), + 'timezones': get_available_timezones(), + } + # Only add CSRF token if it exists and is enabled + if hasattr(request, 'csrf_token'): + context['csrf_token_value'] = request.csrf_token + return context + @email_bp.route('/settings') def settings(): """Display and edit server settings.""" - settings = load_settings() - return render_template('settings.html', settings=settings) + return render_template('settings.html', **get_template_context()) @email_bp.route('/settings_update', methods=['POST']) def settings_update(): @@ -195,3 +206,40 @@ def get_server_ip(): except Exception as e: logger.error(f"Error getting public IP: {e}") return jsonify({'status': 'error', 'message': str(e)}) + +@email_bp.route('/test_attachments_path', methods=['POST']) +def test_attachments_path(): + """Test if the attachments path is writable.""" + path = request.form.get('path') + if not path: + return jsonify({'success': False, 'message': 'No path provided'}) + + # Convert to absolute path if relative + if not os.path.isabs(path): + path = os.path.abspath(os.path.join(os.path.dirname(SETTINGS_PATH), path)) + + try: + # Create path if it doesn't exist + os.makedirs(path, exist_ok=True) + + # Try to create a test file + test_file = os.path.join(path, '.write_test') + with open(test_file, 'w') as f: + f.write('test') + os.remove(test_file) + return jsonify({ + 'success': True, + 'message': 'Attachments path is valid and writable', + 'absolute_path': path + }) + except Exception as e: + logger.error(f"Error testing attachments path: {e}") + return jsonify({ + 'success': False, + 'message': f'Error: {str(e)}', + 'absolute_path': path + }) + +def get_available_timezones(): + """Get a list of all available timezones sorted alphabetically.""" + return sorted(zoneinfo.available_timezones()) diff --git a/email_server/server_web_ui/templates/add_ip.html b/email_server/server_web_ui/templates/add_ip.html index 97d26b0..5a5e8ce 100644 --- a/email_server/server_web_ui/templates/add_ip.html +++ b/email_server/server_web_ui/templates/add_ip.html @@ -67,6 +67,18 @@ +
+
+ + +
+ If enabled, the full message body and attachments will be stored and viewable in logs. Otherwise, only headers and subject are stored. +
+
+
+
diff --git a/email_server/server_web_ui/templates/add_sender.html b/email_server/server_web_ui/templates/add_sender.html index 134dfed..7a72fda 100644 --- a/email_server/server_web_ui/templates/add_sender.html +++ b/email_server/server_web_ui/templates/add_sender.html @@ -70,6 +70,18 @@
+
+
+ + +
+ If enabled, the full message body and attachments will be stored and viewable in logs. Otherwise, only headers and subject are stored. +
+
+
+
diff --git a/email_server/server_web_ui/templates/base.html b/email_server/server_web_ui/templates/base.html index 60e385b..838f942 100644 --- a/email_server/server_web_ui/templates/base.html +++ b/email_server/server_web_ui/templates/base.html @@ -131,6 +131,20 @@ + + {% block extra_css %}{% endblock %} @@ -337,6 +351,16 @@ + + + {% block extra_js %}{% endblock %} diff --git a/email_server/server_web_ui/templates/dashboard.html b/email_server/server_web_ui/templates/dashboard.html index ae0d07d..760f4f5 100644 --- a/email_server/server_web_ui/templates/dashboard.html +++ b/email_server/server_web_ui/templates/dashboard.html @@ -7,6 +7,7 @@
+
@@ -24,19 +25,21 @@
+
+
- Users + Senders
-

{{ user_count }}

- Authenticated users +

{{ sender_count }}

+ Authenticated senders
@@ -44,9 +47,11 @@
+
+
@@ -64,6 +69,7 @@
+
@@ -75,11 +81,28 @@ Status
-
+ {% set health = check_health() %} +
Service Status:
" + + 'SMTP Server: ' + health.services.smtp_server|title + '
' + + 'Web Frontend: ' + health.services.web_frontend|title + '
' + + 'Database: ' + health.services.database|title + '
' + ) | safe }}"> - Online + {{ health.status|title }} - Server running + + {% if health.services.smtp_server == 'running' and health.services.database == 'ok' %} + All services running + {% else %} + {% if health.services.smtp_server == 'stopped' %}SMTP Server stopped{% endif %} + {% if health.services.database == 'error' %}Database error{% endif %} + {% endif %} +
@@ -111,7 +134,7 @@ Time From - To + Recipients Status DKIM @@ -121,7 +144,7 @@ - {{ email.created_at.strftime('%H:%M:%S') }} + {{ email.created_at|format_datetime }} @@ -130,22 +153,74 @@ - - {{ email.to_address }} - +
+
+ {% if email.to_address %} + {% for rcpt in email.to_address.split(',') %} + {% if rcpt.strip() %} +
+ To: + {{ rcpt.strip() }} +
+ {% endif %} + {% endfor %} + {% endif %} + + {% if email.cc_addresses %} + {% for rcpt in email.cc_addresses.split(',') %} + {% if rcpt.strip() %} +
+ CC: + {{ rcpt.strip() }} +
+ {% endif %} + {% endfor %} + {% endif %} + + {% if email.bcc_addresses %} + {% for rcpt in email.bcc_addresses.split(',') %} + {% if rcpt.strip() %} +
+ BCC: + {{ rcpt.strip() }} +
+ {% endif %} + {% endfor %} + {% endif %} + + {% if not email.to_address and not email.cc_addresses and not email.bcc_addresses %} +
No recipients
+ {% endif %} +
+
- {% if email.status == 'relayed' %} - - - Sent - + {% set delivered = recipient_logs_map[email.id]|selectattr('status', 'equalto', 'success')|list %} + {% set failed = recipient_logs_map[email.id]|selectattr('status', 'ne', 'success')|list %} + {% if delivered and failed %} + {% set overall_status = 'partial' %} + {% elif delivered %} + {% set overall_status = 'relayed' %} {% else %} - - - Failed - + {% set overall_status = 'failed' %} {% endif %} + {% if overall_status == 'relayed' %} + + + Sent + + {% elif overall_status == 'partial' %} + + + Partial Fail + + {% else %} + + + Failed + + {% endif %} + {% if email.dkim_signed %} @@ -204,7 +279,7 @@
- {{ auth.created_at.strftime('%H:%M:%S') }} + {{ auth.created_at|format_datetime }}
@@ -248,7 +323,7 @@
- Add User + Add Sender
@@ -282,4 +357,40 @@ location.reload(); }, 30000); + + + + {% endblock %} diff --git a/email_server/server_web_ui/templates/dkim.html b/email_server/server_web_ui/templates/dkim.html index 267f983..1fc5f26 100644 --- a/email_server/server_web_ui/templates/dkim.html +++ b/email_server/server_web_ui/templates/dkim.html @@ -331,6 +331,30 @@ {% block extra_js %} {% endblock %} diff --git a/email_server/server_web_ui/templates/sidebar_email.html b/email_server/server_web_ui/templates/sidebar_email.html index a249df6..c1a6032 100644 --- a/email_server/server_web_ui/templates/sidebar_email.html +++ b/email_server/server_web_ui/templates/sidebar_email.html @@ -44,7 +44,7 @@ class="nav-link text-white {{ 'active' if request.endpoint in ['email.senders_list', 'email.add_user'] else '' }}"> Allowed Senders - {{ user_count if user_count is defined else '' }} + {{ sender_count if sender_count is defined else '' }} @@ -98,7 +98,7 @@ Server Settings - + {#
  • @@ -114,6 +114,7 @@ Logs & Activity
    • + #} @@ -122,12 +123,22 @@
    Server Status - + {% set health = check_health() %} + Service Status:
    " + + 'SMTP Server: ' + health.services.smtp_server|title + '
    ' + + 'Web Frontend: ' + health.services.web_frontend|title + '
    ' + + 'Database: ' + health.services.database|title + '
    ' + ) | safe }}"> - Online + {{ health.status|title }}
    - @@ -170,6 +181,10 @@ font-size: 0.7rem; } +.status-indicator { + cursor: pointer; +} + /* Responsive design */ @media (max-width: 768px) { .sidebar { @@ -186,3 +201,16 @@ } } + + diff --git a/email_server/server_web_ui/templates/view_message_content.html b/email_server/server_web_ui/templates/view_message_content.html new file mode 100644 index 0000000..438476b --- /dev/null +++ b/email_server/server_web_ui/templates/view_message_content.html @@ -0,0 +1,98 @@ +{% extends "base.html" %} + +{% block title %}View Full Message - Email Log{% endblock %} + +{% block content %} +
    +

    Full Message Content

    +
    + From: {{ log.mail_from }}
    + To: {{ log.to_address }}
    + CC: {{ log.cc_addresses or 'None' }}
    + BCC: {{ log.bcc_addresses or 'None' }}
    + Subject: {{ log.subject or 'N/A' }}
    + Date: {{ log.created_at.strftime('%Y-%m-%d %H:%M:%S') }}
    +
    + + {% if log.attachments %} +
    +
    + Attachments: +
    +
    + +
    +
    + {% endif %} + +
    +
    + Message Content: +
    +
    + 
    {{ log.message_body }}
    +
    +
    + +
    +
    + Message Headers: +
    +
    + 
    {{ log.email_headers }}
    +
    +
    + + Back to Logs +
    +{% endblock %} diff --git a/email_server/server_web_ui/utils.py b/email_server/server_web_ui/utils.py index 9e74392..119bcb3 100644 --- a/email_server/server_web_ui/utils.py +++ b/email_server/server_web_ui/utils.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def get_public_ip() -> str: """Get the public IP address of the server.""" try: - response1 = requests.get('https://ifconfig.me/ip', timeout=3, verify=False) + response1 = requests.get('http://ifconfig.me/ip', timeout=3, verify=False) ip = response1.text.strip() if ip and ip != 'unknown': @@ -24,7 +24,7 @@ def get_public_ip() -> str: except Exception: try: # Fallback method - response = requests.get('https://httpbin.org/ip', timeout=3, verify=False) + response = requests.get('http://httpbin.org/ip', timeout=3, verify=False) ip = response.json()['origin'].split(',')[0].strip() if ip and ip != 'unknown': return ip diff --git a/email_server/server_web_ui/view_message.py b/email_server/server_web_ui/view_message.py new file mode 100644 index 0000000..16c3095 --- /dev/null +++ b/email_server/server_web_ui/view_message.py @@ -0,0 +1,131 @@ +""" +Route to view full email message content if stored. +""" +from flask import render_template, abort, flash, redirect, Response, send_file, request, url_for +from email_server.models import Session, EmailLog, EmailAttachment +from email_server.tool_box import get_logger +from .routes import email_bp +import os + +logger = get_logger() + +@email_bp.route('/msg/content/') +def view_message_content(log_id): + """View the full message content for an email log if stored.""" + session = Session() + try: + # Get log with attachments + log = session.query(EmailLog).filter_by(id=log_id).first() + if not log: + abort(404) + + # Get attachments for this log + attachments = session.query(EmailAttachment).filter_by(email_log_id=log_id).all() + log.attachments = attachments + + return render_template('view_message_content.html', log=log) + finally: + session.close() +@email_bp.route('/msg/attachment//download') +def download_attachment(attachment_id): + session = Session() + try: + attachment = session.query(EmailAttachment).get(attachment_id) + if not attachment or not os.path.isfile(attachment.file_path): + flash('Attachment not found.', 'danger') + return redirect(url_for('email.logs', type='emails')) + + # Get the normalized content type and handle special cases + content_type = attachment.content_type.lower() if attachment.content_type else 'application/octet-stream' + extension = os.path.splitext(attachment.filename.lower())[1][1:] if '.' in attachment.filename else '' + + # Force download if requested + as_attachment = request.args.get('download', '').lower() == 'true' + + # Map of extensions to content types for common files + content_type_map = { + 'txt': 'text/plain', + 'csv': 'text/csv', + 'pdf': 'application/pdf', + 'jpg': 'image/jpeg', + 'jpeg': 'image/jpeg', + 'png': 'image/png', + 'gif': 'image/gif', + 'svg': 'image/svg+xml', + 'html': 'text/html', + 'htm': 'text/html', + 'json': 'application/json', + 'xml': 'text/xml', + 'md': 'text/markdown', + } + + # Update content type based on file extension if needed + if content_type == 'application/octet-stream' and extension in content_type_map: + content_type = content_type_map[extension] + + # Special handling for CSV files + if content_type == 'text/csv' and not as_attachment: + try: + with open(attachment.file_path, 'r') as f: + csv_content = f.read() + # Create a simple HTML table view for CSV + html_content = '' + + # Convert CSV to HTML table + for i, line in enumerate(csv_content.split('\n')): + if not line.strip(): + continue + html_content += '' + if i == 0: # Header row + html_content += ''.join(f'' for cell in line.split(',')) + else: + html_content += ''.join(f'' for cell in line.split(',')) + html_content += '' + + html_content += '
    {cell}{cell}
    ' + return Response(html_content, mimetype='text/html') + except Exception as e: + logger.warning(f"Failed to create CSV preview: {e}") + # Fall back to normal file handling + + # Determine if the file should be viewed in browser + if as_attachment: + # Force download + return send_file( + attachment.file_path, + as_attachment=True, + download_name=attachment.filename + ) + else: + # Try to display in browser + return send_file( + attachment.file_path, + mimetype=content_type + ) + finally: + session.close() + + +@email_bp.route('/msg/attachment//delete', methods=['POST', 'GET']) +def delete_attachment(attachment_id): + session = Session() + try: + attachment = session.query(EmailAttachment).get(attachment_id) + if not attachment: + flash('Attachment not found.', 'danger') + return redirect(url_for('email.logs', type='emails')) + # Remove file from disk + if os.path.isfile(attachment.file_path): + os.remove(attachment.file_path) + # Remove from DB + session.delete(attachment) + session.commit() + flash('Attachment deleted.', 'success') + return redirect(url_for('email.logs', type='emails')) + finally: + session.close() \ No newline at end of file diff --git a/email_server/settings_loader.py b/email_server/settings_loader.py index 8e84ae5..02d2984 100644 --- a/email_server/settings_loader.py +++ b/email_server/settings_loader.py @@ -14,8 +14,8 @@ DEFAULTS = { '; Server configuration for SMTP ports and hostname': None, '; Plain SMTP port for internal/whitelisted IPs': None, 'SMTP_PORT': '4025', - '; STARTTLS SMTP port for authenticated users': None, - 'SMTP_TLS_PORT': '40587', + '; TLS SMTP port for authenticated users': None, + 'SMTP_TLS_PORT': '40465', '; Server hostname for HELO/EHLO identification': None, 'HOSTNAME': 'mail.example.com', '; Override HELO hostname': None, @@ -24,6 +24,8 @@ DEFAULTS = { 'BIND_IP': '0.0.0.0', '; Custom server banner (to make it empty use "" must be double quotes)': None, 'server_banner': "", + '; Time zone for the server': None, + 'TIME_ZONE': 'Europe/London', }, 'Database': { '; Database configuration': None, @@ -38,7 +40,7 @@ DEFAULTS = { }, 'Relay': { '; Timeout in seconds for external SMTP connections': None, - 'RELAY_TIMEOUT': '10', + 'RELAY_TIMEOUT': '30', }, 'TLS': { '; TLS/SSL certificate configuration': None, diff --git a/email_server/smtp_handler.py b/email_server/smtp_handler.py index db100a1..9af22fb 100644 --- a/email_server/smtp_handler.py +++ b/email_server/smtp_handler.py @@ -8,33 +8,59 @@ Security Features: - Enhanced header management """ -import uuid -from datetime import datetime +import email.utils +import os +import mimetypes from aiosmtpd.smtp import SMTP as AIOSMTP, AuthResult from aiosmtpd.controller import Controller -from email_server.auth import EnhancedAuthenticator, EnhancedIPAuthenticator, validate_sender_authorization, get_authenticated_domain_id +from email_server.auth import EnhancedAuthenticator, EnhancedIPAuthenticator, validate_sender_authorization from email_server.email_relay import EmailRelay from email_server.dkim_manager import DKIMManager from email_server.settings_loader import load_settings -from email_server.tool_box import get_logger +from email_server.tool_box import get_logger, ensure_folder_exists, generate_message_id, get_current_time +from email import policy +from email.parser import BytesParser +from email_server.models import Session, EmailAttachment, EmailLog logger = get_logger() +settings = load_settings() + +helo_hostname = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost')) class CustomSMTP(AIOSMTP): - """Custom SMTP class with configurable banner.""" + """Custom SMTP class with configurable banner and secure AUTH handling.""" def __init__(self, *args, **kwargs): # Sets Custom SMTP banner from settings - settings = load_settings() _banner_message = settings['Server'].get('server_banner', '') if _banner_message == '""': _banner_message = '' self.custom_banner = _banner_message - + + # Store authenticator and auth_require_tls for later use + self._custom_authenticator = kwargs.get('authenticator', None) + self._custom_auth_require_tls = kwargs.get('auth_require_tls', False) super().__init__(*args, **kwargs) # Override the __ident__ to use our custom banner self.__ident__ = self.custom_banner + def _get_auth_methods(self): + # Only advertise AUTH if authenticator is set and (not auth_require_tls or connection is secure) + if self._custom_authenticator and (not self._custom_auth_require_tls or self.session and self.session.ssl): + return super()._get_auth_methods() + return [] + + async def smtp_AUTH(self, arg): + """ + Override AUTH command to close connection after failed authentication. + """ + result = await super().smtp_AUTH(arg) + # If authentication failed, close the connection immediately + if isinstance(result, AuthResult) and not result.success: + if hasattr(self, 'session') and hasattr(self.session, 'transport') and self.session.transport: + self.session.transport.close() + return result + class EnhancedCombinedAuthenticator: """ Enhanced combined authenticator with sender validation support. @@ -77,61 +103,45 @@ class EnhancedCustomSMTPHandler: self.auth_methods = ['LOGIN', 'PLAIN'] def _ensure_required_headers(self, content: str, envelope, message_id: str, custom_headers: list = None) -> str: - """Ensure all required email headers are present and properly formatted. - - Following RFC 5322 header order and best practices for spam score reduction. - Optimized based on Gmail's header structure for better deliverability. - - Args: - content (str): Email content. - envelope: SMTP envelope. - message_id (str): Generated message ID. - custom_headers (list): List of (name, value) tuples for custom headers. - - Returns: - str: Email content with all required headers properly formatted. - """ - import email.utils - from email_server.settings_loader import load_settings - + """Ensure all required email headers are present and properly formatted.""" try: - settings = load_settings() - fallback_hostname = settings.get('Server', 'HOSTNAME', fallback='localhost') - server_hostname = settings.get('Server', 'helo_hostname', fallback=fallback_hostname) - - logger.debug(f"Processing headers for message {message_id}") - - # Parse the message properly - if isinstance(content, bytes): - content = content.decode('utf-8', errors='replace') - - # Split content into lines and normalize line endings - lines = content.replace('\r\n', '\n').replace('\r', '\n').split('\n') - + lines = content.splitlines() + for idx, line in enumerate(lines): + if not isinstance(line, str): + logger.error(f"_ensure_required_headers: Non-string line at index {idx}: {type(line)}: {line}") + logger.error(f"_ensure_required_headers: Full content object: {repr(content)}") + raise TypeError(f"_ensure_required_headers: Non-string line in content.splitlines(): {type(line)} at index {idx}") # Find header/body boundary and collect existing headers body_start = 0 existing_headers = {} original_header_order = [] - for i, line in enumerate(lines): if line.strip() == '': body_start = i + 1 break + if not isinstance(line, str): + logger.error(f"_ensure_required_headers: Header line is not a string: {type(line)}: {line}") + continue if ':' in line and not line.startswith((' ', '\t')): - header_name, header_value = line.split(':', 1) + try: + header_name, header_value = line.split(':', 1) + except Exception as e: + logger.error(f"_ensure_required_headers: Failed to split header line: {line} - {e}") + continue + if not isinstance(header_name, str) or not isinstance(header_value, str): + logger.error(f"_ensure_required_headers: Non-string header_name or header_value: {type(header_name)}, {type(header_value)}: {header_name}, {header_value}") + continue header_name_lower = header_name.strip().lower() header_value = header_value.strip() - # Handle continuation lines j = i + 1 while j < len(lines) and lines[j].startswith((' ', '\t')): header_value += ' ' + lines[j].strip() j += 1 - existing_headers[header_name_lower] = header_value original_header_order.append((header_name.strip(), header_value)) logger.debug(f"Found existing header: {header_name_lower} = {header_value}") - + # Extract body and clean it body_lines = lines[body_start:] if body_start < len(lines) else [] while body_lines and body_lines[-1].strip() == '': @@ -143,10 +153,26 @@ class EnhancedCustomSMTPHandler: # 1. Message-ID (critical for spam filters) if 'message-id' in existing_headers: - required_headers.append(f"Message-ID: {existing_headers['message-id']}") + # Parse existing Message-ID + existing_msg_id = existing_headers['message-id'].strip('<>') + if '@' in existing_msg_id: + prefix, hostname = existing_msg_id.rsplit('@', 1) + hostname = hostname.rstrip('>') + if hostname.lower() != helo_hostname.lower(): + # If hostname is wrong, modify it to use our hostname + message_id = f"{prefix}@{helo_hostname}" + else: + # If hostname is correct, keep original ID + message_id = existing_msg_id + else: + # Malformed Message-ID, generate new one + message_id = generate_message_id() else: - domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else server_hostname.replace('mail.', '') - required_headers.append(f"Message-ID: <{message_id}@{domain}>") + # No Message-ID found, generate new one + message_id = generate_message_id() + + # Add the Message-ID header with the final ID + required_headers.append(f"Message-ID: <{message_id}>") # 2. Date (critical for spam filters) if 'date' in existing_headers: @@ -161,75 +187,49 @@ class EnhancedCustomSMTPHandler: else: required_headers.append("MIME-Version: 1.0") - # 4. User-Agent (if present, helps with reputation) - if 'user-agent' in existing_headers: - required_headers.append(f"User-Agent: {existing_headers['user-agent']}") - - # 5. Content-Language (if present) - if 'content-language' in existing_headers: - required_headers.append(f"Content-Language: {existing_headers['content-language']}") - - # 6. To (primary recipients - critical) + # 4. To (primary recipients - critical) if 'to' in existing_headers: required_headers.append(f"To: {existing_headers['to']}") else: - to_list = ', '.join(envelope.rcpt_tos) - required_headers.append(f"To: {to_list}") + required_headers.append(f"To: {', '.join([rcpt for rcpt in envelope.rcpt_tos])}") + + # 5. Cc (if present) + if 'cc' in existing_headers: + required_headers.append(f"Cc: {existing_headers['cc']}") - # 7. From (sender identification - critical) + # 6. From (sender identification - critical) if 'from' in existing_headers: required_headers.append(f"From: {existing_headers['from']}") else: required_headers.append(f"From: {envelope.mail_from}") - # 8. Subject (message topic - critical) + # 7. Subject (message topic - critical) if 'subject' in existing_headers: required_headers.append(f"Subject: {existing_headers['subject']}") else: required_headers.append("Subject: ") - # 9. Content-Type (media type information) + # 8. Content-Type (media type information) if 'content-type' in existing_headers: required_headers.append(f"Content-Type: {existing_headers['content-type']}") else: required_headers.append("Content-Type: text/plain; charset=UTF-8; format=flowed") - # 10. Content-Transfer-Encoding + # 9. Content-Transfer-Encoding if 'content-transfer-encoding' in existing_headers: required_headers.append(f"Content-Transfer-Encoding: {existing_headers['content-transfer-encoding']}") else: required_headers.append("Content-Transfer-Encoding: 7bit") - - # Add custom headers after essential headers but before misc headers + + # Add custom headers after essential headers if custom_headers: for header_name, header_value in custom_headers: - # Skip if already added in essential headers - if header_name.lower() not in ['message-id', 'date', 'mime-version', 'user-agent', - 'content-language', 'to', 'from', 'subject', - 'content-type', 'content-transfer-encoding']: + header_name_lower = header_name.lower() + # Skip if header already exists + if header_name_lower not in existing_headers: required_headers.append(f"{header_name}: {header_value}") logger.debug(f"Added custom header: {header_name}: {header_value}") - - # Add any other existing headers that weren't handled above - essential_headers = { - 'message-id', 'date', 'from', 'to', 'subject', - 'mime-version', 'content-type', 'content-transfer-encoding', - 'user-agent', 'content-language' - } - - # Preserve original header names and values for non-essential headers - for header_name, header_value in original_header_order: - if header_name.lower() not in essential_headers: - # Skip custom headers we already added - skip = False - if custom_headers: - for custom_name, _ in custom_headers: - if header_name.lower() == custom_name.lower(): - skip = True - break - if not skip: - required_headers.append(f"{header_name}: {header_value}") - + # Build final message final_content = '\r\n'.join(required_headers) if body.strip(): @@ -237,91 +237,347 @@ class EnhancedCustomSMTPHandler: else: final_content += '\r\n\r\n' - logger.debug(f"Final headers for message {message_id}:") - for header in required_headers: - logger.debug(f" {header}") - return final_content except Exception as e: - logger.error(f"Error ensuring headers: {e}") import traceback + logger.error(f"Error ensuring headers: {e}") logger.error(f"Traceback: {traceback.format_exc()}") + logger.error(f"Locals: {locals()}") # Fallback to original content if parsing fails return content async def handle_DATA(self, server, session, envelope): - """Handle incoming email data with improved header management.""" + """Handle incoming email data with improved header management and logging.""" try: - message_id = str(uuid.uuid4()) - logger.debug(f'Received email {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}') - # Convert content to string if it's bytes if isinstance(envelope.content, bytes): content = envelope.content.decode('utf-8', errors='replace') else: content = envelope.content + + # Extract Message-ID from the content + for line in content.splitlines(): + if line.lower().startswith('message-id:'): + message_id_extracted = line[11:].strip().strip('<>') # Remove "Message-ID:" and brackets + if '@' in message_id_extracted: + prefix, hostname = message_id_extracted.rsplit('@', 1) + hostname = hostname.rstrip('>') + if hostname.lower() != helo_hostname.lower(): + # If hostname is wrong, modify it to use our hostname + message_id = f"{prefix}@{helo_hostname}" + else: + # If hostname is correct, keep original ID + message_id = message_id_extracted + break + + logger.debug(f'Processing email with ID: {message_id} from {envelope.mail_from} to {envelope.rcpt_tos}') + + # Get authenticated username from session + username = getattr(session, 'username', None) + if not username: + # Check if IP authentication was used + client_ip = getattr(session, 'peer', ['unknown'])[0].split(':')[0] if hasattr(session, 'peer') else None + if client_ip: + from email_server.models import get_whitelisted_ip + sender_domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else None + ip_auth = get_whitelisted_ip(client_ip, sender_domain) + if ip_auth: + username = f"IP:{client_ip}" + logger.debug(f'Authenticated username: {username}') + + # Convert content to string if it's bytes + if isinstance(envelope.content, bytes): + content = envelope.content.decode('utf-8', errors='replace') + raw_bytes = envelope.content + else: + content = envelope.content + raw_bytes = envelope.content.encode('utf-8', errors='replace') + # Extract domain from sender for DKIM signing sender_domain = envelope.mail_from.split('@')[1] if '@' in envelope.mail_from else None - + # Get custom headers before processing custom_headers = [] if sender_domain: custom_headers = self.dkim_manager.get_active_custom_headers(sender_domain) - + # Add beneficial headers for spam score improvement client_ip = getattr(session, 'peer', ['unknown'])[0] if hasattr(session, 'peer') else None if client_ip: - # Add X-Originating-IP header (helps with reputation) custom_headers.append(('X-Originating-IP', f'[{client_ip}]')) - - # Add X-Mailer header for identification custom_headers.append(('X-Mailer', 'NetBro Mail Server 1.0')) - - # Add X-Priority header (normal priority) custom_headers.append(('X-Priority', '3')) - + # Ensure required headers are present (including custom headers) content = self._ensure_required_headers(content, envelope, message_id, custom_headers) - + # DKIM-sign the final version of the message (only once, after all modifications) signed_content = content dkim_signed = False if sender_domain: signed_content = self.dkim_manager.sign_email(content, sender_domain) + if not isinstance(signed_content, (str, bytes)): + logger.error(f"DKIMManager.sign_email returned non-str/bytes: {type(signed_content)}: {signed_content}") + raise TypeError(f"DKIMManager.sign_email returned non-str/bytes: {type(signed_content)}") dkim_signed = signed_content != content if dkim_signed: logger.debug(f'Email {message_id} signed with DKIM for domain {sender_domain}') + + # Extract headers for logging + to_address = '' + cc_addresses = '' + bcc_addresses = '' + subject = '' + split_lines = content.splitlines() + for idx, line in enumerate(split_lines): + if not isinstance(line, str): + logger.error(f"DIAGNOSTIC: Non-string line at index {idx}: {type(line)}: {line}") + logger.error(f"DIAGNOSTIC: Full content object: {repr(content)}") + raise TypeError(f"DIAGNOSTIC: Non-string line in content.splitlines(): {type(line)} at index {idx}") + try: + + for line in split_lines: + if line.strip() == '': + break + if not isinstance(line, str): + logger.error(f"Header line is not a string: {type(line)}: {line}") + continue + try: + lower_line = line.lower() + except Exception as e: + logger.error(f"Failed to call lower() on line: {line} (type: {type(line)}) - {e}") + import traceback + logger.error(traceback.format_exc()) + logger.error(f"Full content.splitlines(): {split_lines}") + continue + if lower_line.startswith('to:'): + to_address = line[3:].strip() + elif lower_line.startswith('cc:'): + cc_addresses = line[3:].strip() + elif lower_line.startswith('subject:'): + subject = line[8:].strip() + except Exception as e: + logger.error(f"Exception in header extraction loop: {e}") + import traceback + logger.error(traceback.format_exc()) + logger.error(f"Full content.splitlines(): {split_lines}") + + # Check if message content should be stored (sender or IP whitelist) + from email_server.models import get_sender_by_email, get_whitelisted_ip + store_message = False + sender_obj = get_sender_by_email(envelope.mail_from) + if sender_obj and getattr(sender_obj, 'store_message_content', False): + store_message = True + elif client_ip: + domain_name = sender_domain + ip_obj = get_whitelisted_ip(client_ip, domain_name) + if ip_obj and getattr(ip_obj, 'store_message_content', False): + store_message = True + + attachments_to_save = [] + # Get attachments path from settings + attachments_path = settings['Attachments'].get('attachments_path', 'email_server/server_data/attachments') + saved_attachments = [] + logger.debug(f"Using attachments base path: {attachments_path}") + email_log_id = None + + if store_message: + # Parse the message for attachments using the email library + msg = BytesParser(policy=policy.default).parsebytes(raw_bytes) + if msg.is_multipart(): + # Get storage path for this sender + storage_path = self.get_attachment_storage_path( + attachments_base_path=attachments_path, + sender_domain=sender_domain, + username=username, + client_ip=client_ip + ) + ensure_folder_exists(storage_path) + + for part in msg.walk(): + content_disposition = part.get_content_disposition() + if content_disposition == 'attachment': + filename = part.get_filename() + if not filename: + continue + + # Get file data and validate + file_data = part.get_payload(decode=True) + if not file_data: + continue + + # Get proper content type + content_type = self.get_content_type(part, filename) + size = len(file_data) + + # Strip @domain from message_id for filename + clean_message_id = message_id.split('@')[0] if '@' in message_id else message_id + + # Build a unique file path + safe_filename = f"{clean_message_id}_{filename}" + file_path = os.path.join(storage_path, safe_filename) + + try: + # Ensure the directory exists before saving + ensure_folder_exists(file_path) + + # Save the file + with open(file_path, 'wb') as f: + f.write(file_data) + logger.debug(f"Saved attachment {filename} ({content_type}) to {file_path}") + + attachments_to_save.append({ + 'filename': filename, + 'content_type': content_type, + 'file_path': file_path, + 'size': size + }) + except Exception as e: + logger.error(f"Failed to save attachment {filename}: {str(e)}") + continue + + # Parse addresses to determine recipient types + def parse_addresses(addr_str): + if not isinstance(addr_str, str): + logger.warning(f"Expected string for address header, got {type(addr_str)}: {addr_str}") + return [] + + return [addr.strip().lower() for addr in addr_str.split(',') if isinstance(addr, str) and addr.strip()] - # Relay the email (no further modifications allowed) - success = self.email_relay.relay_email( - envelope.mail_from, - envelope.rcpt_tos, - signed_content + to_list = parse_addresses(to_address) + cc_list = parse_addresses(cc_addresses) + + # Map recipients to their types based on headers + recipient_type_map = {} + for rcpt in envelope.rcpt_tos: + if not isinstance(rcpt, str): + logger.warning(f"Expected string for recipient, got {type(rcpt)}: {rcpt}") + continue + rcpt_l = rcpt.lower() + if rcpt_l in to_list: + recipient_type_map[rcpt] = 'to' + elif rcpt_l in cc_list: + recipient_type_map[rcpt] = 'cc' + else: + recipient_type_map[rcpt] = 'bcc' # Any recipient not in To/Cc is a Bcc + + # Build recipient results + recipient_results = [] + recipient_types = [] + for rcpt in envelope.rcpt_tos: + rtype = recipient_type_map[rcpt] + recipient_results.append({'recipient': rcpt, 'recipient_type': rtype, 'status': 'pending'}) + recipient_types.append(rtype) + + # Relay the email and get per-recipient results + relay_results = await self.email_relay.relay_email_async( + envelope.mail_from, + envelope.rcpt_tos, + signed_content, + username=username, + cc_addresses=cc_addresses, + bcc_addresses=None, # BCC addresses are handled through envelope.rcpt_tos + recipient_types=recipient_types ) + + # Update status in recipient_results + for result in relay_results: + for r in recipient_results: + if r['recipient'] == result['recipient'] and r['recipient_type'] == result.get('recipient_type', 'to'): + r.update(result) + break + + # Determine overall status + status = 'relayed' if all(r['status'] == 'success' for r in recipient_results) else 'failed' + + # Extract headers and parse message content + msg = BytesParser(policy=policy.default).parsebytes(raw_bytes) - # Log the email - status = 'relayed' if success else 'failed' + # Extract headers + email_headers = [] + for name, value in msg.items(): + email_headers.append(f"{name}: {value}") + email_headers = '\n'.join(email_headers) + + # Extract only the text content, not attachments + message_body = "" + if msg.is_multipart(): + for part in msg.walk(): + if part.get_content_maintype() == 'text' and part.get_content_disposition() is None: + # This is likely the main message text + charset = part.get_content_charset() or 'utf-8' + try: + part_content = part.get_payload(decode=True).decode(charset) + message_body += part_content + "\n" + except Exception as e: + logger.warning(f"Failed to decode message part: {e}") + else: + # Not multipart - if it's text, use it as is + if msg.get_content_maintype() == 'text': + charset = msg.get_content_charset() or 'utf-8' + try: + message_body = msg.get_payload(decode=True).decode(charset) + except Exception as e: + logger.warning(f"Failed to decode message: {e}") + + # Trim any extra whitespace + message_body = message_body.strip() + + # Get client IP without port + client_ip = getattr(session, 'peer', ['unknown'])[0].split(':')[0] if hasattr(session, 'peer') else 'unknown' + + # Log the email with all details self.email_relay.log_email( message_id=message_id, - peer=session.peer, + peer=client_ip, mail_from=envelope.mail_from, - rcpt_tos=envelope.rcpt_tos, - content=content, # Log original content, not signed + to_address=to_address, + cc_addresses=cc_addresses, + bcc_addresses=', '.join([r['recipient'] for r in recipient_results if r['recipient_type'] == 'bcc']), + subject=subject, + email_headers=email_headers, + message_body=message_body, status=status, - dkim_signed=dkim_signed + dkim_signed=dkim_signed, + username=username, + recipient_results=recipient_results ) - - if success: + + # Save attachments to DB, linked to the correct EmailLog + if attachments_to_save: + db_session = Session() + try: + email_log = db_session.query(EmailLog).filter_by(message_id=message_id).first() + if email_log: + for att in attachments_to_save: + attachment = EmailAttachment( + email_log_id=email_log.id, + filename=att['filename'], + content_type=att['content_type'], + file_path=att['file_path'], + size=att['size'] + ) + db_session.add(attachment) + db_session.commit() + except Exception as e: + logger.error(f"Failed to save attachments to DB: {e}") + db_session.rollback() + finally: + db_session.close() + + if status == 'relayed': logger.debug(f'Email {message_id} successfully relayed') return '250 Message accepted for delivery' else: logger.error(f'Email {message_id} failed to relay') return '550 Message relay failed' - except Exception as e: + import traceback logger.error(f'Error handling email: {e}') + logger.error(f'Traceback: {traceback.format_exc()}') + logger.error(f'Locals: {locals()}') return '550 Internal server error' async def handle_RCPT(self, server, session, envelope, address, rcpt_options): @@ -352,10 +608,80 @@ class EnhancedCustomSMTPHandler: logger.info(f'MAIL FROM accepted: {address} - {message}') return '250 OK' + def get_attachment_storage_path(self, attachments_base_path: str, sender_domain: str, username: str = None, client_ip: str = None) -> str: + """Generate the storage path for attachments based on sender domain, authentication, and date. + + Args: + attachments_base_path: Base path for attachments storage + sender_domain: Domain of the sender + username: Authenticated username (if any) + client_ip: Client IP address (if IP-based authentication) + + Returns: + str: Full path where attachments should be stored, format: + base/domain/[username|ip]/YYYY-DD-MMM/ + """ + + # Get current date in YYYY-DD-MMM format using consistent time function + current_date = get_current_time().strftime('%Y-%d-%b') # e.g., 2025-14-Jun + + # Sanitize domain name for folder name + safe_domain = sender_domain.replace('/', '_').replace('\\', '_') + domain_path = os.path.join(attachments_base_path, safe_domain) + + # Determine auth-based subfolder path + if username: + # Sanitize username for folder name + safe_username = username.replace('/', '_').replace('\\', '_') + auth_path = os.path.join(domain_path, safe_username) + elif client_ip: + # Sanitize IP for folder name + safe_ip = client_ip.replace(':', '_') + auth_path = os.path.join(domain_path, safe_ip) + else: + # Fallback to domain-only path + auth_path = domain_path + + # Add date-based subfolder + return os.path.join(auth_path, current_date) + + def get_content_type(self, part, filename): + """Get the correct content type for a file, trying multiple methods.""" + + # First try the part's content type + content_type = part.get_content_type() + + # If it's octet-stream, try to guess from filename + if content_type == 'application/octet-stream': + guessed_type, _ = mimetypes.guess_type(filename) + if guessed_type: + content_type = guessed_type + else: + # Use specific types for common extensions + ext = filename.lower().split('.')[-1] if '.' in filename else '' + type_map = { + 'txt': 'text/plain', + 'csv': 'text/csv', + 'jpg': 'image/jpeg', + 'jpeg': 'image/jpeg', + 'png': 'image/png', + 'gif': 'image/gif', + 'pdf': 'application/pdf', + 'json': 'application/json', + 'xml': 'application/xml', + 'html': 'text/html', + 'htm': 'text/html', + } + content_type = type_map.get(ext, 'application/octet-stream') + + return content_type + class TLSController(Controller): - """Custom controller with TLS support - modeled after the working original.""" + """ + Custom controller for direct TLS (SMTPS, port 465) support. + """ - def __init__(self, handler, ssl_context, hostname='localhost', port=40587): + def __init__(self, handler, ssl_context, hostname='localhost', port=40465): logger.debug(f"TLSController __init__: ssl_context={ssl_context is not None}") self._ssl_context = ssl_context # Use private attribute to avoid conflicts self.smtp_hostname = hostname # Store for HELO identification @@ -365,10 +691,11 @@ class TLSController(Controller): logger.debug(f"TLSController factory: ssl_context={self._ssl_context is not None}") logger.debug(f"TLSController factory: ssl_context object={self._ssl_context}") logger.debug(f"TLSController factory: hostname={self.smtp_hostname}") + # This is direct TLS (SMTPS, port 465 style) smtp_instance = CustomSMTP( self.handler, tls_context=self._ssl_context, - require_starttls=False, # Don't require STARTTLS immediately, but make it available + require_starttls=False, # Direct TLS: do not advertise or require STARTTLS auth_require_tls=True, # If auth is used, require TLS authenticator=self.handler.combined_authenticator, decode_data=True, @@ -378,17 +705,18 @@ class TLSController(Controller): return smtp_instance class PlainController(Controller): - """Controller for plain SMTP with username/password and IP-based authentication.""" + """Controller for plain SMTP with authentication and IP whitelist fallback.""" def __init__(self, handler, hostname='localhost', port=4025): self.smtp_hostname = hostname # Store for HELO identification super().__init__(handler, hostname='0.0.0.0', port=port) # Bind to all interfaces def factory(self): + # Pass authenticator and set auth_require_tls=False to enable AUTH on plain port return CustomSMTP( self.handler, authenticator=self.handler.combined_authenticator, - auth_require_tls=False, # Allow AUTH over plain text (not recommended for production) + auth_require_tls=False, # Allow AUTH on plain port decode_data=True, hostname=self.smtp_hostname # Use proper hostname for HELO ) diff --git a/email_server/tool_box.py b/email_server/tool_box.py index 97efd1e..0a2fa3f 100644 --- a/email_server/tool_box.py +++ b/email_server/tool_box.py @@ -5,8 +5,13 @@ Utility functions for the email server. import os import logging from email_server.settings_loader import load_settings +from datetime import datetime +import pytz +import time +import random settings = load_settings() +helo_hostname = settings['Server'].get('helo_hostname', settings['Server'].get('hostname', 'localhost')) def ensure_folder_exists(filepath): """ @@ -56,4 +61,19 @@ def get_logger(name=None): name = name if ext == '.py' else base else: name = '__main__' - return logging.getLogger(name) \ No newline at end of file + return logging.getLogger(name) + +def get_current_time(): + """Get current time with timezone from settings.""" + timezone = pytz.timezone(settings['Server'].get('time_zone', 'UTC')) + return datetime.now(timezone) + +def generate_message_id(hostname=helo_hostname) -> str: + """Generate a consistent Message-ID for both email headers and database storage. + + Returns: + str: Message-ID in format YYYYMMDDhhmmss.RANDOM@hostname without brackets + """ + timestamp = time.strftime('%Y%m%d%H%M%S') + random_id = ''.join([str(random.randint(0, 9)) for _ in range(6)]) + return f"{timestamp}.{random_id}@{hostname}" \ No newline at end of file diff --git a/migrations/README b/migrations/README deleted file mode 100644 index 0e04844..0000000 --- a/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Single-database configuration for Flask. diff --git a/migrations/add_email_attachments_table.sql b/migrations/add_email_attachments_table.sql new file mode 100644 index 0000000..65be149 --- /dev/null +++ b/migrations/add_email_attachments_table.sql @@ -0,0 +1,11 @@ +-- Migration: Add EmailAttachment table for storing email attachments on disk +CREATE TABLE IF NOT EXISTS esrv_email_attachments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email_log_id INTEGER NOT NULL, + filename TEXT NOT NULL, + content_type TEXT, + file_path TEXT NOT NULL, + size INTEGER, + uploaded_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(email_log_id) REFERENCES esrv_email_logs(id) ON DELETE CASCADE +); diff --git a/migrations/alembic.ini b/migrations/alembic.ini deleted file mode 100644 index ec9d45c..0000000 --- a/migrations/alembic.ini +++ /dev/null @@ -1,50 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic,flask_migrate - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[logger_flask_migrate] -level = INFO -handlers = -qualname = flask_migrate - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py deleted file mode 100644 index 4c97092..0000000 --- a/migrations/env.py +++ /dev/null @@ -1,113 +0,0 @@ -import logging -from logging.config import fileConfig - -from flask import current_app - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') - - -def get_engine(): - try: - # this works with Flask-SQLAlchemy<3 and Alchemical - return current_app.extensions['migrate'].db.get_engine() - except (TypeError, AttributeError): - # this works with Flask-SQLAlchemy>=3 - return current_app.extensions['migrate'].db.engine - - -def get_engine_url(): - try: - return get_engine().url.render_as_string(hide_password=False).replace( - '%', '%%') - except AttributeError: - return str(get_engine().url).replace('%', '%%') - - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -config.set_main_option('sqlalchemy.url', get_engine_url()) -target_db = current_app.extensions['migrate'].db - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def get_metadata(): - if hasattr(target_db, 'metadatas'): - return target_db.metadatas[None] - return target_db.metadata - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=get_metadata(), literal_binds=True - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - - # this callback is used to prevent an auto-migration from being generated - # when there are no changes to the schema - # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html - def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): - script = directives[0] - if script.upgrade_ops.is_empty(): - directives[:] = [] - logger.info('No changes in schema detected.') - - conf_args = current_app.extensions['migrate'].configure_args - if conf_args.get("process_revision_directives") is None: - conf_args["process_revision_directives"] = process_revision_directives - - connectable = get_engine() - - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=get_metadata(), - **conf_args - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako deleted file mode 100644 index 2c01563..0000000 --- a/migrations/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/3ce273a1be20_initial_migration.py b/migrations/versions/3ce273a1be20_initial_migration.py deleted file mode 100644 index 6ef2a27..0000000 --- a/migrations/versions/3ce273a1be20_initial_migration.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Initial migration - -Revision ID: 3ce273a1be20 -Revises: -Create Date: 2025-06-07 15:25:35.603295 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '3ce273a1be20' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('esrv_dkim_keys') - op.drop_table('esrv_auth_logs') - op.drop_table('esrv_users') - op.drop_table('esrv_domains') - op.drop_table('esrv_whitelisted_ips') - op.drop_table('esrv_email_logs') - op.drop_table('esrv_custom_headers') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('esrv_custom_headers', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('domain_id', sa.INTEGER(), nullable=False), - sa.Column('header_name', sa.VARCHAR(), nullable=False), - sa.Column('header_value', sa.VARCHAR(), nullable=False), - sa.Column('is_active', sa.BOOLEAN(), nullable=True), - sa.Column('created_at', sa.DATETIME(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('esrv_email_logs', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('message_id', sa.VARCHAR(), nullable=False), - sa.Column('timestamp', sa.DATETIME(), nullable=False), - sa.Column('peer', sa.VARCHAR(), nullable=False), - sa.Column('mail_from', sa.VARCHAR(), nullable=False), - sa.Column('rcpt_tos', sa.VARCHAR(), nullable=False), - sa.Column('content', sa.TEXT(), nullable=False), - sa.Column('status', sa.VARCHAR(), nullable=False), - sa.Column('dkim_signed', sa.BOOLEAN(), nullable=True), - sa.Column('from_address', sa.VARCHAR(), server_default=sa.text("'unknown'"), nullable=False), - sa.Column('to_address', sa.VARCHAR(), server_default=sa.text("'unknown'"), nullable=False), - sa.Column('subject', sa.TEXT(), nullable=True), - sa.Column('message', sa.TEXT(), nullable=True), - sa.Column('created_at', sa.DATETIME(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('message_id') - ) - op.create_table('esrv_whitelisted_ips', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('ip_address', sa.VARCHAR(), nullable=False), - sa.Column('domain_id', sa.INTEGER(), nullable=False), - sa.Column('is_active', sa.BOOLEAN(), nullable=True), - sa.Column('created_at', sa.DATETIME(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('esrv_domains', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('domain_name', sa.VARCHAR(), nullable=False), - sa.Column('is_active', sa.BOOLEAN(), nullable=True), - sa.Column('created_at', sa.DATETIME(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('domain_name') - ) - op.create_table('esrv_users', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('email', sa.VARCHAR(), nullable=False), - sa.Column('password_hash', sa.VARCHAR(), nullable=False), - sa.Column('domain_id', sa.INTEGER(), nullable=False), - sa.Column('can_send_as_domain', sa.BOOLEAN(), nullable=True), - sa.Column('is_active', sa.BOOLEAN(), nullable=True), - sa.Column('created_at', sa.DATETIME(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('email') - ) - op.create_table('esrv_auth_logs', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('auth_type', sa.VARCHAR(), nullable=False), - sa.Column('identifier', sa.VARCHAR(), nullable=False), - sa.Column('ip_address', sa.VARCHAR(), nullable=True), - sa.Column('success', sa.BOOLEAN(), nullable=False), - sa.Column('message', sa.TEXT(), nullable=True), - sa.Column('created_at', sa.DATETIME(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('esrv_dkim_keys', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('domain_id', sa.INTEGER(), nullable=False), - sa.Column('selector', sa.VARCHAR(), nullable=False), - sa.Column('private_key', sa.TEXT(), nullable=False), - sa.Column('public_key', sa.TEXT(), nullable=False), - sa.Column('is_active', sa.BOOLEAN(), nullable=True), - sa.Column('created_at', sa.DATETIME(), nullable=True), - sa.Column('replaced_at', sa.DATETIME(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index f6c4f4d..fa74ad5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ bcrypt dnspython dkimpy cryptography +aiosmtplib # Web Frontend Dependencies Flask @@ -19,7 +20,7 @@ Flask-SQLAlchemy Jinja2 Werkzeug requests -Flask-Migrate +pytz gunicorn # Additional utilities diff --git a/tests/bash_send_email.sh b/tests/bash_send_email.sh index 105ed0e..b4984b1 100644 --- a/tests/bash_send_email.sh +++ b/tests/bash_send_email.sh @@ -1,69 +1,108 @@ #!/bin/bash -sender="test@example.com" +# apt-get install -y swaks receiver="info@example.com" -password="testpass123" +EMAIL_SERVER="localhost" #"pymta.example.com" "localhost" +EMAIL_SERVER_auth="10.100.111.1" # IP for authenticated server ( not localhost), use your main interface ip + +sender="test@example.com" +username="test@example.com" +password="ZjDvcjPSs-nwK2Ghj5vQY7L4LdmTpmn_AEZMokJTFS" # password you setup for the user! domain="example.com" body_content_file="@tests/email_body.txt" SMTP_PORT=4025 -SMTP_TLS_PORT=40587 -cc_recipient="targetcc@example.com" -bcc_recipient="targetbcc@example.com" +SMTP_TLS_PORT=40465 +cc_recipient="ccrecipient@example.com" +bcc_recipient="bccrecipient@example.com" <