checking encryption, fixing module calling

email_server/__init__.py

@@ -1,9 +1,3 @@
-from .auth import *
+"""
-from .dkim_manager import *
+PyMTA Server email package
-from .server_runner import *
+"""
-from .tls_utils import *
-from .cli_tools import *
-from .email_relay import *
-from .models import *
-from .smtp_handler import *
-from .config import *

email_server/auth.py

@@ -5,7 +5,7 @@ Authentication modules for the SMTP server.
 import logging
 from datetime import datetime
 from aiosmtpd.smtp import AuthResult, LoginPassword
-from .models import Session, User, Domain, WhitelistedIP, AuthLog, check_password
+from email_server.models import Session, User, Domain, WhitelistedIP, AuthLog, check_password
 
 logger = logging.getLogger(__name__)
 

email_server/cli_tools.py

@@ -3,9 +3,8 @@ Command-line tools for managing the SMTP server.
 """
 
 import argparse
-import sys
+from email_server.models import Session, Domain, User, WhitelistedIP, hash_password, create_tables
-from .models import Session, Domain, User, WhitelistedIP, hash_password, create_tables
+from email_server.dkim_manager import DKIMManager
-from .dkim_manager import DKIMManager
 import logging
 
 logging.basicConfig(level=logging.INFO)

email_server/config.py

@@ -6,6 +6,7 @@ Configuration settings for the SMTP server.
 SMTP_PORT = 4025
 SMTP_TLS_PORT = 40587
 HOSTNAME = 'localhost'
+BIND_IP = '0.0.0.0'
 
 # Database settings
 DATABASE_URL = 'sqlite:///email_server/server_data/smtp_server.db'

email_server/dkim_manager.py

@@ -7,8 +7,8 @@ import dkim
 from cryptography.hazmat.primitives import serialization, hashes
 from cryptography.hazmat.primitives.asymmetric import rsa
 from datetime import datetime
-from .models import Session, Domain, DKIMKey
+from email_server.models import Session, Domain, DKIMKey
-from .config import DKIM_SELECTOR, DKIM_KEY_SIZE
+from email_server.config import DKIM_SELECTOR, DKIM_KEY_SIZE
 
 logger = logging.getLogger(__name__)
 

email_server/email_relay.py

@@ -4,9 +4,10 @@ Email relay functionality for the SMTP server.
 
 import dns.resolver
 import smtplib
+import ssl
 import logging
 from datetime import datetime
-from .models import Session, EmailLog
+from email_server.models import Session, EmailLog
 
 logger = logging.getLogger(__name__)
 
@@ -14,33 +15,112 @@ class EmailRelay:
     """Handles relaying emails to recipient mail servers."""
     
     def __init__(self):
-        self.timeout = 10
+        self.timeout = 30  # Increased timeout for TLS negotiations
     
     def relay_email(self, mail_from, rcpt_tos, content):
-        """Relay email to recipient's mail server."""
+        """Relay email to recipient's mail server with opportunistic TLS."""
         try:
             for rcpt in rcpt_tos:
                 domain = rcpt.split('@')[1]
+                
+                # Resolve MX record for the domain
                 try:
                     mx_records = dns.resolver.resolve(domain, 'MX')
+                    # Sort by priority (lower number = higher priority)
+                    mx_records = sorted(mx_records, key=lambda x: x.preference)
                     mx_host = mx_records[0].exchange.to_text().rstrip('.')
+                    logger.info(f'Found MX record for {domain}: {mx_host}')
                 except Exception as e:
                     logger.error(f'Failed to resolve MX for {domain}: {e}')
                     return False
 
-                try:
+                # Try to relay with opportunistic TLS
-                    with smtplib.SMTP(mx_host, 25, timeout=self.timeout) as relay_server:
+                if not self._relay_with_opportunistic_tls(mail_from, rcpt, content, mx_host):
-                        relay_server.set_debuglevel(1)
-                        relay_server.sendmail(mail_from, rcpt, content)
-                    logger.info(f'Relayed email to {rcpt} via {mx_host}')
-                except Exception as e:
-                    logger.error(f'Failed to relay email to {rcpt}: {e}')
                     return False
+                    
             return True
         except Exception as e:
             logger.error(f'General relay error: {e}')
             return False
     
+    def _relay_with_opportunistic_tls(self, mail_from, rcpt, content, mx_host):
+        """Relay email with opportunistic TLS (like Gmail does)."""
+        try:
+            # First, try with STARTTLS (encrypted)
+            try:
+                with smtplib.SMTP(mx_host, 25, timeout=self.timeout) as relay_server:
+                    relay_server.set_debuglevel(1)
+                    
+                    # Try to enable TLS if the server supports it
+                    try:
+                        # Check if server supports STARTTLS
+                        relay_server.ehlo()
+                        if relay_server.has_extn('starttls'):
+                            logger.info(f'Starting TLS connection to {mx_host}')
+                            context = ssl.create_default_context()
+                            # Allow self-signed certificates for mail servers
+                            context.check_hostname = False
+                            context.verify_mode = ssl.CERT_NONE
+                            relay_server.starttls(context=context)
+                            relay_server.ehlo()  # Say hello again after STARTTLS
+                            logger.info(f'TLS connection established to {mx_host}')
+                        else:
+                            logger.warning(f'Server {mx_host} does not support STARTTLS, using plain text')
+                    except Exception as tls_e:
+                        logger.warning(f'STARTTLS failed with {mx_host}, continuing with plain text: {tls_e}')
+                    
+                    # Send the email
+                    relay_server.sendmail(mail_from, rcpt, content)
+                    logger.info(f'Successfully relayed email to {rcpt} via {mx_host}')
+                    return True
+                    
+            except Exception as e:
+                logger.error(f'Failed to relay email to {rcpt} via {mx_host}: {e}')
+                
+                # Fallback: try alternative MX records if available
+                try:
+                    domain = rcpt.split('@')[1]
+                    mx_records = dns.resolver.resolve(domain, 'MX')
+                    mx_records = sorted(mx_records, key=lambda x: x.preference)
+                    
+                    # Try other MX records
+                    for mx_record in mx_records[1:3]:  # Try up to 2 backup MX records
+                        backup_mx = mx_record.exchange.to_text().rstrip('.')
+                        logger.info(f'Trying backup MX record: {backup_mx}')
+                        
+                        try:
+                            with smtplib.SMTP(backup_mx, 25, timeout=self.timeout) as backup_server:
+                                backup_server.set_debuglevel(1)
+                                
+                                # Try TLS with backup server too
+                                try:
+                                    backup_server.ehlo()
+                                    if backup_server.has_extn('starttls'):
+                                        context = ssl.create_default_context()
+                                        context.check_hostname = False
+                                        context.verify_mode = ssl.CERT_NONE
+                                        backup_server.starttls(context=context)
+                                        backup_server.ehlo()
+                                        logger.info(f'TLS connection established to backup {backup_mx}')
+                                except Exception:
+                                    logger.warning(f'STARTTLS failed with backup {backup_mx}, using plain text')
+                                
+                                backup_server.sendmail(mail_from, rcpt, content)
+                                logger.info(f'Successfully relayed email to {rcpt} via backup {backup_mx}')
+                                return True
+                        except Exception as backup_e:
+                            logger.warning(f'Backup MX {backup_mx} also failed: {backup_e}')
+                            continue
+                            
+                except Exception as fallback_e:
+                    logger.error(f'All MX records failed for {rcpt}: {fallback_e}')
+                
+                return False
+                
+        except Exception as e:
+            logger.error(f'Unexpected error in TLS relay: {e}')
+            return False
+    
     def log_email(self, message_id, peer, mail_from, rcpt_tos, content, status, dkim_signed=False):
         """Log email activity to database."""
         session_db = Session()
@@ -68,4 +148,4 @@ class EmailRelay:
             session_db.rollback()
             logger.error(f'Error logging email: {e}')
         finally:
-            session_db.close()
+            session_db.close()

email_server/models.py

@@ -6,7 +6,10 @@ from sqlalchemy import create_engine, Column, Integer, String, DateTime, Text, B
 from sqlalchemy.orm import declarative_base, sessionmaker
 from datetime import datetime
 import bcrypt
-from .config import DATABASE_URL
+from email_server.config import DATABASE_URL
+from email_server.tool_box import ensure_folder_exists
+
+ensure_folder_exists(DATABASE_URL)
 
 # SQLAlchemy setup
 Base = declarative_base()

email_server/server_runner.py

@@ -5,15 +5,13 @@ Main server file that ties all modules together.
 
 import asyncio
 import logging
-import sys
-import os
 
 # Import our modules
-from .config import SMTP_PORT, SMTP_TLS_PORT, HOSTNAME, LOG_LEVEL
+from email_server.config import SMTP_PORT, SMTP_TLS_PORT, HOSTNAME, LOG_LEVEL, BIND_IP
-from .models import create_tables
+from email_server.models import create_tables
-from .smtp_handler import CustomSMTPHandler, PlainController
+from email_server.smtp_handler import CustomSMTPHandler, PlainController
-from .tls_utils import generate_self_signed_cert, create_ssl_context
+from email_server.tls_utils import generate_self_signed_cert, create_ssl_context
-from .dkim_manager import DKIMManager
+from email_server.dkim_manager import DKIMManager
 from aiosmtpd.controller import Controller
 from aiosmtpd.smtp import SMTP as AIOSMTP
 
@@ -98,7 +96,8 @@ async def start_server():
     handler_plain = CustomSMTPHandler()
     controller_plain = PlainController(
         handler_plain,
-        hostname=HOSTNAME,
+        hostname=BIND_IP,
+        server_hostname="TestEnvironment",
         port=SMTP_PORT
     )
     controller_plain.start()
@@ -122,21 +121,15 @@ async def start_server():
     
     controller_tls = TLSController(
         handler_tls,
-        hostname=HOSTNAME,
+        hostname=BIND_IP,
+        server_hostname="TestEnvironment",
         port=SMTP_TLS_PORT
     )
     controller_tls.start()
-    logger.info(f'Starting STARTTLS SMTP server on {HOSTNAME}:{SMTP_TLS_PORT}...')
+    logger.info(f'  - Plain SMTP (IP whitelist): {BIND_IP}:{SMTP_PORT}')
-    
+    logger.info(f'  - STARTTLS SMTP (auth required): {BIND_IP}:{SMTP_TLS_PORT}')
-    logger.info('Both SMTP servers are running:')
-    logger.info(f'  - Plain SMTP (IP whitelist): {HOSTNAME}:{SMTP_PORT}')
-    logger.info(f'  - STARTTLS SMTP (auth required): {HOSTNAME}:{SMTP_TLS_PORT}')
-    logger.info('  - DKIM signing enabled for configured domains')
-    logger.info('')
     logger.info('Management commands:')
     logger.info('  python cli_tools.py --help')
-    logger.info('')
-    logger.info('Press Ctrl+C to stop the servers...')
     
     try:
         await asyncio.Event().wait()
@@ -144,14 +137,4 @@ async def start_server():
         logger.info('Shutting down SMTP servers...')
         controller_plain.stop()
         controller_tls.stop()
-        logger.info('SMTP servers stopped.')
+        logger.info('SMTP servers stopped.')
-
-if __name__ == '__main__':
-    try:
-        asyncio.run(start_server())
-    except KeyboardInterrupt:
-        logger.info('Server interrupted by user')
-        sys.exit(0)
-    except Exception as e:
-        logger.error(f'Server error: {e}')
-        sys.exit(1)

email_server/smtp_handler.py

@@ -7,9 +7,9 @@ import uuid
 from datetime import datetime
 from aiosmtpd.smtp import SMTP as AIOSMTP, AuthResult
 from aiosmtpd.controller import Controller
-from .auth import Authenticator, IPAuthenticator
+from email_server.auth import Authenticator, IPAuthenticator
-from .email_relay import EmailRelay
+from email_server.email_relay import EmailRelay
-from .dkim_manager import DKIMManager
+from email_server.dkim_manager import DKIMManager
 
 logger = logging.getLogger(__name__)
 
@@ -124,7 +124,7 @@ class TLSController(Controller):
         return AIOSMTP(
             self.handler,
             tls_context=self.ssl_context,
-            require_starttls=False,  # Don't force STARTTLS, but make it available
+            require_starttls=True,  # Don't force STARTTLS, but make it available
             auth_require_tls=True,   # If auth is used, require TLS
             authenticator=self.handler.combined_authenticator,
             decode_data=True,

email_server/tls_utils.py

@@ -6,10 +6,14 @@ import ssl
 import os
 import logging
 from OpenSSL import crypto
-from .config import TLS_CERT_FILE, TLS_KEY_FILE
+from email_server.config import TLS_CERT_FILE, TLS_KEY_FILE
+from email_server.tool_box import ensure_folder_exists
 
 logger = logging.getLogger(__name__)
 
+ensure_folder_exists(TLS_CERT_FILE)
+ensure_folder_exists(TLS_KEY_FILE)
+
 def generate_self_signed_cert():
     """Generate self-signed SSL certificate if it doesn't exist."""
     if os.path.exists(TLS_CERT_FILE) and os.path.exists(TLS_KEY_FILE):
@@ -26,6 +30,8 @@ def generate_self_signed_cert():
         # Generate certificate
         cert = crypto.X509()
         cert.get_subject().CN = 'localhost'
+        cert.get_subject().O = 'PyMTA Server'
+        cert.get_subject().C = 'GB'
         cert.set_serial_number(1000)
         cert.gmtime_adj_notBefore(0)
         cert.gmtime_adj_notAfter(365 * 24 * 60 * 60)  # Valid for 1 year

email_server/tool_box.py

@@ -0,0 +1,13 @@
+"""
+Utility functions for the email server.
+"""
+
+import os
+
+def ensure_folder_exists(filepath):
+    """
+    Ensure that the folder for the given filepath exists.
+    """
+    if filepath.startswith("sqlite:///"):
+        filepath = filepath.replace("sqlite:///", "", 1)
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)

start_server.py

@@ -1,6 +1,10 @@
-from email_server import start_server, logger
+from email_server.server_runner import start_server
 import asyncio
 import sys
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
 if __name__ == '__main__':
     try:

tests/email_body.txt

@@ -0,0 +1,10 @@
+Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.
+
+Why do we use it?
+It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like).
+
+
+Where does it come from?
+Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.
+
+The standard chunk of Lorem Ipsum used since the 1500s is reproduced below for those interested. Sections 1.10.32 and 1.10.33 from "de Finibus Bonorum et Malorum" by Cicero are also reproduced in their exact original form, accompanied by English versions from the 1914 translation by H. Rackham."

tests/run_tests_manually.md

@@ -1,10 +1,10 @@
 ## setup domain and account for sending email:
 ```bash
-python email_server/cli_tools.py add-domain example.com
+python -m email_server.cli_tools.py add-domain example.com
-python email_server/cli_tools.py add-user test@example.com testpass123 example.com
+python -m email_server.cli_tools.py add-user test@example.com testpass123 example.com
-python email_server/cli_tools.py add-ip 127.0.0.1 example.com 
+python -m email_server.cli_tools.py add-ip 127.0.0.1 example.com 
-python email_server/cli_tools.py add-ip 10.100.111.1 example.com 
+python -m email_server.cli_tools.py add-ip 10.100.111.1 example.com 
-python email_server/cli_tools.py generate-dkim example.com
+python -m email_server.cli_tools.py generate-dkim example.com
 ```
 
 ## Check db logs
@@ -15,6 +15,18 @@ python email_server/cli_tools.py generate-dkim example.com
 
 ## Linux send emails using `swaks`
 ```bash
+# multiline test with body from the email_body.txt file:
+swaks --to info@example.com \
+      --from test@example.com \
+      --server localhost \
+      --port 40587 \
+      --auth LOGIN \
+      --auth-user test@example.com \
+      --auth-password testpass123 \
+      --tls \
+      --header "Subject: This is the subject" \
+      --body @tests/email_body.txt
+
 swaks --to info@example.com \
       --from test@example.com \
       --server localhost \