Add flexible import folder functionality

- Add IMPORT_FOLDER_NAME configuration variable - Support organized import (subfolders) or consolidated import (all to INBOX) - Implement get_destination_folder() method for folder mapping - Add email.utils import for proper date handling - Update logging to show source → destination folder mappings - Auto-create import folders as needed Configuration examples: - IMPORT_FOLDER_NAME=Imported → organized subfolders - IMPORT_FOLDER_NAME= → all emails to INBOX - Update .env.template with comprehensive explanations - Add Host Europe and German hosting provider examples - Include detailed configuration guides and troubleshooting - Add realistic migration scenarios - Update README.md with complete documentation - Add feature overview and configuration guide - Include usage examples and troubleshooting section - Document both import modes with clear examples
2025-09-24 14:09:30 +02:00 · 2025-09-24 14:09:30 +02:00 · 481e32bb73
commit 481e32bb73
parent bd54ae469b
3 changed files with 469 additions and 92 deletions
--- a/.env.template
+++ b/.env.template
@ -1,27 +1,150 @@
-# Source IMAP Server Configuration (Host Europe)
-SOURCE_IMAP_SERVER=
-SOURCE_IMAP_PORT=
-SOURCE_IMAP_USE_SSL=
-SOURCE_EMAIL=
-SOURCE_PASSWORD=
+# ============================================================================
+# EMAIL MIGRATION CONFIGURATION FILE
+# ============================================================================
+# Copy this file to '.env' and fill in your actual values.
+# NEVER commit the .env file to version control - it contains passwords!

-# Destination IMAP Server Configuration (Securehost.de)
-DEST_IMAP_SERVER=
-DEST_IMAP_PORT=
-DEST_IMAP_USE_SSL=
-DEST_EMAIL=
-DEST_PASSWORD=
+# ============================================================================
+# SOURCE EMAIL ACCOUNT (migrating FROM) - Host Europe Example
+# ============================================================================

-# Migration Settings
-TEMP_DOWNLOAD_DIR=
-LOG_LEVEL=
-BATCH_SIZE=
-PRESERVE_FLAGS=
-PRESERVE_DATES=
+# SOURCE_IMAP_SERVER: IMAP server hostname
+# Host Europe: wpxxxxxxxx.mail.server-he.de | Gmail: imap.gmail.com | Outlook: outlook.office365.com
+SOURCE_IMAP_SERVER=wp123456.mail.server-he.de

-# Optional: Folder filtering (comma-separated list, leave empty for all folders)
+# SOURCE_IMAP_PORT: IMAP port number
+# 993 (SSL recommended) | 143 (TLS/STARTTLS)
+SOURCE_IMAP_PORT=993
+
+# SOURCE_EMAIL: Your source email address
+# Example: user@yourdomain.de
+SOURCE_EMAIL=user@example-domain.de
+
+# SOURCE_PASSWORD: Email account password
+# Use your Host Europe email password (NOT app password needed here)
+SOURCE_PASSWORD=your_hosteurope_password
+
+# SOURCE_IMAP_USE_SSL: Use SSL encryption
+# True (port 993) | False (port 143 with TLS)
+SOURCE_IMAP_USE_SSL=True
+
+# ============================================================================
+# DESTINATION EMAIL ACCOUNT (migrating TO) - German Hosting Example
+# ============================================================================
+
+# DEST_IMAP_SERVER: Destination IMAP server
+# Common formats: mail.yourdomain.de | imap.provider.de
+DEST_IMAP_SERVER=mail.yourdomain.de
+
+# DEST_IMAP_PORT: Destination IMAP port
+# 993 (SSL) | 143 (TLS/STARTTLS)
+DEST_IMAP_PORT=993
+
+# DEST_EMAIL: Your destination email address
+# Example: user@newdomain.de
+DEST_EMAIL=user@newdomain.de
+
+# DEST_PASSWORD: Destination email password
+# Your hosting provider email password
+DEST_PASSWORD=your_destination_password
+
+# DEST_IMAP_USE_SSL: Use SSL for destination
+# True (recommended) | False
+DEST_IMAP_USE_SSL=True
+
+# ============================================================================
+# IMPORT FOLDER CONFIGURATION
+# ============================================================================
+
+# IMPORT_FOLDER_NAME: Where to organize imported emails
+#
+# Set to folder name: Creates organized subfolders
+#   "Imported" → Imported/INBOX, Imported/Sent, Imported/Drafts
+#   "Migration" → Migration/INBOX, Migration/Sent, etc.
+#
+# Leave empty: All emails go to main INBOX
+#   "" → All emails regardless of source folder → INBOX
+#
+# Examples: Imported | Migration_2024 | HostEurope_Backup | (empty)
+IMPORT_FOLDER_NAME=Imported
+
+# ============================================================================
+# FOLDER FILTERING
+# ============================================================================
+
+# INCLUDE_FOLDERS: Only migrate these folders (comma-separated)
+# Empty = migrate all folders
+# Host Europe common folders: INBOX,Sent,Drafts,Trash
 INCLUDE_FOLDERS=
-EXCLUDE_FOLDERS=

-# Connection timeout in seconds
-IMAP_TIMEOUT=
+# EXCLUDE_FOLDERS: Skip these folders (comma-separated)
+# Recommended: skip trash and spam folders
+EXCLUDE_FOLDERS=Trash,Spam,Junk
+
+# ============================================================================
+# MIGRATION SETTINGS
+# ============================================================================
+
+# BATCH_SIZE: Emails processed at once
+# 10 (conservative) | 50 (balanced) | 100 (aggressive)
+BATCH_SIZE=50
+
+# PRESERVE_FLAGS: Keep read/unread status
+# True (recommended) | False
+PRESERVE_FLAGS=True
+
+# PRESERVE_DATES: Keep original email dates
+# True (recommended) | False
+PRESERVE_DATES=True
+
+# ============================================================================
+# TECHNICAL SETTINGS
+# ============================================================================
+
+# IMAP_TIMEOUT: Connection timeout in seconds
+# 60 (default) | 120 (slow connections) | 30 (fast connections)
+IMAP_TIMEOUT=60
+
+# LOG_LEVEL: Logging detail level
+# INFO (recommended) | DEBUG (troubleshooting) | ERROR (minimal)
+LOG_LEVEL=INFO
+
+# TEMP_DOWNLOAD_DIR: Temporary files directory
+# ./temp_emails (default) | /tmp/emails | C:\Temp\emails
+TEMP_DOWNLOAD_DIR=./temp_emails
+
+# ============================================================================
+# PROVIDER-SPECIFIC EXAMPLES
+# ============================================================================
+
+# Host Europe Settings:
+# SOURCE_IMAP_SERVER=wpxxxxxxxx.mail.server-he.de
+# SOURCE_IMAP_PORT=993
+# SOURCE_IMAP_USE_SSL=True
+# Note: Replace 'wpxxxxxxxx' with your actual server name from Host Europe KIS
+
+# Common German Hosting Providers:
+# Strato: imap.strato.de:993
+# 1und1/IONOS: imap.1und1.de:993
+# All-Inkl: mail.all-inkl.com:993
+# Hetzner: mail.your-server.de:993
+
+# Gmail (if needed):
+# Gmail: imap.gmail.com:993 (requires app password)
+
+# ============================================================================
+# MIGRATION EXAMPLES
+# ============================================================================
+
+# Example 1: Host Europe to new hosting with organization
+# IMPORT_FOLDER_NAME=HostEurope_Migration
+# Result: Clean separation in destination
+
+# Example 2: Consolidate everything to main inbox
+# IMPORT_FOLDER_NAME=
+# Result: All emails in main INBOX
+
+# Example 3: Test migration with one folder
+# INCLUDE_FOLDERS=INBOX
+# BATCH_SIZE=10
+# LOG_LEVEL=DEBUG
--- a/README.md
+++ b/README.md
@ -1,20 +1,239 @@
 # Email Migration Script

-A Python script to migrate emails from one IMAP account to another, preserving folder structure and metadata.
+A Python script to migrate emails from one IMAP account to another with flexible folder organization options.
+
+## Features
+
+- **Flexible Import Options**: Choose between organized folder structure or consolidated inbox
+- **Folder Filtering**: Include/exclude specific folders from migration
+- **Metadata Preservation**: Maintain email flags, dates, and other metadata
+- **Batch Processing**: Configurable batch sizes for optimal performance
+- **Comprehensive Logging**: Detailed logs for monitoring and troubleshooting
+- **SSL Security**: Secure connections with SSL/TLS support

 ## Quick Start

-1. Edit .env file with your email credentials
-2. Run: python3 email_migration.py
+1. **Install**: No dependencies required - uses Python standard library
+2. **Configure**: Copy `.env.template` to `.env` and add your email credentials
+3. **Run**: `python3 email_migration.py`

 ## Requirements
+
 - Python 3.6+ (pre-installed on Linux & macOS)
- No additional dependencies required
+- IMAP access enabled on both email accounts
+- App passwords (recommended for Gmail, Yahoo, Outlook)

-## Configuration
-Edit the .env file with your email account settings.
+## Import Folder Options

-## Important Notes
- Use app passwords, not regular passwords
- Test with a small folder first
- Check email_migration.log for detailed logs
+### Option 1: Organized Import (Recommended)
+
+```env
+IMPORT_FOLDER_NAME=Imported
+```
+
+**Result**: All source folders become subfolders within "Imported"
+- Source `INBOX` → Destination `Imported/INBOX`
+- Source `Sent` → Destination `Imported/Sent`
+- Source `Drafts` → Destination `Imported/Drafts`
+
+**Benefits**: 
+- Preserves original folder structure
+- Keeps imported emails separate from existing emails
+- Easy to locate and organize imported content
+
+### Option 2: Consolidated Import
+
+```env
+IMPORT_FOLDER_NAME=
+```
+
+**Result**: All emails go directly to destination INBOX
+- All source folders → Destination `INBOX`
+
+**Benefits**:
+- Simple single-folder result
+- Useful for merging multiple accounts
+- Good for basic email consolidation
+
+## Configuration Guide
+
+### Email Account Setup
+
+```env
+# Source account (migrating FROM)
+SOURCE_IMAP_SERVER=imap.gmail.com
+SOURCE_EMAIL=old@gmail.com
+SOURCE_PASSWORD=your_app_password
+
+# Destination account (migrating TO)  
+DEST_IMAP_SERVER=imap.gmail.com
+DEST_EMAIL=new@gmail.com
+DEST_PASSWORD=your_app_password
+```
+
+### Common IMAP Servers
+
+| Provider | IMAP Server | Port | SSL |
+|----------|-------------|------|-----|
+| Gmail | `imap.gmail.com` | 993 | Yes |
+| Outlook/Hotmail | `outlook.office365.com` | 993 | Yes |
+| Yahoo | `imap.mail.yahoo.com` | 993 | Yes |
+| Apple iCloud | `imap.mail.me.com` | 993 | Yes |
+
+### App Password Setup
+
+**Gmail**: Google Account → Security → App Passwords  
+**Yahoo**: Account Security → Generate app password  
+**Outlook**: Account Security → App passwords  
+
+⚠️ **Important**: Use app passwords, not regular login passwords!
+
+## Advanced Configuration
+
+### Folder Filtering
+
+```env
+# Only migrate specific folders
+INCLUDE_FOLDERS=INBOX,Sent,Important
+
+# Skip unwanted folders  
+EXCLUDE_FOLDERS=Trash,Spam,Junk
+```
+
+### Performance Tuning
+
+```env
+# Batch size (emails per batch)
+BATCH_SIZE=50          # Default balance
+BATCH_SIZE=10          # Conservative (slow connections)
+BATCH_SIZE=100         # Aggressive (fast connections)
+
+# Connection timeout
+IMAP_TIMEOUT=60        # Default
+IMAP_TIMEOUT=120       # Slow connections
+```
+
+### Logging and Debugging
+
+```env
+LOG_LEVEL=INFO         # Normal operation
+LOG_LEVEL=DEBUG        # Detailed troubleshooting
+LOG_LEVEL=ERROR        # Errors only
+```
+
+## Usage Examples
+
+### Example 1: Full Gmail Migration with Organization
+```bash
+# .env configuration
+SOURCE_EMAIL=old@gmail.com
+DEST_EMAIL=new@gmail.com
+IMPORT_FOLDER_NAME=OldAccount
+EXCLUDE_FOLDERS=Trash,Spam
+
+# Run migration
+python3 email_migration.py
+```
+
+### Example 2: Test Migration (INBOX only)
+```bash
+# .env configuration  
+INCLUDE_FOLDERS=INBOX
+BATCH_SIZE=10
+LOG_LEVEL=DEBUG
+
+# Run migration
+python3 email_migration.py
+```
+
+### Example 3: Consolidate Multiple Accounts
+```bash
+# .env configuration
+IMPORT_FOLDER_NAME=
+# This puts all emails in main INBOX
+
+# Run migration
+python3 email_migration.py
+```
+
+## Monitoring Progress
+
+The script provides detailed logging in multiple places:
+
+1. **Console Output**: Real-time progress updates
+2. **Log File**: `email_migration.log` with detailed information
+3. **Final Summary**: Complete migration statistics
+
+Example output:
+```
+Email Migration Script
+==================================================
+[INFO] Import folder configuration: All emails will be imported to subfolders within "Imported"
+[INFO] Found 5 folders to process
+[INFO] Processing folder: INBOX
+[INFO] Migrating 'INBOX' -> 'Imported/INBOX'
+[INFO] Downloaded 150/150 messages from INBOX
+[INFO] Uploaded 150/150 messages to Imported/INBOX
+[INFO] Folder 'INBOX' -> 'Imported/INBOX' completed: 150 downloaded, 150 uploaded
+
+Migration completed!
+Folders processed: 5
+Total emails downloaded: 1250
+Total emails uploaded: 1250
+Errors encountered: 0
+```
+
+## Troubleshooting
+
+### Common Issues
+
+**Authentication Failed**
+- Use app passwords instead of regular passwords
+- Enable IMAP access in email account settings
+- Check server settings and ports
+
+**Connection Timeout**
+- Increase `IMAP_TIMEOUT` value
+- Reduce `BATCH_SIZE` for stability
+- Check network connection
+
+**Folder Creation Failed**
+- Verify destination account has folder creation permissions
+- Check for special characters in folder names
+- Some providers have folder name restrictions
+
+**Partial Migration**
+- Check logs in `email_migration.log`
+- Re-run script - it will skip already migrated emails
+- Use `INCLUDE_FOLDERS` to retry specific folders
+
+### Getting Help
+
+1. **Enable Debug Logging**: Set `LOG_LEVEL=DEBUG`
+2. **Check Log File**: Review `email_migration.log`
+3. **Test Small First**: Use `INCLUDE_FOLDERS=INBOX` and `BATCH_SIZE=10`
+4. **Verify Credentials**: Test account access with email client first
+
+## Security Notes
+
+- Never commit `.env` file to version control
+- Use app passwords instead of account passwords
+- Ensure SSL is enabled (`*_IMAP_USE_SSL=True`)
+- Store credentials securely
+- Test with non-critical accounts first
+
+## File Structure
+
+```
+email_migration/
+├── email_migration.py     # Main script
+├── .env.template         # Configuration template
+├── .env                  # Your configuration (create from template)
+├── email_migration.log   # Generated log file
+├── temp_emails/          # Temporary files (auto-created)
+└── README.md            # This file
+```
+
+## License
+
+This script is provided as-is for educational and personal use. Test thoroughly before production use.
--- a/email_migration.py
+++ b/email_migration.py
@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import imaplib
 import email
+import email.utils
 import ssl
 import logging
 from datetime import datetime
@ -138,6 +139,13 @@ class EmailMigrator:
        self.preserve_flags = config.get('PRESERVE_FLAGS', 'True').lower() == 'true'
        self.preserve_dates = config.get('PRESERVE_DATES', 'True').lower() == 'true'

+        # New: Import folder configuration
+        self.import_folder_name = config.get('IMPORT_FOLDER_NAME', '').strip()
+        if not self.import_folder_name:
+            self.import_folder_name = None
+
+        self.logger.info(f"Import folder configuration: {'All emails will be imported to subfolders within \"' + self.import_folder_name + '\"' if self.import_folder_name else 'All emails will be imported directly to INBOX'}")
+
        include_str = config.get('INCLUDE_FOLDERS', '')
        exclude_str = config.get('EXCLUDE_FOLDERS', '')

@ -164,6 +172,23 @@ class EmailMigrator:
            timeout
        )

+    def get_destination_folder(self, source_folder):
+        """
+        Determine the destination folder based on the import configuration.
+
+        Args:
+            source_folder (str): Original folder name from source
+
+        Returns:
+            str: Destination folder name
+        """
+        if self.import_folder_name:
+            # Import into subfolders within the specified import folder
+            return f"{self.import_folder_name}/{source_folder}"
+        else:
+            # Import all emails directly to INBOX
+            return "INBOX"
+
    def should_process_folder(self, folder):
        if self.include_folders and folder not in self.include_folders:
            return False
@ -201,8 +226,8 @@ class EmailMigrator:
        self.logger.info(f"Successfully downloaded {len(emails)} messages from {folder}")
        return emails

-    def upload_emails_to_folder(self, emails, folder):
-        self.logger.info(f"Uploading {len(emails)} emails to folder: {folder}")
+    def upload_emails_to_folder(self, emails, destination_folder):
+        self.logger.info(f"Uploading {len(emails)} emails to folder: {destination_folder}")
        uploaded = 0
        for i, email_data in enumerate(emails, 1):
            try:
@ -216,33 +241,37 @@ class EmailMigrator:
                    except:
                        pass

-                if self.destination.append_message(folder, message, flags, date_obj):
+                if self.destination.append_message(destination_folder, message, flags, date_obj):
                    uploaded += 1

                if i % self.batch_size == 0:
-                    self.logger.info(f"Uploaded {i}/{len(emails)} messages to {folder}")
+                    self.logger.info(f"Uploaded {i}/{len(emails)} messages to {destination_folder}")
            except Exception as e:
-                self.logger.error(f"Error uploading message to {folder}: {e}")
+                self.logger.error(f"Error uploading message to {destination_folder}: {e}")
                continue

-        self.logger.info(f"Successfully uploaded {uploaded}/{len(emails)} messages to {folder}")
+        self.logger.info(f"Successfully uploaded {uploaded}/{len(emails)} messages to {destination_folder}")
        return uploaded

-    def migrate_folder(self, folder):
+    def migrate_folder(self, source_folder):
        stats = {'downloaded': 0, 'uploaded': 0}

-        if not self.should_process_folder(folder):
-            self.logger.info(f"Skipping folder: {folder} (filtered)")
+        if not self.should_process_folder(source_folder):
+            self.logger.info(f"Skipping folder: {source_folder} (filtered)")
            return stats

        try:
-            emails = self.download_emails_from_folder(folder)
+            # Determine destination folder based on configuration
+            destination_folder = self.get_destination_folder(source_folder)
+            self.logger.info(f"Migrating '{source_folder}' -> '{destination_folder}'")
+
+            emails = self.download_emails_from_folder(source_folder)
            stats['downloaded'] = len(emails)

            if emails:
-                stats['uploaded'] = self.upload_emails_to_folder(emails, folder)
+                stats['uploaded'] = self.upload_emails_to_folder(emails, destination_folder)
        except Exception as e:
-            self.logger.error(f"Error migrating folder {folder}: {e}")
+            self.logger.error(f"Error migrating folder {source_folder}: {e}")

        return stats

@ -262,6 +291,11 @@ class EmailMigrator:
            folders = self.source.get_folders()
            self.logger.info(f"Found {len(folders)} folders to process")

+            # Create the main import folder if specified
+            if self.import_folder_name:
+                self.logger.info(f"Creating main import folder: {self.import_folder_name}")
+                self.destination.create_folder(self.import_folder_name)
+
            for folder in folders:
                try:
                    self.logger.info(f"Processing folder: {folder}")
@ -271,7 +305,8 @@ class EmailMigrator:
                    total_stats['total_downloaded'] += stats['downloaded']
                    total_stats['total_uploaded'] += stats['uploaded']

-                    self.logger.info(f"Folder '{folder}' completed: {stats['downloaded']} downloaded, {stats['uploaded']} uploaded")
+                    destination_folder = self.get_destination_folder(folder)
+                    self.logger.info(f"Folder '{folder}' -> '{destination_folder}' completed: {stats['downloaded']} downloaded, {stats['uploaded']} uploaded")
                except Exception as e:
                    self.logger.error(f"Error processing folder {folder}: {e}")
                    total_stats['errors'] += 1