Add flexible import folder functionality
- Add IMPORT_FOLDER_NAME configuration variable - Support organized import (subfolders) or consolidated import (all to INBOX) - Implement get_destination_folder() method for folder mapping - Add email.utils import for proper date handling - Update logging to show source → destination folder mappings - Auto-create import folders as needed Configuration examples: - IMPORT_FOLDER_NAME=Imported → organized subfolders - IMPORT_FOLDER_NAME= → all emails to INBOX - Update .env.template with comprehensive explanations - Add Host Europe and German hosting provider examples - Include detailed configuration guides and troubleshooting - Add realistic migration scenarios - Update README.md with complete documentation - Add feature overview and configuration guide - Include usage examples and troubleshooting section - Document both import modes with clear examples
This commit is contained in:
parent
bd54ae469b
commit
481e32bb73
3 changed files with 469 additions and 92 deletions
167
.env.template
167
.env.template
|
|
@ -1,27 +1,150 @@
|
||||||
# Source IMAP Server Configuration (Host Europe)
|
# ============================================================================
|
||||||
SOURCE_IMAP_SERVER=
|
# EMAIL MIGRATION CONFIGURATION FILE
|
||||||
SOURCE_IMAP_PORT=
|
# ============================================================================
|
||||||
SOURCE_IMAP_USE_SSL=
|
# Copy this file to '.env' and fill in your actual values.
|
||||||
SOURCE_EMAIL=
|
# NEVER commit the .env file to version control - it contains passwords!
|
||||||
SOURCE_PASSWORD=
|
|
||||||
|
|
||||||
# Destination IMAP Server Configuration (Securehost.de)
|
# ============================================================================
|
||||||
DEST_IMAP_SERVER=
|
# SOURCE EMAIL ACCOUNT (migrating FROM) - Host Europe Example
|
||||||
DEST_IMAP_PORT=
|
# ============================================================================
|
||||||
DEST_IMAP_USE_SSL=
|
|
||||||
DEST_EMAIL=
|
|
||||||
DEST_PASSWORD=
|
|
||||||
|
|
||||||
# Migration Settings
|
# SOURCE_IMAP_SERVER: IMAP server hostname
|
||||||
TEMP_DOWNLOAD_DIR=
|
# Host Europe: wpxxxxxxxx.mail.server-he.de | Gmail: imap.gmail.com | Outlook: outlook.office365.com
|
||||||
LOG_LEVEL=
|
SOURCE_IMAP_SERVER=wp123456.mail.server-he.de
|
||||||
BATCH_SIZE=
|
|
||||||
PRESERVE_FLAGS=
|
|
||||||
PRESERVE_DATES=
|
|
||||||
|
|
||||||
# Optional: Folder filtering (comma-separated list, leave empty for all folders)
|
# SOURCE_IMAP_PORT: IMAP port number
|
||||||
|
# 993 (SSL recommended) | 143 (TLS/STARTTLS)
|
||||||
|
SOURCE_IMAP_PORT=993
|
||||||
|
|
||||||
|
# SOURCE_EMAIL: Your source email address
|
||||||
|
# Example: user@yourdomain.de
|
||||||
|
SOURCE_EMAIL=user@example-domain.de
|
||||||
|
|
||||||
|
# SOURCE_PASSWORD: Email account password
|
||||||
|
# Use your Host Europe email password (NOT app password needed here)
|
||||||
|
SOURCE_PASSWORD=your_hosteurope_password
|
||||||
|
|
||||||
|
# SOURCE_IMAP_USE_SSL: Use SSL encryption
|
||||||
|
# True (port 993) | False (port 143 with TLS)
|
||||||
|
SOURCE_IMAP_USE_SSL=True
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# DESTINATION EMAIL ACCOUNT (migrating TO) - German Hosting Example
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# DEST_IMAP_SERVER: Destination IMAP server
|
||||||
|
# Common formats: mail.yourdomain.de | imap.provider.de
|
||||||
|
DEST_IMAP_SERVER=mail.yourdomain.de
|
||||||
|
|
||||||
|
# DEST_IMAP_PORT: Destination IMAP port
|
||||||
|
# 993 (SSL) | 143 (TLS/STARTTLS)
|
||||||
|
DEST_IMAP_PORT=993
|
||||||
|
|
||||||
|
# DEST_EMAIL: Your destination email address
|
||||||
|
# Example: user@newdomain.de
|
||||||
|
DEST_EMAIL=user@newdomain.de
|
||||||
|
|
||||||
|
# DEST_PASSWORD: Destination email password
|
||||||
|
# Your hosting provider email password
|
||||||
|
DEST_PASSWORD=your_destination_password
|
||||||
|
|
||||||
|
# DEST_IMAP_USE_SSL: Use SSL for destination
|
||||||
|
# True (recommended) | False
|
||||||
|
DEST_IMAP_USE_SSL=True
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# IMPORT FOLDER CONFIGURATION
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# IMPORT_FOLDER_NAME: Where to organize imported emails
|
||||||
|
#
|
||||||
|
# Set to folder name: Creates organized subfolders
|
||||||
|
# "Imported" → Imported/INBOX, Imported/Sent, Imported/Drafts
|
||||||
|
# "Migration" → Migration/INBOX, Migration/Sent, etc.
|
||||||
|
#
|
||||||
|
# Leave empty: All emails go to main INBOX
|
||||||
|
# "" → All emails regardless of source folder → INBOX
|
||||||
|
#
|
||||||
|
# Examples: Imported | Migration_2024 | HostEurope_Backup | (empty)
|
||||||
|
IMPORT_FOLDER_NAME=Imported
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# FOLDER FILTERING
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# INCLUDE_FOLDERS: Only migrate these folders (comma-separated)
|
||||||
|
# Empty = migrate all folders
|
||||||
|
# Host Europe common folders: INBOX,Sent,Drafts,Trash
|
||||||
INCLUDE_FOLDERS=
|
INCLUDE_FOLDERS=
|
||||||
EXCLUDE_FOLDERS=
|
|
||||||
|
|
||||||
# Connection timeout in seconds
|
# EXCLUDE_FOLDERS: Skip these folders (comma-separated)
|
||||||
IMAP_TIMEOUT=
|
# Recommended: skip trash and spam folders
|
||||||
|
EXCLUDE_FOLDERS=Trash,Spam,Junk
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# MIGRATION SETTINGS
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# BATCH_SIZE: Emails processed at once
|
||||||
|
# 10 (conservative) | 50 (balanced) | 100 (aggressive)
|
||||||
|
BATCH_SIZE=50
|
||||||
|
|
||||||
|
# PRESERVE_FLAGS: Keep read/unread status
|
||||||
|
# True (recommended) | False
|
||||||
|
PRESERVE_FLAGS=True
|
||||||
|
|
||||||
|
# PRESERVE_DATES: Keep original email dates
|
||||||
|
# True (recommended) | False
|
||||||
|
PRESERVE_DATES=True
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# TECHNICAL SETTINGS
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# IMAP_TIMEOUT: Connection timeout in seconds
|
||||||
|
# 60 (default) | 120 (slow connections) | 30 (fast connections)
|
||||||
|
IMAP_TIMEOUT=60
|
||||||
|
|
||||||
|
# LOG_LEVEL: Logging detail level
|
||||||
|
# INFO (recommended) | DEBUG (troubleshooting) | ERROR (minimal)
|
||||||
|
LOG_LEVEL=INFO
|
||||||
|
|
||||||
|
# TEMP_DOWNLOAD_DIR: Temporary files directory
|
||||||
|
# ./temp_emails (default) | /tmp/emails | C:\Temp\emails
|
||||||
|
TEMP_DOWNLOAD_DIR=./temp_emails
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# PROVIDER-SPECIFIC EXAMPLES
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Host Europe Settings:
|
||||||
|
# SOURCE_IMAP_SERVER=wpxxxxxxxx.mail.server-he.de
|
||||||
|
# SOURCE_IMAP_PORT=993
|
||||||
|
# SOURCE_IMAP_USE_SSL=True
|
||||||
|
# Note: Replace 'wpxxxxxxxx' with your actual server name from Host Europe KIS
|
||||||
|
|
||||||
|
# Common German Hosting Providers:
|
||||||
|
# Strato: imap.strato.de:993
|
||||||
|
# 1und1/IONOS: imap.1und1.de:993
|
||||||
|
# All-Inkl: mail.all-inkl.com:993
|
||||||
|
# Hetzner: mail.your-server.de:993
|
||||||
|
|
||||||
|
# Gmail (if needed):
|
||||||
|
# Gmail: imap.gmail.com:993 (requires app password)
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# MIGRATION EXAMPLES
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Example 1: Host Europe to new hosting with organization
|
||||||
|
# IMPORT_FOLDER_NAME=HostEurope_Migration
|
||||||
|
# Result: Clean separation in destination
|
||||||
|
|
||||||
|
# Example 2: Consolidate everything to main inbox
|
||||||
|
# IMPORT_FOLDER_NAME=
|
||||||
|
# Result: All emails in main INBOX
|
||||||
|
|
||||||
|
# Example 3: Test migration with one folder
|
||||||
|
# INCLUDE_FOLDERS=INBOX
|
||||||
|
# BATCH_SIZE=10
|
||||||
|
# LOG_LEVEL=DEBUG
|
||||||
|
|
|
||||||
239
README.md
239
README.md
|
|
@ -1,20 +1,239 @@
|
||||||
# Email Migration Script
|
# Email Migration Script
|
||||||
|
|
||||||
A Python script to migrate emails from one IMAP account to another, preserving folder structure and metadata.
|
A Python script to migrate emails from one IMAP account to another with flexible folder organization options.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Flexible Import Options**: Choose between organized folder structure or consolidated inbox
|
||||||
|
- **Folder Filtering**: Include/exclude specific folders from migration
|
||||||
|
- **Metadata Preservation**: Maintain email flags, dates, and other metadata
|
||||||
|
- **Batch Processing**: Configurable batch sizes for optimal performance
|
||||||
|
- **Comprehensive Logging**: Detailed logs for monitoring and troubleshooting
|
||||||
|
- **SSL Security**: Secure connections with SSL/TLS support
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
1. Edit .env file with your email credentials
|
1. **Install**: No dependencies required - uses Python standard library
|
||||||
2. Run: python3 email_migration.py
|
2. **Configure**: Copy `.env.template` to `.env` and add your email credentials
|
||||||
|
3. **Run**: `python3 email_migration.py`
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
- Python 3.6+ (pre-installed on Linux & macOS)
|
- Python 3.6+ (pre-installed on Linux & macOS)
|
||||||
- No additional dependencies required
|
- IMAP access enabled on both email accounts
|
||||||
|
- App passwords (recommended for Gmail, Yahoo, Outlook)
|
||||||
|
|
||||||
## Configuration
|
## Import Folder Options
|
||||||
Edit the .env file with your email account settings.
|
|
||||||
|
|
||||||
## Important Notes
|
### Option 1: Organized Import (Recommended)
|
||||||
- Use app passwords, not regular passwords
|
|
||||||
- Test with a small folder first
|
```env
|
||||||
- Check email_migration.log for detailed logs
|
IMPORT_FOLDER_NAME=Imported
|
||||||
|
```
|
||||||
|
|
||||||
|
**Result**: All source folders become subfolders within "Imported"
|
||||||
|
- Source `INBOX` → Destination `Imported/INBOX`
|
||||||
|
- Source `Sent` → Destination `Imported/Sent`
|
||||||
|
- Source `Drafts` → Destination `Imported/Drafts`
|
||||||
|
|
||||||
|
**Benefits**:
|
||||||
|
- Preserves original folder structure
|
||||||
|
- Keeps imported emails separate from existing emails
|
||||||
|
- Easy to locate and organize imported content
|
||||||
|
|
||||||
|
### Option 2: Consolidated Import
|
||||||
|
|
||||||
|
```env
|
||||||
|
IMPORT_FOLDER_NAME=
|
||||||
|
```
|
||||||
|
|
||||||
|
**Result**: All emails go directly to destination INBOX
|
||||||
|
- All source folders → Destination `INBOX`
|
||||||
|
|
||||||
|
**Benefits**:
|
||||||
|
- Simple single-folder result
|
||||||
|
- Useful for merging multiple accounts
|
||||||
|
- Good for basic email consolidation
|
||||||
|
|
||||||
|
## Configuration Guide
|
||||||
|
|
||||||
|
### Email Account Setup
|
||||||
|
|
||||||
|
```env
|
||||||
|
# Source account (migrating FROM)
|
||||||
|
SOURCE_IMAP_SERVER=imap.gmail.com
|
||||||
|
SOURCE_EMAIL=old@gmail.com
|
||||||
|
SOURCE_PASSWORD=your_app_password
|
||||||
|
|
||||||
|
# Destination account (migrating TO)
|
||||||
|
DEST_IMAP_SERVER=imap.gmail.com
|
||||||
|
DEST_EMAIL=new@gmail.com
|
||||||
|
DEST_PASSWORD=your_app_password
|
||||||
|
```
|
||||||
|
|
||||||
|
### Common IMAP Servers
|
||||||
|
|
||||||
|
| Provider | IMAP Server | Port | SSL |
|
||||||
|
|----------|-------------|------|-----|
|
||||||
|
| Gmail | `imap.gmail.com` | 993 | Yes |
|
||||||
|
| Outlook/Hotmail | `outlook.office365.com` | 993 | Yes |
|
||||||
|
| Yahoo | `imap.mail.yahoo.com` | 993 | Yes |
|
||||||
|
| Apple iCloud | `imap.mail.me.com` | 993 | Yes |
|
||||||
|
|
||||||
|
### App Password Setup
|
||||||
|
|
||||||
|
**Gmail**: Google Account → Security → App Passwords
|
||||||
|
**Yahoo**: Account Security → Generate app password
|
||||||
|
**Outlook**: Account Security → App passwords
|
||||||
|
|
||||||
|
⚠️ **Important**: Use app passwords, not regular login passwords!
|
||||||
|
|
||||||
|
## Advanced Configuration
|
||||||
|
|
||||||
|
### Folder Filtering
|
||||||
|
|
||||||
|
```env
|
||||||
|
# Only migrate specific folders
|
||||||
|
INCLUDE_FOLDERS=INBOX,Sent,Important
|
||||||
|
|
||||||
|
# Skip unwanted folders
|
||||||
|
EXCLUDE_FOLDERS=Trash,Spam,Junk
|
||||||
|
```
|
||||||
|
|
||||||
|
### Performance Tuning
|
||||||
|
|
||||||
|
```env
|
||||||
|
# Batch size (emails per batch)
|
||||||
|
BATCH_SIZE=50 # Default balance
|
||||||
|
BATCH_SIZE=10 # Conservative (slow connections)
|
||||||
|
BATCH_SIZE=100 # Aggressive (fast connections)
|
||||||
|
|
||||||
|
# Connection timeout
|
||||||
|
IMAP_TIMEOUT=60 # Default
|
||||||
|
IMAP_TIMEOUT=120 # Slow connections
|
||||||
|
```
|
||||||
|
|
||||||
|
### Logging and Debugging
|
||||||
|
|
||||||
|
```env
|
||||||
|
LOG_LEVEL=INFO # Normal operation
|
||||||
|
LOG_LEVEL=DEBUG # Detailed troubleshooting
|
||||||
|
LOG_LEVEL=ERROR # Errors only
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage Examples
|
||||||
|
|
||||||
|
### Example 1: Full Gmail Migration with Organization
|
||||||
|
```bash
|
||||||
|
# .env configuration
|
||||||
|
SOURCE_EMAIL=old@gmail.com
|
||||||
|
DEST_EMAIL=new@gmail.com
|
||||||
|
IMPORT_FOLDER_NAME=OldAccount
|
||||||
|
EXCLUDE_FOLDERS=Trash,Spam
|
||||||
|
|
||||||
|
# Run migration
|
||||||
|
python3 email_migration.py
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example 2: Test Migration (INBOX only)
|
||||||
|
```bash
|
||||||
|
# .env configuration
|
||||||
|
INCLUDE_FOLDERS=INBOX
|
||||||
|
BATCH_SIZE=10
|
||||||
|
LOG_LEVEL=DEBUG
|
||||||
|
|
||||||
|
# Run migration
|
||||||
|
python3 email_migration.py
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example 3: Consolidate Multiple Accounts
|
||||||
|
```bash
|
||||||
|
# .env configuration
|
||||||
|
IMPORT_FOLDER_NAME=
|
||||||
|
# This puts all emails in main INBOX
|
||||||
|
|
||||||
|
# Run migration
|
||||||
|
python3 email_migration.py
|
||||||
|
```
|
||||||
|
|
||||||
|
## Monitoring Progress
|
||||||
|
|
||||||
|
The script provides detailed logging in multiple places:
|
||||||
|
|
||||||
|
1. **Console Output**: Real-time progress updates
|
||||||
|
2. **Log File**: `email_migration.log` with detailed information
|
||||||
|
3. **Final Summary**: Complete migration statistics
|
||||||
|
|
||||||
|
Example output:
|
||||||
|
```
|
||||||
|
Email Migration Script
|
||||||
|
==================================================
|
||||||
|
[INFO] Import folder configuration: All emails will be imported to subfolders within "Imported"
|
||||||
|
[INFO] Found 5 folders to process
|
||||||
|
[INFO] Processing folder: INBOX
|
||||||
|
[INFO] Migrating 'INBOX' -> 'Imported/INBOX'
|
||||||
|
[INFO] Downloaded 150/150 messages from INBOX
|
||||||
|
[INFO] Uploaded 150/150 messages to Imported/INBOX
|
||||||
|
[INFO] Folder 'INBOX' -> 'Imported/INBOX' completed: 150 downloaded, 150 uploaded
|
||||||
|
|
||||||
|
Migration completed!
|
||||||
|
Folders processed: 5
|
||||||
|
Total emails downloaded: 1250
|
||||||
|
Total emails uploaded: 1250
|
||||||
|
Errors encountered: 0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Common Issues
|
||||||
|
|
||||||
|
**Authentication Failed**
|
||||||
|
- Use app passwords instead of regular passwords
|
||||||
|
- Enable IMAP access in email account settings
|
||||||
|
- Check server settings and ports
|
||||||
|
|
||||||
|
**Connection Timeout**
|
||||||
|
- Increase `IMAP_TIMEOUT` value
|
||||||
|
- Reduce `BATCH_SIZE` for stability
|
||||||
|
- Check network connection
|
||||||
|
|
||||||
|
**Folder Creation Failed**
|
||||||
|
- Verify destination account has folder creation permissions
|
||||||
|
- Check for special characters in folder names
|
||||||
|
- Some providers have folder name restrictions
|
||||||
|
|
||||||
|
**Partial Migration**
|
||||||
|
- Check logs in `email_migration.log`
|
||||||
|
- Re-run script - it will skip already migrated emails
|
||||||
|
- Use `INCLUDE_FOLDERS` to retry specific folders
|
||||||
|
|
||||||
|
### Getting Help
|
||||||
|
|
||||||
|
1. **Enable Debug Logging**: Set `LOG_LEVEL=DEBUG`
|
||||||
|
2. **Check Log File**: Review `email_migration.log`
|
||||||
|
3. **Test Small First**: Use `INCLUDE_FOLDERS=INBOX` and `BATCH_SIZE=10`
|
||||||
|
4. **Verify Credentials**: Test account access with email client first
|
||||||
|
|
||||||
|
## Security Notes
|
||||||
|
|
||||||
|
- Never commit `.env` file to version control
|
||||||
|
- Use app passwords instead of account passwords
|
||||||
|
- Ensure SSL is enabled (`*_IMAP_USE_SSL=True`)
|
||||||
|
- Store credentials securely
|
||||||
|
- Test with non-critical accounts first
|
||||||
|
|
||||||
|
## File Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
email_migration/
|
||||||
|
├── email_migration.py # Main script
|
||||||
|
├── .env.template # Configuration template
|
||||||
|
├── .env # Your configuration (create from template)
|
||||||
|
├── email_migration.log # Generated log file
|
||||||
|
├── temp_emails/ # Temporary files (auto-created)
|
||||||
|
└── README.md # This file
|
||||||
|
```
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
This script is provided as-is for educational and personal use. Test thoroughly before production use.
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import imaplib
|
import imaplib
|
||||||
import email
|
import email
|
||||||
|
import email.utils
|
||||||
import ssl
|
import ssl
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
@ -38,7 +39,7 @@ class IMAPConnection:
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.connection = None
|
self.connection = None
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def connect(self):
|
def connect(self):
|
||||||
try:
|
try:
|
||||||
if self.use_ssl:
|
if self.use_ssl:
|
||||||
|
|
@ -46,7 +47,7 @@ class IMAPConnection:
|
||||||
self.connection = imaplib.IMAP4_SSL(self.server, self.port, ssl_context=context)
|
self.connection = imaplib.IMAP4_SSL(self.server, self.port, ssl_context=context)
|
||||||
else:
|
else:
|
||||||
self.connection = imaplib.IMAP4(self.server, self.port)
|
self.connection = imaplib.IMAP4(self.server, self.port)
|
||||||
|
|
||||||
self.connection.sock.settimeout(self.timeout)
|
self.connection.sock.settimeout(self.timeout)
|
||||||
self.connection.login(self.email, self.password)
|
self.connection.login(self.email, self.password)
|
||||||
self.logger.info(f"Connected to {self.server} as {self.email}")
|
self.logger.info(f"Connected to {self.server} as {self.email}")
|
||||||
|
|
@ -54,7 +55,7 @@ class IMAPConnection:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Failed to connect to {self.server}: {e}")
|
self.logger.error(f"Failed to connect to {self.server}: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def disconnect(self):
|
def disconnect(self):
|
||||||
if self.connection:
|
if self.connection:
|
||||||
try:
|
try:
|
||||||
|
|
@ -63,7 +64,7 @@ class IMAPConnection:
|
||||||
self.logger.info(f"Disconnected from {self.server}")
|
self.logger.info(f"Disconnected from {self.server}")
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def get_folders(self):
|
def get_folders(self):
|
||||||
try:
|
try:
|
||||||
status, folders = self.connection.list()
|
status, folders = self.connection.list()
|
||||||
|
|
@ -78,7 +79,7 @@ class IMAPConnection:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error getting folders: {e}")
|
self.logger.error(f"Error getting folders: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def select_folder(self, folder):
|
def select_folder(self, folder):
|
||||||
try:
|
try:
|
||||||
status, response = self.connection.select(f'"{folder}"')
|
status, response = self.connection.select(f'"{folder}"')
|
||||||
|
|
@ -88,7 +89,7 @@ class IMAPConnection:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error selecting folder '{folder}': {e}")
|
self.logger.error(f"Error selecting folder '{folder}': {e}")
|
||||||
return False, 0
|
return False, 0
|
||||||
|
|
||||||
def get_message_ids(self):
|
def get_message_ids(self):
|
||||||
try:
|
try:
|
||||||
status, messages = self.connection.search(None, 'ALL')
|
status, messages = self.connection.search(None, 'ALL')
|
||||||
|
|
@ -97,7 +98,7 @@ class IMAPConnection:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error getting message IDs: {e}")
|
self.logger.error(f"Error getting message IDs: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def fetch_message(self, msg_id):
|
def fetch_message(self, msg_id):
|
||||||
try:
|
try:
|
||||||
status, msg_data = self.connection.fetch(msg_id, '(RFC822)')
|
status, msg_data = self.connection.fetch(msg_id, '(RFC822)')
|
||||||
|
|
@ -107,7 +108,7 @@ class IMAPConnection:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error fetching message {msg_id}: {e}")
|
self.logger.error(f"Error fetching message {msg_id}: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def append_message(self, folder, message, flags='', date_time=None):
|
def append_message(self, folder, message, flags='', date_time=None):
|
||||||
try:
|
try:
|
||||||
self.create_folder(folder)
|
self.create_folder(folder)
|
||||||
|
|
@ -120,7 +121,7 @@ class IMAPConnection:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error appending message to folder '{folder}': {e}")
|
self.logger.error(f"Error appending message to folder '{folder}': {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def create_folder(self, folder):
|
def create_folder(self, folder):
|
||||||
try:
|
try:
|
||||||
status, response = self.connection.create(f'"{folder}"')
|
status, response = self.connection.create(f'"{folder}"')
|
||||||
|
|
@ -137,15 +138,22 @@ class EmailMigrator:
|
||||||
self.batch_size = int(config.get('BATCH_SIZE', '50'))
|
self.batch_size = int(config.get('BATCH_SIZE', '50'))
|
||||||
self.preserve_flags = config.get('PRESERVE_FLAGS', 'True').lower() == 'true'
|
self.preserve_flags = config.get('PRESERVE_FLAGS', 'True').lower() == 'true'
|
||||||
self.preserve_dates = config.get('PRESERVE_DATES', 'True').lower() == 'true'
|
self.preserve_dates = config.get('PRESERVE_DATES', 'True').lower() == 'true'
|
||||||
|
|
||||||
|
# New: Import folder configuration
|
||||||
|
self.import_folder_name = config.get('IMPORT_FOLDER_NAME', '').strip()
|
||||||
|
if not self.import_folder_name:
|
||||||
|
self.import_folder_name = None
|
||||||
|
|
||||||
|
self.logger.info(f"Import folder configuration: {'All emails will be imported to subfolders within \"' + self.import_folder_name + '\"' if self.import_folder_name else 'All emails will be imported directly to INBOX'}")
|
||||||
|
|
||||||
include_str = config.get('INCLUDE_FOLDERS', '')
|
include_str = config.get('INCLUDE_FOLDERS', '')
|
||||||
exclude_str = config.get('EXCLUDE_FOLDERS', '')
|
exclude_str = config.get('EXCLUDE_FOLDERS', '')
|
||||||
|
|
||||||
self.include_folders = [f.strip() for f in include_str.split(',') if f.strip()] if include_str else []
|
self.include_folders = [f.strip() for f in include_str.split(',') if f.strip()] if include_str else []
|
||||||
self.exclude_folders = [f.strip() for f in exclude_str.split(',') if f.strip()] if exclude_str else []
|
self.exclude_folders = [f.strip() for f in exclude_str.split(',') if f.strip()] if exclude_str else []
|
||||||
|
|
||||||
timeout = int(config.get('IMAP_TIMEOUT', '60'))
|
timeout = int(config.get('IMAP_TIMEOUT', '60'))
|
||||||
|
|
||||||
self.source = IMAPConnection(
|
self.source = IMAPConnection(
|
||||||
config['SOURCE_IMAP_SERVER'],
|
config['SOURCE_IMAP_SERVER'],
|
||||||
int(config['SOURCE_IMAP_PORT']),
|
int(config['SOURCE_IMAP_PORT']),
|
||||||
|
|
@ -154,7 +162,7 @@ class EmailMigrator:
|
||||||
config.get('SOURCE_IMAP_USE_SSL', 'True').lower() == 'true',
|
config.get('SOURCE_IMAP_USE_SSL', 'True').lower() == 'true',
|
||||||
timeout
|
timeout
|
||||||
)
|
)
|
||||||
|
|
||||||
self.destination = IMAPConnection(
|
self.destination = IMAPConnection(
|
||||||
config['DEST_IMAP_SERVER'],
|
config['DEST_IMAP_SERVER'],
|
||||||
int(config['DEST_IMAP_PORT']),
|
int(config['DEST_IMAP_PORT']),
|
||||||
|
|
@ -163,24 +171,41 @@ class EmailMigrator:
|
||||||
config.get('DEST_IMAP_USE_SSL', 'True').lower() == 'true',
|
config.get('DEST_IMAP_USE_SSL', 'True').lower() == 'true',
|
||||||
timeout
|
timeout
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def get_destination_folder(self, source_folder):
|
||||||
|
"""
|
||||||
|
Determine the destination folder based on the import configuration.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source_folder (str): Original folder name from source
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: Destination folder name
|
||||||
|
"""
|
||||||
|
if self.import_folder_name:
|
||||||
|
# Import into subfolders within the specified import folder
|
||||||
|
return f"{self.import_folder_name}/{source_folder}"
|
||||||
|
else:
|
||||||
|
# Import all emails directly to INBOX
|
||||||
|
return "INBOX"
|
||||||
|
|
||||||
def should_process_folder(self, folder):
|
def should_process_folder(self, folder):
|
||||||
if self.include_folders and folder not in self.include_folders:
|
if self.include_folders and folder not in self.include_folders:
|
||||||
return False
|
return False
|
||||||
if self.exclude_folders and folder in self.exclude_folders:
|
if self.exclude_folders and folder in self.exclude_folders:
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def download_emails_from_folder(self, folder):
|
def download_emails_from_folder(self, folder):
|
||||||
self.logger.info(f"Downloading emails from folder: {folder}")
|
self.logger.info(f"Downloading emails from folder: {folder}")
|
||||||
success, count = self.source.select_folder(folder)
|
success, count = self.source.select_folder(folder)
|
||||||
if not success:
|
if not success:
|
||||||
self.logger.error(f"Failed to select source folder: {folder}")
|
self.logger.error(f"Failed to select source folder: {folder}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
message_ids = self.source.get_message_ids()
|
message_ids = self.source.get_message_ids()
|
||||||
self.logger.info(f"Found {len(message_ids)} messages in folder: {folder}")
|
self.logger.info(f"Found {len(message_ids)} messages in folder: {folder}")
|
||||||
|
|
||||||
emails = []
|
emails = []
|
||||||
for i, msg_id in enumerate(message_ids, 1):
|
for i, msg_id in enumerate(message_ids, 1):
|
||||||
try:
|
try:
|
||||||
|
|
@ -191,129 +216,139 @@ class EmailMigrator:
|
||||||
'folder': folder,
|
'folder': folder,
|
||||||
'original_id': msg_id.decode() if isinstance(msg_id, bytes) else str(msg_id)
|
'original_id': msg_id.decode() if isinstance(msg_id, bytes) else str(msg_id)
|
||||||
})
|
})
|
||||||
|
|
||||||
if i % self.batch_size == 0:
|
if i % self.batch_size == 0:
|
||||||
self.logger.info(f"Downloaded {i}/{len(message_ids)} messages from {folder}")
|
self.logger.info(f"Downloaded {i}/{len(message_ids)} messages from {folder}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error downloading message {msg_id} from {folder}: {e}")
|
self.logger.error(f"Error downloading message {msg_id} from {folder}: {e}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self.logger.info(f"Successfully downloaded {len(emails)} messages from {folder}")
|
self.logger.info(f"Successfully downloaded {len(emails)} messages from {folder}")
|
||||||
return emails
|
return emails
|
||||||
|
|
||||||
def upload_emails_to_folder(self, emails, folder):
|
def upload_emails_to_folder(self, emails, destination_folder):
|
||||||
self.logger.info(f"Uploading {len(emails)} emails to folder: {folder}")
|
self.logger.info(f"Uploading {len(emails)} emails to folder: {destination_folder}")
|
||||||
uploaded = 0
|
uploaded = 0
|
||||||
for i, email_data in enumerate(emails, 1):
|
for i, email_data in enumerate(emails, 1):
|
||||||
try:
|
try:
|
||||||
message = email_data['message']
|
message = email_data['message']
|
||||||
flags = '\\Seen' if self.preserve_flags else ''
|
flags = '\\Seen' if self.preserve_flags else ''
|
||||||
date_obj = None
|
date_obj = None
|
||||||
|
|
||||||
if self.preserve_dates and message.get('Date'):
|
if self.preserve_dates and message.get('Date'):
|
||||||
try:
|
try:
|
||||||
date_obj = email.utils.parsedate_to_datetime(message['Date'])
|
date_obj = email.utils.parsedate_to_datetime(message['Date'])
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if self.destination.append_message(folder, message, flags, date_obj):
|
if self.destination.append_message(destination_folder, message, flags, date_obj):
|
||||||
uploaded += 1
|
uploaded += 1
|
||||||
|
|
||||||
if i % self.batch_size == 0:
|
if i % self.batch_size == 0:
|
||||||
self.logger.info(f"Uploaded {i}/{len(emails)} messages to {folder}")
|
self.logger.info(f"Uploaded {i}/{len(emails)} messages to {destination_folder}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error uploading message to {folder}: {e}")
|
self.logger.error(f"Error uploading message to {destination_folder}: {e}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self.logger.info(f"Successfully uploaded {uploaded}/{len(emails)} messages to {folder}")
|
self.logger.info(f"Successfully uploaded {uploaded}/{len(emails)} messages to {destination_folder}")
|
||||||
return uploaded
|
return uploaded
|
||||||
|
|
||||||
def migrate_folder(self, folder):
|
def migrate_folder(self, source_folder):
|
||||||
stats = {'downloaded': 0, 'uploaded': 0}
|
stats = {'downloaded': 0, 'uploaded': 0}
|
||||||
|
|
||||||
if not self.should_process_folder(folder):
|
if not self.should_process_folder(source_folder):
|
||||||
self.logger.info(f"Skipping folder: {folder} (filtered)")
|
self.logger.info(f"Skipping folder: {source_folder} (filtered)")
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
try:
|
try:
|
||||||
emails = self.download_emails_from_folder(folder)
|
# Determine destination folder based on configuration
|
||||||
|
destination_folder = self.get_destination_folder(source_folder)
|
||||||
|
self.logger.info(f"Migrating '{source_folder}' -> '{destination_folder}'")
|
||||||
|
|
||||||
|
emails = self.download_emails_from_folder(source_folder)
|
||||||
stats['downloaded'] = len(emails)
|
stats['downloaded'] = len(emails)
|
||||||
|
|
||||||
if emails:
|
if emails:
|
||||||
stats['uploaded'] = self.upload_emails_to_folder(emails, folder)
|
stats['uploaded'] = self.upload_emails_to_folder(emails, destination_folder)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error migrating folder {folder}: {e}")
|
self.logger.error(f"Error migrating folder {source_folder}: {e}")
|
||||||
|
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
def run_migration(self):
|
def run_migration(self):
|
||||||
self.logger.info("Starting email migration...")
|
self.logger.info("Starting email migration...")
|
||||||
total_stats = {'folders_processed': 0, 'total_downloaded': 0, 'total_uploaded': 0, 'errors': 0}
|
total_stats = {'folders_processed': 0, 'total_downloaded': 0, 'total_uploaded': 0, 'errors': 0}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not self.source.connect():
|
if not self.source.connect():
|
||||||
self.logger.error("Failed to connect to source server")
|
self.logger.error("Failed to connect to source server")
|
||||||
return total_stats
|
return total_stats
|
||||||
|
|
||||||
if not self.destination.connect():
|
if not self.destination.connect():
|
||||||
self.logger.error("Failed to connect to destination server")
|
self.logger.error("Failed to connect to destination server")
|
||||||
return total_stats
|
return total_stats
|
||||||
|
|
||||||
folders = self.source.get_folders()
|
folders = self.source.get_folders()
|
||||||
self.logger.info(f"Found {len(folders)} folders to process")
|
self.logger.info(f"Found {len(folders)} folders to process")
|
||||||
|
|
||||||
|
# Create the main import folder if specified
|
||||||
|
if self.import_folder_name:
|
||||||
|
self.logger.info(f"Creating main import folder: {self.import_folder_name}")
|
||||||
|
self.destination.create_folder(self.import_folder_name)
|
||||||
|
|
||||||
for folder in folders:
|
for folder in folders:
|
||||||
try:
|
try:
|
||||||
self.logger.info(f"Processing folder: {folder}")
|
self.logger.info(f"Processing folder: {folder}")
|
||||||
stats = self.migrate_folder(folder)
|
stats = self.migrate_folder(folder)
|
||||||
|
|
||||||
total_stats['folders_processed'] += 1
|
total_stats['folders_processed'] += 1
|
||||||
total_stats['total_downloaded'] += stats['downloaded']
|
total_stats['total_downloaded'] += stats['downloaded']
|
||||||
total_stats['total_uploaded'] += stats['uploaded']
|
total_stats['total_uploaded'] += stats['uploaded']
|
||||||
|
|
||||||
self.logger.info(f"Folder '{folder}' completed: {stats['downloaded']} downloaded, {stats['uploaded']} uploaded")
|
destination_folder = self.get_destination_folder(folder)
|
||||||
|
self.logger.info(f"Folder '{folder}' -> '{destination_folder}' completed: {stats['downloaded']} downloaded, {stats['uploaded']} uploaded")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error processing folder {folder}: {e}")
|
self.logger.error(f"Error processing folder {folder}: {e}")
|
||||||
total_stats['errors'] += 1
|
total_stats['errors'] += 1
|
||||||
finally:
|
finally:
|
||||||
self.source.disconnect()
|
self.source.disconnect()
|
||||||
self.destination.disconnect()
|
self.destination.disconnect()
|
||||||
|
|
||||||
return total_stats
|
return total_stats
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
print("Email Migration Script")
|
print("Email Migration Script")
|
||||||
print("=" * 50)
|
print("=" * 50)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
config = load_env_file()
|
config = load_env_file()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error loading .env file: {e}")
|
print(f"Error loading .env file: {e}")
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
required_vars = ['SOURCE_IMAP_SERVER', 'SOURCE_EMAIL', 'SOURCE_PASSWORD',
|
required_vars = ['SOURCE_IMAP_SERVER', 'SOURCE_EMAIL', 'SOURCE_PASSWORD',
|
||||||
'DEST_IMAP_SERVER', 'DEST_EMAIL', 'DEST_PASSWORD']
|
'DEST_IMAP_SERVER', 'DEST_EMAIL', 'DEST_PASSWORD']
|
||||||
|
|
||||||
missing_vars = [var for var in required_vars if not config.get(var)]
|
missing_vars = [var for var in required_vars if not config.get(var)]
|
||||||
if missing_vars:
|
if missing_vars:
|
||||||
print(f"Error: Missing required environment variables: {', '.join(missing_vars)}")
|
print(f"Error: Missing required environment variables: {', '.join(missing_vars)}")
|
||||||
print("Please check your .env file.")
|
print("Please check your .env file.")
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
migrator = EmailMigrator(config)
|
migrator = EmailMigrator(config)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
stats = migrator.run_migration()
|
stats = migrator.run_migration()
|
||||||
|
|
||||||
print("\nMigration completed!")
|
print("\nMigration completed!")
|
||||||
print(f"Folders processed: {stats['folders_processed']}")
|
print(f"Folders processed: {stats['folders_processed']}")
|
||||||
print(f"Total emails downloaded: {stats['total_downloaded']}")
|
print(f"Total emails downloaded: {stats['total_downloaded']}")
|
||||||
print(f"Total emails uploaded: {stats['total_uploaded']}")
|
print(f"Total emails uploaded: {stats['total_uploaded']}")
|
||||||
print(f"Errors encountered: {stats['errors']}")
|
print(f"Errors encountered: {stats['errors']}")
|
||||||
|
|
||||||
if stats['errors'] > 0:
|
if stats['errors'] > 0:
|
||||||
print("\nCheck the log file 'email_migration.log' for error details.")
|
print("\nCheck the log file 'email_migration.log' for error details.")
|
||||||
|
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
print("\nMigration interrupted by user.")
|
print("\nMigration interrupted by user.")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue