#!/usr/bin/env python3
"""
File System → Odoo Sync Service
Monitors Customer Docs folders for new/modified credit applications
Automatically creates/updates Odoo leads with extracted data
"""

import os
import sys
import json
import time
import logging
import hashlib
from datetime import datetime
from pathlib import Path
from typing import Dict, Any, Optional
import xmlrpc.client
import base64
import re

from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

# PDF processing imports
try:
    import fitz  # PyMuPDF
    PYMUPDF_AVAILABLE = True
except ImportError:
    PYMUPDF_AVAILABLE = False

try:
    from PyPDF2 import PdfReader
    PYPDF2_AVAILABLE = True
except ImportError:
    PYPDF2_AVAILABLE = False

# Configuration
CUSTOMER_DOCS_BASE = r"D:\Customer Docs"
ODOO_CONFIG_FILE = "./odoo_config.json"
WATCHER_CONFIG_FILE = "./file_watcher_config.json"
PROCESSED_LOG_FILE = "./processed_files.json"


def format_customer_name(first_name: str, last_name: str) -> str:
    """Format customer name like ADF importer does"""
    first_name = first_name or ''
    last_name = last_name or ''
    return f"{first_name} {last_name}".strip() or "Unknown"


# Set up logging
logging.basicConfig(
    level=logging.DEBUG,  # Enable debug messages for field mapping analysis
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('file_watcher.log'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)


class ProcessedFileTracker:
    """Tracks processed files to prevent duplicate processing"""
    
    def __init__(self, log_file=PROCESSED_LOG_FILE):
        self.log_file = Path(log_file)
        self.processed = self._load_log()
        self.MIN_REPROCESS_TIME = 300  # Minimum 5 minutes between reprocessing same file
    
    def _load_log(self) -> Dict:
        """Load the processed files log"""
        if self.log_file.exists():
            try:
                with open(self.log_file, 'r') as f:
                    data = json.load(f)
                    # Separate monitored folders from processed files
                    if '_monitored_folders' in data:
                        self._monitored_folders = data.pop('_monitored_folders')
                    else:
                        self._monitored_folders = {}
                    return data
            except Exception as e:
                logger.error(f"Error loading processed log: {e}")
                return {}
        return {}
    
    def _save_log(self):
        """Save the processed files log"""
        try:
            # Combine processed files and monitored folders
            save_data = dict(self.processed)
            if hasattr(self, '_monitored_folders') and self._monitored_folders:
                save_data['_monitored_folders'] = self._monitored_folders
            
            with open(self.log_file, 'w') as f:
                json.dump(save_data, f, indent=2)
        except Exception as e:
            logger.error(f"Error saving processed log: {e}")
    
    def is_processed(self, file_path: str, mtime: float) -> bool:
        """Check if a file has already been processed"""
        key = self._get_file_key(file_path)
        if key in self.processed:
            # Check if modification time is different
            last_mtime = self.processed[key].get('mtime', 0)
            last_processed_time = self.processed[key].get('processed_at', '')
            
            # If file was processed recently (within MIN_REPROCESS_TIME), skip it
            if last_processed_time:
                try:
                    from datetime import datetime
                    last_processed_dt = datetime.fromisoformat(last_processed_time)
                    time_since_processed = (datetime.now() - last_processed_dt).total_seconds()
                    if time_since_processed < self.MIN_REPROCESS_TIME:
                        logger.info(f"File processed {time_since_processed:.0f} seconds ago, skipping (min: {self.MIN_REPROCESS_TIME}s)")
                        return True
                except:
                    pass
            
            # Check content hash if available
            if 'content_hash' in self.processed[key]:
                current_hash = self._get_file_hash(file_path)
                if current_hash == self.processed[key]['content_hash']:
                    logger.info(f"File content unchanged (hash match), skipping")
                    return True
            
            return abs(mtime - last_mtime) < 1  # Within 1 second = same file
        return False
    
    def mark_processed(self, file_path: str, mtime: float, lead_id: Optional[int] = None):
        """Mark a file as processed"""
        key = self._get_file_key(file_path)
        
        # Keep existing lead_id if we're updating an existing entry
        existing_lead_id = None
        if key in self.processed:
            existing_lead_id = self.processed[key].get('lead_id')
            logger.info(f"Updating existing processed entry (was lead #{existing_lead_id})")
        
        # Calculate file hash for content comparison
        content_hash = self._get_file_hash(file_path)
        
        self.processed[key] = {
            'mtime': mtime,
            'processed_at': datetime.now().isoformat(),
            'lead_id': lead_id or existing_lead_id,  # Preserve existing lead_id if no new one
            'path': str(file_path),
            'content_hash': content_hash
        }
        
        # If this is a credit app with a lead_id, track the folder for ongoing monitoring
        if lead_id and 'credit app' in Path(file_path).name.lower():
            folder_path = str(Path(file_path).parent)
            self._add_monitored_folder(folder_path, lead_id)
        
        self._save_log()
    
    def _get_file_key(self, file_path: str) -> str:
        """Generate a unique key for a file"""
        # Use normalized path as key
        return str(Path(file_path).resolve())
    
    def _get_file_hash(self, file_path: str) -> str:
        """Calculate hash of file content for duplicate detection"""
        try:
            with open(file_path, 'rb') as f:
                # Read first 1MB and last 1MB for large files
                file_size = Path(file_path).stat().st_size
                if file_size > 2 * 1024 * 1024:  # > 2MB
                    content = f.read(1024 * 1024)  # First 1MB
                    f.seek(-1024 * 1024, 2)  # Last 1MB
                    content += f.read()
                else:
                    content = f.read()
                return hashlib.sha256(content).hexdigest()
        except Exception as e:
            logger.warning(f"Could not calculate file hash: {e}")
            return ""
    
    def get_lead_id_for_folder(self, folder_path: str) -> Optional[int]:
        """Get the lead_id for a monitored folder"""
        try:
            monitored_folders = self.processed.get('_monitored_folders', {})
            folder_info = monitored_folders.get(folder_path, {})
            return folder_info.get('lead_id')
        except Exception as e:
            logger.warning(f"Error getting lead_id for folder: {e}")
            return None


    def get_monitored_folders(self) -> Dict[str, Dict]:
        """Get all monitored folders"""
        return self.processed.get('_monitored_folders', {})
    
    def _add_monitored_folder(self, folder_path: str, lead_id: int):
        """Add a folder to the monitored folders list"""
        if not hasattr(self, '_monitored_folders') or not self._monitored_folders:
            self._monitored_folders = {}
        
        self._monitored_folders[folder_path] = {
            'lead_id': lead_id,
            'added_at': datetime.now().isoformat(),
            'active': True
        }
        logger.info(f"Now monitoring folder for additional files: {folder_path} (Lead #{lead_id})")
    
    def get_lead_id_for_folder(self, folder_path: str) -> Optional[int]:
        """Get the lead ID associated with a folder"""
        folder_path = str(Path(folder_path).resolve())
        
        if hasattr(self, '_monitored_folders') and self._monitored_folders:
            folder_info = self._monitored_folders.get(folder_path)
            if folder_info and folder_info.get('active'):
                return folder_info.get('lead_id')
        
        return None
    
    def get_monitored_folders(self) -> Dict[str, Dict]:
        """Get all actively monitored folders"""
        if not hasattr(self, '_monitored_folders') or not self._monitored_folders:
            return {}
        
        return {path: info for path, info in self._monitored_folders.items() 
                if info.get('active', True)}
    
    def deactivate_folder_monitoring(self, folder_path: str):
        """Deactivate monitoring for a specific folder"""
        folder_path = str(Path(folder_path).resolve())
        
        if hasattr(self, '_monitored_folders') and folder_path in self._monitored_folders:
            self._monitored_folders[folder_path]['active'] = False
            self._save_log()


class OdooClient:
    """Client for connecting to Odoo via XML-RPC API"""
    
    def __init__(self, config: Dict):
        self.url = config['url'].rstrip('/')
        self.database = config['database']
        self.username = config['username']
        self.password = config['password']
        self.uid = None
        self.common = None
        self.models = None
        self.connected = False
    
    def connect(self) -> bool:
        """Connect and authenticate with Odoo"""
        try:
            self.common = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/common')
            self.uid = self.common.authenticate(self.database, self.username, self.password, {})
            
            if not self.uid:
                logger.error("Odoo authentication failed")
                return False
            
            self.models = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/object')
            self.connected = True
            logger.info(f"Connected to Odoo (user ID: {self.uid})")
            return True
            
        except Exception as e:
            logger.error(f"Failed to connect to Odoo: {e}")
            self.connected = False
            return False
    
    def search_leads_by_phone(self, phone: str) -> list:
        """Search for leads by phone number"""
        if not self.connected:
            return []
        
        try:
            # Normalize phone to last 10 digits
            phone_digits = re.sub(r'\D', '', phone)
            phone_normalized = phone_digits[-10:] if len(phone_digits) >= 10 else phone_digits
            
            # Search for leads with matching phone
            domain = ['|', ('phone', 'like', phone_normalized), ('mobile', 'like', phone_normalized)]
            
            lead_ids = self.models.execute_kw(
                self.database, self.uid, self.password,
                'crm.lead', 'search', [domain],
                {'limit': 10, 'order': 'create_date desc'}
            )
            
            if lead_ids:
                # Get lead details
                leads = self.models.execute_kw(
                    self.database, self.uid, self.password,
                    'crm.lead', 'read', [lead_ids],
                    {'fields': ['name', 'phone', 'mobile', 'tag_ids', 'create_date']}
                )
                
                # Get tag names
                for lead in leads:
                    if lead.get('tag_ids'):
                        tags = self.models.execute_kw(
                            self.database, self.uid, self.password,
                            'crm.tag', 'read', [lead['tag_ids']],
                            {'fields': ['name']}
                        )
                        lead['tags'] = [tag['name'] for tag in tags]
                    else:
                        lead['tags'] = []
                
                return leads
                
        except Exception as e:
            logger.error(f"Error searching leads: {e}")
        
        return []
    
    def create_lead(self, data: Dict[str, Any]) -> Optional[int]:
        """Create a new lead in Odoo with all credit application data"""
        if not self.connected:
            return None
        
        try:
            # Build customer name from extracted data using ADF formatting
            first_name = data.get('first_name', '')
            last_name = data.get('last_name', '')
            customer_name = format_customer_name(first_name, last_name)
            
            if customer_name == "Unknown":
                # Fallback to contact_name if first/last name not found
                customer_name = data.get('contact_name', 'Unknown Customer')
            
            # Check for vehicle information to build title
            vehicle_year = data.get('vehicle_year', '')
            vehicle_make = data.get('vehicle_make', '')
            vehicle_model = data.get('vehicle_model', '')
            
            if vehicle_year and vehicle_make and vehicle_model:
                opportunity_name = f"{vehicle_year} {vehicle_make} {vehicle_model}".strip()
                logger.info(f"Using vehicle title: {opportunity_name}")
            elif vehicle_make and vehicle_model:  # Missing year
                opportunity_name = f"{vehicle_make} {vehicle_model}".strip()
                logger.info(f"Using vehicle title (no year): {opportunity_name}")
            else:
                opportunity_name = customer_name
                logger.info(f"Using customer name: {opportunity_name}")
            
            # Build comprehensive lead values with all extracted data
            lead_vals = {
                'name': opportunity_name,
                'contact_name': customer_name,
                'x_customer_name': customer_name,  # Add missing x_customer_name field like ADF importer
                # Don't set partner_name to avoid duplicate contacts when converting to opportunity
                'phone': data.get('mobile_phone', data.get('home_phone', data.get('phone', ''))),
                'mobile': data.get('mobile_phone', ''),
                'email_from': data.get('email', ''),
                'street': data.get('address', ''),
                'street2': data.get('address2', ''),
                'city': data.get('city', ''),
                'state_id': self._get_state_id(data.get('state', '')),
                'zip': data.get('zip', ''),
                'type': 'lead',  # Set as lead type
                'x_lead_type': 'credit_app',  # Credit application lead type
                'description': f"Credit application processed from file system on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
            }
            
            # Helper function to convert residence type values
            def convert_residence_type(value):
                if not value:
                    return ''
                value_lower = str(value).lower()
                if 'rent' in value_lower or value_lower == 'yes':
                    return 'rent'
                elif 'own' in value_lower or 'mortgage' in value_lower or value_lower == 'no':
                    return 'own'
                else:
                    return 'other'

            # Helper function to combine years and months into time strings
            def combine_time_fields(years_value, months_value):
                """Combine separate year and month values into a time string"""
                if not years_value and not months_value:
                    return ''
                
                years = str(years_value).strip() if years_value else '0'
                months = str(months_value).strip() if months_value else '0'
                
                # Handle numeric values
                try:
                    years_num = int(float(years)) if years and years != '0' else 0
                    months_num = int(float(months)) if months and months != '0' else 0
                except (ValueError, TypeError):
                    years_num = 0
                    months_num = 0
                
                # Build time string
                parts = []
                if years_num > 0:
                    parts.append(f"{years_num} year{'s' if years_num != 1 else ''}")
                if months_num > 0:
                    parts.append(f"{months_num} month{'s' if months_num != 1 else ''}")
                
                return ' '.join(parts) if parts else ''

            # Combine time fields for applicant
            time_at_residence_combined = combine_time_fields(
                data.get('time_at_residence_years'), 
                data.get('time_at_residence_months')
            )
            time_on_job_combined = combine_time_fields(
                data.get('time_on_job_years'), 
                data.get('time_on_job_months')
            )
            
            # Combine time fields for co-applicant
            co_time_at_residence_combined = combine_time_fields(
                data.get('co_time_at_residence_years'), 
                data.get('co_time_at_residence_months')
            )
            co_time_on_job_combined = combine_time_fields(
                data.get('co_time_on_job_years'), 
                data.get('co_time_on_job_months')
            )
            
            # Convert date formats from MM/DD/YYYY to YYYY-MM-DD for Odoo
            def convert_date_format(date_str):
                if not date_str:
                    return ''
                try:
                    # Try MM/DD/YYYY format first
                    if '/' in date_str:
                        parts = date_str.split('/')
                        if len(parts) == 3:
                            month, day, year = parts
                            # Ensure proper zero-padding for month and day
                            return f"{year}-{month.zfill(2)}-{day.zfill(2)}"
                    return date_str
                except Exception as e:
                    logger.warning(f"Error converting date format: {date_str} - {e}")
                    return ''
            
            # Extract numeric value from text for float fields
            def extract_numeric(value):
                if not value:
                    return ''
                if isinstance(value, (int, float)):
                    return value
                # Extract first number from string
                import re
                match = re.search(r'[\d,]+\.?\d*', str(value))
                if match:
                    # Remove commas and return
                    return match.group().replace(',', '')
                return ''
            
            dob = convert_date_format(data.get('dob', data.get('date_of_birth', '')))
            co_dob = convert_date_format(data.get('co_dob', ''))
            dl_exp_date = convert_date_format(data.get('dl_exp_date', ''))
            co_dl_exp_date = convert_date_format(data.get('co_dl_exp_date', ''))

            # Add credit application specific fields if they exist in the lead model
            credit_fields = {
                # Personal Information
                'x_first_name': data.get('first_name', ''),
                'x_last_name': data.get('last_name', ''),
                'x_ssn': data.get('ssn', ''),
                'x_date_of_birth': dob,  # Use converted date format
                'x_dl_number': data.get('drivers_license', ''),
                'x_dl_state': data.get('dl_state', ''),
                'x_dl_exp_date': dl_exp_date,  # Use converted date format
                'x_home_phone': data.get('home_phone', ''),
                'x_email_address': data.get('email', ''),
                
                # Address and Housing Information
                'x_address': data.get('address', ''),
                'x_city': data.get('city', ''),
                'x_state': data.get('state', ''),
                'x_zip': data.get('zip', ''),
                'x_residence_type': convert_residence_type(data.get('residence_type', '')),
                'x_rent_mortgage': extract_numeric(data.get('rent_mortgage', '')),
                'x_time_at_residence': time_at_residence_combined or data.get('time_at_residence', ''),
                # 'x_landlord_name': data.get('landlord_name', ''),  # Uncomment after module upgrade
                # 'x_bank_name': data.get('bank_name', ''),  # Uncomment after module upgrade
                
                # Employment Information
                'x_employer': data.get('employer', ''),
                'x_employer_address': data.get('employer_address', ''),
                'x_employer_city': data.get('employer_city', ''),
                'x_employer_state': data.get('employer_state', ''),
                'x_employer_zip': data.get('employer_zip', ''),
                'x_work_phone': data.get('work_phone', ''),
                'x_occupation': data.get('occupation', ''),
                'x_monthly_income': extract_numeric(data.get('monthly_income', '')),
                'x_time_on_job': time_on_job_combined or data.get('time_on_job', ''),
                'x_hourly_pay': extract_numeric(data.get('hourly_rate', '')),
                'x_hours_per_week': extract_numeric(data.get('hours_per_week', '')),
                
                # Vehicle Information
                'x_vehicle_year': data.get('vehicle_year', ''),
                'x_vehicle_make': data.get('vehicle_make', ''),
                'x_vehicle_model': data.get('vehicle_model', ''),
                'x_vehicle_trim': data.get('vehicle_trim', ''),
                'x_vehicle_vin': data.get('vehicle_vin', ''),
                'x_vehicle_mileage': data.get('vehicle_mileage', ''),
                'x_vehicle_price': data.get('list_price', data.get('sale_price', '')),
                'x_vehicle_sale_price': data.get('sale_price', ''),
                'x_vehicle_down_total': data.get('down_payment', ''),
                'x_desired_down_payment': data.get('desired_down_payment', ''),
                
                # Co-Applicant Information (if present)
                'x_co_applicant_first_name': data.get('co_first_name', ''),
                'x_co_applicant_last_name': data.get('co_last_name', ''),
                # 'x_co_applicant_middle_name': data.get('co_middle_name', ''),  # Uncomment after module upgrade
                'x_co_applicant_mobile_phone': data.get('co_mobile_phone', ''),
                'x_co_applicant_home_phone': data.get('co_home_phone', ''),
                'x_co_applicant_email': data.get('co_email', ''),
                'x_co_applicant_ssn': data.get('co_ssn', ''),
                'x_co_applicant_dob': co_dob,  # Use converted date format
                'x_co_applicant_dl_number': data.get('co_drivers_license', ''),
                'x_co_applicant_dl_state': data.get('co_dl_state', ''),
                'x_co_applicant_dl_exp_date': co_dl_exp_date,  # Use converted date format
                
                # Co-Applicant Address Information
                'x_co_applicant_address': data.get('co_address', ''),
                'x_co_applicant_city': data.get('co_city', ''),
                'x_co_applicant_state': data.get('co_state', ''),
                'x_co_applicant_zip': data.get('co_zip', ''),
                'x_co_applicant_residence_type': convert_residence_type(data.get('co_residence_type', '')),
                'x_co_applicant_rent_mortgage': extract_numeric(data.get('co_rent_mortgage', '')),
                'x_co_applicant_time_at_residence': co_time_at_residence_combined or data.get('co_time_at_residence', ''),
                
                # Co-Applicant Employment Information
                'x_co_applicant_employer': data.get('co_employer', ''),
                'x_co_applicant_employer_address': data.get('co_employer_address', ''),
                'x_co_applicant_work_phone': data.get('co_work_phone', ''),
                'x_co_applicant_monthly_income': extract_numeric(data.get('co_monthly_income', '')),
                'x_co_applicant_occupation': data.get('co_occupation', ''),
                'x_co_applicant_time_on_job': co_time_on_job_combined or data.get('co_time_on_job', ''),
            }
            
            # Only add fields that have values
            for field_name, field_value in credit_fields.items():
                if field_value:
                    lead_vals[field_name] = field_value
            
            # Remove empty values from basic fields
            lead_vals = {k: v for k, v in lead_vals.items() if v}
            
            # Create the lead
            lead_id = self.models.execute_kw(
                self.database, self.uid, self.password,
                'crm.lead', 'create', [lead_vals]
            )
            
            logger.info(f"Created new lead ID: {lead_id} with {len(lead_vals)} fields")
            return lead_id
            
        except Exception as e:
            logger.error(f"Error creating lead: {e}")
            # Try with just basic fields if extended fields fail
            try:
                customer_name = format_customer_name(data.get('first_name', ''), data.get('last_name', ''))
                
                # Still try to use vehicle info for title even in fallback
                vehicle_year = data.get('vehicle_year', '')
                vehicle_make = data.get('vehicle_make', '')
                vehicle_model = data.get('vehicle_model', '')
                
                if vehicle_year and vehicle_make and vehicle_model:
                    opportunity_name = f"{vehicle_year} {vehicle_make} {vehicle_model}".strip()
                elif vehicle_make and vehicle_model:
                    opportunity_name = f"{vehicle_make} {vehicle_model}".strip()
                else:
                    opportunity_name = customer_name
                
                basic_vals = {
                    'name': opportunity_name,
                    'contact_name': customer_name,
                    'x_customer_name': customer_name,  # Add missing x_customer_name field like ADF importer
                    # Don't set partner_name to avoid duplicate contacts when converting to opportunity
                    'phone': data.get('mobile_phone', data.get('home_phone', data.get('phone', ''))),
                    'mobile': data.get('mobile_phone', ''),
                    'email_from': data.get('email', ''),
                    'type': 'lead',  # Set as lead type
                    'x_lead_type': 'credit_app',  # Credit application lead type
                    'description': f"Credit application processed (basic fields only) on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
                }
                basic_vals = {k: v for k, v in basic_vals.items() if v}
                
                lead_id = self.models.execute_kw(
                    self.database, self.uid, self.password,
                    'crm.lead', 'create', [basic_vals]
                )
                
                logger.info(f"Created basic lead ID: {lead_id} (extended fields not available)")
                return lead_id
                
            except Exception as e2:
                logger.error(f"Error creating basic lead: {e2}")
                return None
    
    def update_lead(self, lead_id: int, data: Dict[str, Any]) -> bool:
        """Update an existing lead with all credit application data"""
        if not self.connected:
            return False

        try:
            # Convert date formats from MM/DD/YYYY to YYYY-MM-DD for Odoo
            def convert_date_format(date_str):
                if not date_str:
                    return ''
                try:
                    # Try MM/DD/YYYY format first
                    if '/' in date_str:
                        parts = date_str.split('/')
                        if len(parts) == 3:
                            month, day, year = parts
                            # Ensure proper zero-padding for month and day
                            return f"{year}-{month.zfill(2)}-{day.zfill(2)}"
                    return date_str
                except Exception as e:
                    logger.warning(f"Error converting date format: {date_str} - {e}")
                    return ''

            # Extract numeric value from text for float fields
            def extract_numeric(value):
                if not value:
                    return ''
                if isinstance(value, (int, float)):
                    return value
                # Extract first number from string
                import re
                match = re.search(r'[\d,]+\.?\d*', str(value))
                if match:
                    # Remove commas and return
                    return match.group().replace(',', '')
                return ''

            # Convert date fields
            dob = convert_date_format(data.get('dob', data.get('date_of_birth', '')))
            co_dob = convert_date_format(data.get('co_dob', ''))
            dl_exp_date = convert_date_format(data.get('dl_exp_date', ''))
            co_dl_exp_date = convert_date_format(data.get('co_dl_exp_date', ''))

            # Build comprehensive update values
            update_vals = {}
            
            # Basic contact info
            if data.get('first_name') or data.get('last_name'):
                customer_name = format_customer_name(data.get('first_name', ''), data.get('last_name', ''))
                update_vals['contact_name'] = customer_name
                update_vals['x_customer_name'] = customer_name  # Add missing x_customer_name field like ADF importer
                # Don't set partner_name to avoid duplicate contacts when converting to opportunity
            if data.get('mobile_phone'):
                update_vals['mobile'] = data['mobile_phone']
            if data.get('email'):
                update_vals['email_from'] = data['email']
            if data.get('address'):
                update_vals['street'] = data['address']
            if data.get('address2'):
                update_vals['street2'] = data['address2']
            if data.get('city'):
                update_vals['city'] = data['city']
            if data.get('state'):
                state_id = self._get_state_id(data['state'])
                if state_id:
                    update_vals['state_id'] = state_id
            if data.get('zip'):
                update_vals['zip'] = data['zip']
            
            # Helper function to convert residence type values
            def convert_residence_type(value):
                if not value:
                    return ''
                value_lower = str(value).lower()
                if 'rent' in value_lower or value_lower == 'yes':
                    return 'rent'
                elif 'own' in value_lower or 'mortgage' in value_lower or value_lower == 'no':
                    return 'own'
                else:
                    return 'other'

            # Helper function to combine years and months into time strings
            def combine_time_fields(years_value, months_value):
                """Combine separate year and month values into a time string"""
                if not years_value and not months_value:
                    return ''
                
                years = str(years_value).strip() if years_value else '0'
                months = str(months_value).strip() if months_value else '0'
                
                # Handle numeric values
                try:
                    years_num = int(float(years)) if years and years != '0' else 0
                    months_num = int(float(months)) if months and months != '0' else 0
                except (ValueError, TypeError):
                    years_num = 0
                    months_num = 0
                
                # Build time string
                parts = []
                if years_num > 0:
                    parts.append(f"{years_num} year{'s' if years_num != 1 else ''}")
                if months_num > 0:
                    parts.append(f"{months_num} month{'s' if months_num != 1 else ''}")
                
                return ' '.join(parts) if parts else ''

            # Combine time fields for applicant
            time_at_residence_combined = combine_time_fields(
                data.get('time_at_residence_years'), 
                data.get('time_at_residence_months')
            )
            time_on_job_combined = combine_time_fields(
                data.get('time_on_job_years'), 
                data.get('time_on_job_months')
            )
            
            # Combine time fields for co-applicant
            co_time_at_residence_combined = combine_time_fields(
                data.get('co_time_at_residence_years'), 
                data.get('co_time_at_residence_months')
            )
            co_time_on_job_combined = combine_time_fields(
                data.get('co_time_on_job_years'), 
                data.get('co_time_on_job_months')
            )

            # Credit application specific fields
            credit_fields = {
                # Personal Information
                'x_first_name': data.get('first_name', ''),
                'x_last_name': data.get('last_name', ''),
                'x_ssn': data.get('ssn', ''),
                'x_date_of_birth': dob,  # Use converted date format
                'x_dl_number': data.get('drivers_license', ''),
                'x_dl_state': data.get('dl_state', ''),
                'x_dl_exp_date': dl_exp_date,  # Use converted date format
                'x_home_phone': data.get('home_phone', ''),
                'x_email_address': data.get('email', ''),
                
                # Address and Housing Information  
                'x_address': data.get('address', ''),
                'x_city': data.get('city', ''),
                'x_state': data.get('state', ''),
                'x_zip': data.get('zip', ''),
                'x_residence_type': convert_residence_type(data.get('residence_type', '')),
                'x_rent_mortgage': extract_numeric(data.get('rent_mortgage', '')),
                'x_time_at_residence': time_at_residence_combined or data.get('time_at_residence', ''),
                
                # Employment Information
                'x_employer': data.get('employer', ''),
                'x_employer_address': data.get('employer_address', ''),
                'x_employer_city': data.get('employer_city', ''),
                'x_employer_state': data.get('employer_state', ''),
                'x_employer_zip': data.get('employer_zip', ''),
                'x_work_phone': data.get('work_phone', ''),
                'x_occupation': data.get('occupation', ''),
                'x_monthly_income': extract_numeric(data.get('monthly_income', '')),
                'x_time_on_job': time_on_job_combined or data.get('time_on_job', ''),
                'x_hourly_pay': extract_numeric(data.get('hourly_rate', '')),
                'x_hours_per_week': extract_numeric(data.get('hours_per_week', '')),
                
                # Vehicle Information
                'x_vehicle_year': data.get('vehicle_year', ''),
                'x_vehicle_make': data.get('vehicle_make', ''),
                'x_vehicle_model': data.get('vehicle_model', ''),
                'x_vehicle_trim': data.get('vehicle_trim', ''),
                'x_vehicle_vin': data.get('vehicle_vin', ''),
                'x_vehicle_mileage': data.get('vehicle_mileage', ''),
                'x_vehicle_price': data.get('list_price', data.get('sale_price', '')),
                'x_vehicle_sale_price': data.get('sale_price', ''),
                'x_vehicle_down_total': data.get('down_payment', ''),
                'x_desired_down_payment': data.get('desired_down_payment', ''),
                
                # Co-Applicant Information
                'x_co_applicant_first_name': data.get('co_first_name', ''),
                'x_co_applicant_last_name': data.get('co_last_name', ''),
                # 'x_co_applicant_middle_name': data.get('co_middle_name', ''),  # Uncomment after module upgrade
                'x_co_applicant_mobile_phone': data.get('co_mobile_phone', ''),
                'x_co_applicant_home_phone': data.get('co_home_phone', ''),
                'x_co_applicant_email': data.get('co_email', ''),
                'x_co_applicant_ssn': data.get('co_ssn', ''),
                'x_co_applicant_dob': co_dob,  # Use converted date format
                'x_co_applicant_dl_number': data.get('co_drivers_license', ''),
                'x_co_applicant_dl_state': data.get('co_dl_state', ''),
                'x_co_applicant_dl_exp_date': co_dl_exp_date,  # Use converted date format
                
                # Co-Applicant Address Information
                'x_co_applicant_address': data.get('co_address', ''),
                'x_co_applicant_city': data.get('co_city', ''),
                'x_co_applicant_state': data.get('co_state', ''),
                'x_co_applicant_zip': data.get('co_zip', ''),
                'x_co_applicant_residence_type': convert_residence_type(data.get('co_residence_type', '')),
                'x_co_applicant_rent_mortgage': extract_numeric(data.get('co_rent_mortgage', '')),
                'x_co_applicant_time_at_residence': co_time_at_residence_combined or data.get('co_time_at_residence', ''),
                
                # Co-Applicant Employment Information
                'x_co_applicant_employer': data.get('co_employer', ''),
                'x_co_applicant_employer_address': data.get('co_employer_address', ''),
                'x_co_applicant_work_phone': data.get('co_work_phone', ''),
                'x_co_applicant_monthly_income': extract_numeric(data.get('co_monthly_income', '')),
                'x_co_applicant_occupation': data.get('co_occupation', ''),
                'x_co_applicant_time_on_job': co_time_on_job_combined or data.get('co_time_on_job', ''),
            }
            
            # Only add fields that have values
            for field_name, field_value in credit_fields.items():
                if field_value:
                    update_vals[field_name] = field_value
            
            if update_vals:
                self.models.execute_kw(
                    self.database, self.uid, self.password,
                    'crm.lead', 'write', [[lead_id], update_vals]
                )
                logger.info(f"Updated lead ID {lead_id} with {len(update_vals)} fields")
            
            return True
            
        except Exception as e:
            logger.error(f"Error updating lead with credit app fields: {e}")
            # Try basic update if extended fields fail
            try:
                basic_vals = {}
                if data.get('first_name') or data.get('last_name'):
                    basic_vals['contact_name'] = f"{data.get('first_name', '')} {data.get('last_name', '')}".strip()
                if data.get('mobile_phone'):
                    basic_vals['mobile'] = data['mobile_phone']
                if data.get('email'):
                    basic_vals['email_from'] = data['email']
                
                if basic_vals:
                    self.models.execute_kw(
                        self.database, self.uid, self.password,
                        'crm.lead', 'write', [[lead_id], basic_vals]
                    )
                    logger.info(f"Updated lead ID {lead_id} with basic fields only")
                
                return True
                
            except Exception as e2:
                logger.error(f"Error updating lead with basic fields: {e2}")
                return False
    
    def get_attachment_by_name(self, lead_id: int, file_name: str) -> Optional[Dict]:
        """Get attachment details by name for a specific lead"""
        if not self.connected:
            return None

        try:
            attachment_ids = self.models.execute_kw(
                self.database, self.uid, self.password,
                'ir.attachment', 'search',
                [['&', ('res_model', '=', 'crm.lead'),
                      ('res_id', '=', lead_id),
                      ('name', '=', file_name)]]
            )

            if attachment_ids:
                attachments = self.models.execute_kw(
                    self.database, self.uid, self.password,
                    'ir.attachment', 'read',
                    [attachment_ids],
                    {'fields': ['id', 'name', 'datas', 'create_date']}
                )
                return attachments[0] if attachments else None
            return None

        except Exception as e:
            logger.error(f"Error getting attachment: {e}")
            return None

    def delete_attachment(self, attachment_id: int) -> bool:
        """Delete an attachment by ID"""
        if not self.connected:
            return False

        try:
            self.models.execute_kw(
                self.database, self.uid, self.password,
                'ir.attachment', 'unlink',
                [[attachment_id]]
            )
            logger.info(f"Deleted attachment ID {attachment_id}")
            return True

        except Exception as e:
            logger.error(f"Error deleting attachment: {e}")
            return False

    def attach_file_to_lead(self, lead_id: int, file_path: str, archive_old: bool = True, trigger_reason: str = None) -> bool:
        """Attach a file to a lead, optionally archiving old version"""
        if not self.connected:
            return False

        try:
            file_path = Path(file_path)
            if not file_path.exists():
                logger.error(f"File not found: {file_path}")
                return False

            # Check for existing attachment with same name
            if archive_old:
                existing_attachment = self.get_attachment_by_name(lead_id, file_path.name)
                if existing_attachment:
                    # Archive the old attachment locally
                    reason = trigger_reason or "Credit app PDF was modified and re-uploaded"
                    self._archive_attachment_to_history(file_path, existing_attachment, reason, lead_id)
                    # Delete old attachment from Odoo
                    self.delete_attachment(existing_attachment['id'])
                    logger.info(f"Archived and deleted old attachment: {file_path.name}")

            # Read file content
            with open(file_path, 'rb') as f:
                file_content = f.read()

            # Create attachment
            attachment_vals = {
                'name': file_path.name,
                'res_model': 'crm.lead',
                'res_id': lead_id,
                'datas': base64.b64encode(file_content).decode('utf-8'),
                'type': 'binary',
                'mimetype': 'application/pdf'
            }

            attachment_id = self.models.execute_kw(
                self.database, self.uid, self.password,
                'ir.attachment', 'create', [attachment_vals]
            )

            logger.info(f"Attached {file_path.name} to lead {lead_id}")
            return True

        except Exception as e:
            logger.error(f"Error attaching file: {e}")
            return False

    def _archive_attachment_to_history(self, current_file_path: Path, attachment: Dict, reason: str = "File modified", lead_id: int = None):
        """Archive an attachment from Odoo to local history folder with changelog"""
        try:
            # Create history folder
            history_folder = current_file_path.parent / "history"
            history_folder.mkdir(exist_ok=True)

            # Generate archive filename with timestamp
            timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S")
            base_name = current_file_path.stem
            extension = current_file_path.suffix
            archive_name = f"{base_name}_ODOO_{timestamp}{extension}"
            archive_path = history_folder / archive_name

            # Decode and save attachment data
            if attachment.get('datas'):
                file_data = base64.b64decode(attachment['datas'])
                with open(archive_path, 'wb') as f:
                    f.write(file_data)
                logger.info(f"Archived old attachment to: {archive_path}")

            # Update changelog
            self._update_history_changelog(history_folder, archive_name, reason, "from_odoo", lead_id)

        except Exception as e:
            logger.error(f"Error archiving attachment: {e}")

    def _archive_local_file_to_history(self, file_path: Path, reason: str = "File modified", lead_id: int = None):
        """Archive a local file to history folder with changelog"""
        try:
            # Create history folder
            history_folder = file_path.parent / "history"
            history_folder.mkdir(exist_ok=True)

            # Generate archive filename with timestamp
            timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S")
            base_name = file_path.stem
            extension = file_path.suffix
            archive_name = f"{base_name}_LOCAL_{timestamp}{extension}"
            archive_path = history_folder / archive_name

            # Copy file to history
            import shutil
            shutil.copy2(file_path, archive_path)
            logger.info(f"Archived local file to: {archive_path}")

            # Update changelog
            self._update_history_changelog(history_folder, archive_name, reason, "from_local", lead_id)

        except Exception as e:
            logger.error(f"Error archiving local file: {e}")

    def _update_history_changelog(self, history_folder: Path, filename: str, reason: str, source: str, lead_id: int = None):
        """Update or create changelog in history folder"""
        try:
            changelog_path = history_folder / "CHANGELOG.md"
            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

            # Create or append to changelog
            if not changelog_path.exists():
                with open(changelog_path, 'w') as f:
                    f.write("# History Changelog\n\n")
                    f.write("This file tracks why files were archived to this history folder.\n\n")
                    f.write("## Archive Log\n\n")

            # Append new entry
            with open(changelog_path, 'a') as f:
                f.write(f"### {timestamp}\n")
                f.write(f"- **File**: {filename}\n")
                f.write(f"- **Source**: {source}\n")
                f.write(f"- **Reason**: {reason}\n")
                f.write(f"- **Lead**: #{lead_id if lead_id else 'N/A'}\n\n")

            logger.debug(f"Updated history changelog: {reason}")

        except Exception as e:
            logger.error(f"Error updating changelog: {e}")
    
    def add_tag_to_lead(self, lead_id: int, tag_name: str) -> bool:
        """Add a tag to a lead"""
        if not self.connected:
            return False
        
        try:
            # Find or create tag
            tag_ids = self.models.execute_kw(
                self.database, self.uid, self.password,
                'crm.tag', 'search', [[('name', '=', tag_name)]]
            )
            
            if not tag_ids:
                # Create the tag
                tag_id = self.models.execute_kw(
                    self.database, self.uid, self.password,
                    'crm.tag', 'create', [{'name': tag_name}]
                )
            else:
                tag_id = tag_ids[0]
            
            # Add tag to lead
            self.models.execute_kw(
                self.database, self.uid, self.password,
                'crm.lead', 'write', [[lead_id], {'tag_ids': [(4, tag_id)]}]
            )
            
            logger.info(f"Added tag '{tag_name}' to lead {lead_id}")
            return True
            
        except Exception as e:
            logger.error(f"Error adding tag: {e}")
            return False
    
    def _get_state_id(self, state_code: str) -> Optional[int]:
        """Get Odoo state ID from state code"""
        if not state_code or not self.connected:
            return None
        
        try:
            state_ids = self.models.execute_kw(
                self.database, self.uid, self.password,
                'res.country.state', 'search',
                [[('code', '=', state_code), ('country_id.code', '=', 'US')]]
            )
            return state_ids[0] if state_ids else None
        except:
            return None


class PDFDataExtractor:
    """Extract form field data from filled PDFs"""
    
    def extract_from_pdf(self, file_path: str) -> Dict[str, Any]:
        """Extract all form field data from a PDF"""
        data = {}
        
        if PYMUPDF_AVAILABLE:
            data = self._extract_with_pymupdf(file_path)
        elif PYPDF2_AVAILABLE:
            data = self._extract_with_pypdf2(file_path)
        else:
            logger.error("No PDF library available for extraction")
        
        return self._normalize_extracted_data(data)
    
    def _extract_with_pymupdf(self, file_path: str) -> Dict[str, Any]:
        """Extract using PyMuPDF"""
        data = {}
        
        try:
            doc = fitz.open(file_path)
            found_fields = []
            mapped_fields = []
            
            for page in doc:
                widgets = page.widgets()
                for widget in widgets:
                    if widget.field_name and widget.field_value:
                        found_fields.append(f"'{widget.field_name}' = '{widget.field_value}'")
                        # Map AutoKing field names to our standard fields
                        field_name = self._map_field_name(widget.field_name)
                        if field_name:
                            # Special handling for Own/Rent fields
                            if field_name == 'residence_type' and widget.field_value.lower() == 'yes':
                                if 'own' in widget.field_name.lower():
                                    data[field_name] = 'Own'
                                elif 'rent' in widget.field_name.lower():
                                    data[field_name] = 'Rent'
                                else:
                                    data[field_name] = widget.field_value
                            elif field_name == 'co_residence_type' and widget.field_value.lower() == 'yes':
                                if 'own' in widget.field_name.lower():
                                    data[field_name] = 'Own'
                                elif 'rent' in widget.field_name.lower():
                                    data[field_name] = 'Rent'
                                else:
                                    data[field_name] = widget.field_value
                            else:
                                data[field_name] = widget.field_value
                            mapped_fields.append(f"'{widget.field_name}' -> '{field_name}'")
                        else:
                            logger.warning(f"Unmapped field: '{widget.field_name}' = '{widget.field_value}'")
            
            doc.close()
            logger.info(f"Found {len(found_fields)} fields, mapped {len(mapped_fields)} fields using PyMuPDF")
            if found_fields:
                logger.debug(f"First 10 found fields: {'; '.join(found_fields[:10])}")
            if mapped_fields:
                logger.debug(f"First 10 mapped fields: {'; '.join(mapped_fields[:10])}")
            
        except Exception as e:
            logger.error(f"PyMuPDF extraction error: {e}")
        
        return data
    
    def _extract_with_pypdf2(self, file_path: str) -> Dict[str, Any]:
        """Extract using PyPDF2"""
        data = {}
        
        try:
            reader = PdfReader(file_path)
            
            if '/AcroForm' in reader.trailer['/Root']:
                fields = reader.get_form_text_fields()
                
                for field_name, field_value in fields.items():
                    if field_value:
                        mapped_name = self._map_field_name(field_name)
                        if mapped_name:
                            data[mapped_name] = field_value
                
                logger.info(f"Extracted {len(data)} fields using PyPDF2")
            
        except Exception as e:
            logger.error(f"PyPDF2 extraction error: {e}")
        
        return data
    
    def _map_field_name(self, pdf_field_name: str) -> Optional[str]:
        """Map PDF field names to our standard field names"""
        # Comprehensive mapping from PDF fields to our standard fields
        field_map = {
            # Personal Information
            'Buyer First Name': 'first_name',
            'First Name': 'first_name',
            'Applicant First Name': 'first_name',
            'Buyer Last Name': 'last_name',
            'Last Name': 'last_name',
            'Applicant Last Name': 'last_name',
            'Middle Name': 'middle_name',
            'Middle Initial': 'middle_name',
            'Date of birth': 'date_of_birth',
            'Date of Birth': 'date_of_birth',
            'DOB': 'date_of_birth',
            'SSN': 'ssn',
            'Social Security': 'ssn',
            'Social Security Number': 'ssn',
            
            # Contact Information
            'Email Address': 'email',
            'Email': 'email',
            'Email Address_2': 'co_email',
            'Cell Phone': 'mobile_phone',
            'Mobile Phone': 'mobile_phone',
            'Home Phone': 'home_phone',
            'Phone': 'mobile_phone',
            'Phone Number': 'mobile_phone',
            'Cell Phone_2': 'co_mobile_phone',
            'Mobile Phone_2': 'co_mobile_phone',
            'Home Phone_2': 'co_home_phone',
            
            # Address Information
            'Current Address': 'address',
            'Address': 'address',
            'Street Address': 'address',
            'Home Address': 'address',
            'Apartment': 'address2',
            'Apt': 'address2',
            'Unit': 'address2',
            'City': 'city',
            'State': 'state',
            'ZIP Code': 'zip',
            'ZIP': 'zip',
            'Zip Code': 'zip',
            
            # Housing Information
            'Own or Rent': 'residence_type',
            'Residence Type': 'residence_type',
            'Housing': 'residence_type',
            'Own 1': 'residence_type',  # "Yes" = own
            'Rent 1': 'residence_type',  # "Yes" = rent
            'Rent/Mortgage': 'rent_mortgage',
            'Rent or Mortgage': 'rent_mortgage',
            'Monthly Payment': 'rent_mortgage',
            'Housing Payment': 'rent_mortgage',
            'Time at Residence': 'time_at_residence',
            'Time at Address': 'time_at_residence',
            'Years at Address': 'time_at_residence',
            'YR': 'time_at_residence_years',
            'MO': 'time_at_residence_months',
            
            # Driver License Information
            'Drivers License': 'drivers_license',
            'Driver License': 'drivers_license',
            'DL Number': 'drivers_license',
            'License Number': 'drivers_license',
            'DL State': 'dl_state',
            'License State': 'dl_state',
            'DL Exp Date': 'dl_exp_date',
            'License Exp': 'dl_exp_date',
            'Expiration Date': 'dl_exp_date',
            
            # Employment Information
            'Current employer': 'employer',
            'Employer': 'employer',
            'Company': 'employer',
            'Employer Name': 'employer',
            'Employer Address': 'employer_address',
            'Work Address': 'employer_address',
            'Company Address': 'employer_address',
            'Employer City': 'employer_city',
            'Work City': 'employer_city',
            'Employer State': 'employer_state',
            'Work State': 'employer_state',
            'Employer ZIP': 'employer_zip',
            'Work ZIP': 'employer_zip',
            'Employer Phone': 'work_phone',
            'Work Phone': 'work_phone',
            'Company Phone': 'work_phone',
            'Position': 'occupation',
            'Occupation': 'occupation',
            'Job Title': 'occupation',
            'Title': 'occupation',
            'Time on Job': 'time_on_job',
            'Years on Job': 'time_on_job',
            'Time at Job': 'time_on_job',
            'Length of Employment': 'time_on_job',
            'How Long': 'time_on_job',
            'Years at Job': 'time_on_job',
            'Years at current job': 'time_on_job',
            'Time with employer': 'time_on_job',
            'Time at current employer': 'time_on_job',
            
            # Job Time - Years and Months Fields
            'Job Years': 'time_on_job_years',
            'Employment Years': 'time_on_job_years',
            'Job Months': 'time_on_job_months',
            'Employment Months': 'time_on_job_months',
            'Years_1': 'time_on_job_years',  # Common pattern for job years
            'Months_1': 'time_on_job_months',  # Common pattern for job months
            'YRS': 'time_on_job_years',  # Job time years (plural vs residence YR)
            'MOS': 'time_on_job_months',  # Job time months (plural vs residence MO)
            
            # Income Information
            'Income': 'monthly_income',
            'Monthly Income': 'monthly_income',
            'Gross Income': 'monthly_income',
            'Monthly Gross': 'monthly_income',
            'Salary': 'monthly_income',
            'Pay': 'monthly_income',
            'Hourly Rate': 'hourly_rate',
            'Hourly Pay': 'hourly_rate',
            'Hours per Week': 'hours_per_week',
            'Weekly Hours': 'hours_per_week',
            
            # Vehicle Information
            'Vehicle Year': 'vehicle_year',
            'Year': 'vehicle_year',
            'Car Year': 'vehicle_year',
            'Vehicle Make': 'vehicle_make',
            'Make': 'vehicle_make',
            'Car Make': 'vehicle_make',
            'Vehicle Model': 'vehicle_model',
            'Model': 'vehicle_model',
            'Car Model': 'vehicle_model',
            'Vehicle Trim': 'vehicle_trim',
            'Trim': 'vehicle_trim',
            'Trim Level': 'vehicle_trim',
            'VIN': 'vehicle_vin',
            'Vehicle VIN': 'vehicle_vin',
            'VIN Number': 'vehicle_vin',
            'Mileage': 'vehicle_mileage',
            'Miles': 'vehicle_mileage',
            'Vehicle Mileage': 'vehicle_mileage',
            'Sale Price': 'sale_price',
            'Sales Price': 'sale_price',
            'Vehicle Price': 'sale_price',
            'Price': 'sale_price',
            'List Price': 'list_price',
            'MSRP': 'list_price',
            'Down Payment': 'down_payment',
            'Down': 'down_payment',
            'Cash Down': 'down_payment',
            'Desired Down Payment': 'desired_down_payment',
            'Desired Down': 'desired_down_payment',
            'Customer Down Payment': 'desired_down_payment',
            'Trade Value': 'trade_value',
            'Trade Allowance': 'trade_allowance',
            'Trade Payoff': 'trade_payoff',
            
            # Additional field variations from AutoKing forms
            'Buyer Middle Initial': 'middle_name',
            'MortgageRent Amount': 'rent_mortgage',
            'Mortgage/Rent Amount': 'rent_mortgage',
            'Rent 1': 'residence_type',  # "Yes" = rent, "No" = own
            'Landlord Name': 'landlord_name',
            'YR': 'time_at_residence_years',
            'MO': 'time_at_residence_months',
            'Years at current residence': 'time_at_residence',
            'Time at current address': 'time_at_residence',
            'Employer address incl City State ZIP': 'employer_address',
            ' of hours workedweek': 'hours_and_pay',
            'Hours worked per week': 'hours_per_week',
            'DownTotal': 'down_payment',
            'Down Total': 'down_payment',
            'FeeTotal': 'total_amount_financed',
            'Name of BankCredit Union': 'bank_name',
            'Supervisor Name': 'supervisor_name',
            
            # Co-Applicant Information
            'Co-Applicant First Name': 'co_first_name',
            'Co-Buyer First Name': 'co_first_name',
            'Co-buyer first name': 'co_first_name',  # lowercase variation
            'Coborrower First Name': 'co_first_name',
            'Co-Applicant Last Name': 'co_last_name',
            'Co-Buyer Last Name': 'co_last_name',
            'Coborrower Last Name': 'co_last_name',
            'Co-Applicant Middle Initial': 'co_middle_name',
            'Co-Buyer Middle Initial': 'co_middle_name',
            'Co-buyer middle initial': 'co_middle_name',  # lowercase variation
            'Co-Applicant Phone': 'co_mobile_phone',
            'Co-Buyer Phone': 'co_mobile_phone',
            'Co-Applicant Cell Phone': 'co_mobile_phone',
            'Co-Buyer Cell Phone': 'co_mobile_phone',
            'Co-Applicant Home Phone': 'co_home_phone',
            'Co-Buyer Home Phone': 'co_home_phone',
            'Co-Applicant Email': 'co_email',
            'Co-Buyer Email': 'co_email',
            'Co-Applicant SSN': 'co_ssn',
            'Co-Buyer SSN': 'co_ssn',
            'Co-Applicant DOB': 'co_dob',
            'Co-Buyer DOB': 'co_dob',
            'Co-Applicant Date of birth': 'co_dob',
            'Co-Buyer Date of birth': 'co_dob',
            
            # Co-Applicant Address Information
            'Co-Applicant Address': 'co_address',
            'Co-Buyer Address': 'co_address',
            'Co-Applicant Current Address': 'co_address',
            'Co-Buyer Current Address': 'co_address',
            'Co-Applicant City': 'co_city',
            'Co-Buyer City': 'co_city',
            'Co-Applicant State': 'co_state',
            'Co-Buyer State': 'co_state',
            'Co-Applicant ZIP': 'co_zip',
            'Co-Buyer ZIP': 'co_zip',
            'Co-Applicant ZIP Code': 'co_zip',
            'Co-Buyer ZIP Code': 'co_zip',
            'Co-Applicant Residence Type': 'co_residence_type',
            'Co-Buyer Residence Type': 'co_residence_type',
            'Co-Applicant Rent/Mortgage': 'co_rent_mortgage',
            'Co-Buyer Rent/Mortgage': 'co_rent_mortgage',
            'Co-Applicant Time at Residence': 'co_time_at_residence',
            'Co-Buyer Time at Residence': 'co_time_at_residence',
            
            # Co-Applicant Employment Information
            'Co-Applicant Employer': 'co_employer',
            'Co-Buyer Employer': 'co_employer',
            'Co-Applicant Current employer': 'co_employer',
            'Co-Buyer Current employer': 'co_employer',
            'Co-Applicant Employer Address': 'co_employer_address',
            'Co-Buyer Employer Address': 'co_employer_address',
            'Co-Applicant Work Phone': 'co_work_phone',
            'Co-Buyer Work Phone': 'co_work_phone',
            'Co-Applicant Employer Phone': 'co_work_phone',
            'Co-Buyer Employer Phone': 'co_work_phone',
            'Co-Applicant Income': 'co_monthly_income',
            'Co-Buyer Income': 'co_monthly_income',
            'Co-Applicant Monthly Income': 'co_monthly_income',
            'Co-Buyer Monthly Income': 'co_monthly_income',
            'Co-Applicant Occupation': 'co_occupation',
            'Co-Buyer Occupation': 'co_occupation',
            'Co-Applicant Position': 'co_occupation',
            'Co-Buyer Position': 'co_occupation',
            'Co-Applicant Time on Job': 'co_time_on_job',
            'Co-Buyer Time on Job': 'co_time_on_job',
            'How Long_2': 'co_time_on_job',
            'Time at Job_2': 'co_time_on_job',
            'Length of Employment_2': 'co_time_on_job',
            'Co-Applicant Years on Job': 'co_time_on_job',
            'Co-Buyer Years on Job': 'co_time_on_job',
            
            # Co-Applicant Job Time - Years and Months Fields  
            'Co-Applicant Job Years': 'co_time_on_job_years',
            'Co-Buyer Job Years': 'co_time_on_job_years',
            'Co-Applicant Employment Years': 'co_time_on_job_years',
            'Co-Buyer Employment Years': 'co_time_on_job_years',
            'Co-Applicant Job Months': 'co_time_on_job_months',
            'Co-Buyer Job Months': 'co_time_on_job_months',
            'Co-Applicant Employment Months': 'co_time_on_job_months',
            'Co-Buyer Employment Months': 'co_time_on_job_months',
            'Years_2': 'co_time_on_job_years',  # Common pattern for co-applicant job years
            'Months_2': 'co_time_on_job_months',  # Common pattern for co-applicant job months
            'YRS_2': 'co_time_on_job_years',  # Co-applicant job time years
            'MOS_2': 'co_time_on_job_months',  # Co-applicant job time months
            
            # Co-Applicant Driver License
            'Co-Applicant Drivers License': 'co_drivers_license',
            'Co-Buyer Drivers License': 'co_drivers_license',
            'Co-Applicant DL Number': 'co_drivers_license',
            'Co-Buyer DL Number': 'co_drivers_license',
            'Co-Applicant DL State': 'co_dl_state',
            'Co-Buyer DL State': 'co_dl_state',
            'Co-Applicant DL Exp Date': 'co_dl_exp_date',
            'Co-Buyer DL Exp Date': 'co_dl_exp_date',
            
            # Numbered co-applicant field variations (from AutoKing forms)
            'Date of birth_2': 'co_dob',
            'SSN_2': 'co_ssn',
            'Home Phone_2': 'co_home_phone',
            'Current address': 'co_address',  # Note: this might conflict with main address
            'City_2': 'co_city',
            'State_2': 'co_state', 
            'ZIP Code_2': 'co_zip',
            'Own 2': 'co_residence_type',  # "Yes" = own
            'Rent 2': 'co_residence_type',  # "Yes" = rent
            'MortgageRent Amount_2': 'co_rent_mortgage',
            'Current employer_2': 'co_employer',
            'YR_2': 'co_time_at_residence_years',
            'MO_2': 'co_time_at_residence_months',
            'Employer address incl City State ZIP_2': 'co_employer_address',
            'Employer phone 2': 'co_work_phone',
            'Supervisor Name_2': 'co_supervisor_name',
            'Position_3': 'co_occupation',  # Note: Position_3 is likely co-applicant occupation
            'Income 2': 'co_monthly_income',
            ' of hours workedweek_2': 'co_hours_and_pay',
            'Relation to Applicant': 'co_relationship',
        }
        
        return field_map.get(pdf_field_name)
    
    def _normalize_extracted_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Normalize extracted data to consistent format"""
        normalized = {}
        
        for field, value in data.items():
            if value:
                # Clean up phone numbers
                if 'phone' in field:
                    value = re.sub(r'\D', '', str(value))
                    if len(value) == 10:
                        value = f"{value[:3]}-{value[3:6]}-{value[6:]}"
                
                # Clean up SSN
                elif field == 'ssn':
                    value = re.sub(r'\D', '', str(value))
                
                # Clean up currency and numeric fields
                elif 'income' in field or 'payment' in field or 'price' in field or 'amount' in field:
                    # Extract just the numeric part from strings like "3000+OT (~1000)"
                    value_str = str(value).replace('$', '').replace(',', '')
                    # Extract first number from strings with extra text
                    numeric_match = re.search(r'[\d.]+', value_str)
                    if numeric_match:
                        value = numeric_match.group()
                    else:
                        value = '0'
                
                # Convert date formats from MM/DD/YYYY to YYYY-MM-DD
                elif 'date' in field or 'dob' in field:
                    value = self._convert_date_format(str(value))
                
                normalized[field] = value
        
        return normalized
    
    def _convert_date_format(self, date_str: str) -> str:
        """Convert date from MM/DD/YYYY to YYYY-MM-DD format"""
        try:
            # Handle MM/DD/YYYY format (common in US forms)
            if '/' in date_str and len(date_str.split('/')) == 3:
                parts = date_str.split('/')
                if len(parts[2]) == 4:  # YYYY format
                    month, day, year = parts
                    # Ensure 2-digit month and day
                    month = month.zfill(2)
                    day = day.zfill(2)
                    return f"{year}-{month}-{day}"
            
            # If already in YYYY-MM-DD format or other format, return as-is
            return date_str
            
        except Exception as e:
            logger.warning(f"Could not convert date format '{date_str}': {e}")
            return date_str


class CreditAppFileHandler(FileSystemEventHandler):
    """Handle file system events for credit application PDFs"""
    
    def __init__(self, odoo_client: OdooClient, tracker: ProcessedFileTracker, extractor: PDFDataExtractor, config: dict):
        self.odoo = odoo_client
        self.tracker = tracker
        self.extractor = extractor
        self.config = config
        self.pending_files = {}  # Track files waiting to be processed
        
        # Supported file types for attachment
        self.SUPPORTED_FILE_TYPES = {'.pdf', '.jpg', '.jpeg', '.png', '.gif', '.tif', '.tiff', 
                                    '.doc', '.docx', '.txt', '.rtf', '.xls', '.xlsx'}
        
        # Files to skip (system files, thumbnails, etc.)
        self.SKIP_FILES = {'thumbs.db', '.ds_store', 'desktop.ini'}
    
    def _is_attachable_file(self, file_path: Path) -> bool:
        """Check if a file should be attached to Odoo leads"""
        file_name_lower = file_path.name.lower()

        # Skip files in history folders
        if 'history' in file_path.parts:
            return False

        # Skip system files and thumbnails
        if file_name_lower in self.SKIP_FILES:
            return False

        # Skip temporary files and partial downloads
        if file_name_lower.startswith(('.tmp', '~')) or file_name_lower.endswith(('.tmp', '.part')):
            return False

        # Skip very large files (over 50MB)
        try:
            if file_path.stat().st_size > 50 * 1024 * 1024:
                return False
        except:
            pass

        # Accept almost all other files (let Odoo handle file type restrictions)
        return True
    
    def _is_file_already_attached(self, lead_id: int, file_name: str) -> bool:
        """Check if a file is already attached to the lead"""
        if not self.odoo.connected:
            return False
        
        try:
            # Search for existing attachments with the same name
            attachment_ids = self.odoo.models.execute_kw(
                self.odoo.database, self.odoo.uid, self.odoo.password,
                'ir.attachment', 'search',
                [['&', ('res_model', '=', 'crm.lead'), 
                      ('res_id', '=', lead_id), 
                      ('name', '=', file_name)]]
            )
            return len(attachment_ids) > 0
        except Exception as e:
            logger.error(f"Error checking existing attachments: {e}")
            return False
    
    def _attach_file_to_existing_lead(self, file_path: str, lead_id: int) -> bool:
        """Attach a file to an existing Odoo lead"""
        try:
            path = Path(file_path)

            # Check if file is attachable
            if not self._is_attachable_file(path):
                logger.debug(f"File not attachable: {path.name}")
                return False

            # Check if already attached
            if self._is_file_already_attached(lead_id, path.name):
                logger.info(f"File already attached to lead {lead_id}: {path.name}")
                return False

            # For non-credit-app files, don't archive old versions
            archive_old = False

            # Use the OdooClient's attach method which handles archiving
            return self.odoo.attach_file_to_lead(lead_id, str(path), archive_old=archive_old)

        except Exception as e:
            logger.error(f"Error attaching file {file_path} to lead {lead_id}: {e}")
            return False
    
    def on_created(self, event):
        """Handle new file creation"""
        if event.is_directory:
            return

        if self._is_credit_app(event.src_path):
            logger.info(f"New credit app detected: {event.src_path}")
            trigger_reason = self._determine_trigger_reason(event.src_path, 'created')
            self._queue_for_processing(event.src_path, trigger_reason)
        else:
            # Check if this is a file in a monitored folder
            folder_path = str(Path(event.src_path).parent)
            lead_id = self.tracker.get_lead_id_for_folder(folder_path)
            if lead_id:
                logger.info(f"New file in monitored folder: {event.src_path}")
                self._queue_for_attachment(event.src_path, lead_id)
    
    def on_modified(self, event):
        """Handle file modification"""
        if event.is_directory:
            return

        if self._is_credit_app(event.src_path):
            logger.info(f"Credit app modified: {event.src_path}")
            # Track the trigger reason
            trigger_reason = self._determine_trigger_reason(event.src_path, 'modified')
            self._queue_for_processing(event.src_path, trigger_reason)
        else:
            # Check if this is a file in a monitored folder
            folder_path = str(Path(event.src_path).parent)
            lead_id = self.tracker.get_lead_id_for_folder(folder_path)
            if lead_id:
                logger.info(f"File modified in monitored folder: {event.src_path}")
                self._queue_for_attachment(event.src_path, lead_id)
    
    def _determine_trigger_reason(self, file_path: str, event_type: str) -> str:
        """Determine why a file change was triggered"""
        path = Path(file_path)
        reasons = []

        # Check file modification details
        try:
            stat = path.stat()
            current_time = time.time()
            mtime_diff = current_time - stat.st_mtime

            if mtime_diff < 5:  # Modified in last 5 seconds
                reasons.append("File just saved")

            # Check if file was opened by specific programs
            if path.suffix.lower() == '.pdf':
                # Check for Adobe Acrobat or PDF editor signatures
                try:
                    with open(path, 'rb') as f:
                        header = f.read(1024)
                        if b'/Producer' in header:
                            if b'Adobe' in header:
                                reasons.append("Edited in Adobe Acrobat")
                            elif b'Chrome' in header:
                                reasons.append("Downloaded from Chrome")
                            elif b'Microsoft' in header:
                                reasons.append("Edited in Microsoft Edge")
                except:
                    pass

            # Check if this is from the API/Extension
            marker_file = path.parent / f".odoo_lead_*"
            if list(path.parent.glob(".odoo_lead_*")):
                reasons.append("Updated via Chrome Extension/API")

            # Check file size change
            if hasattr(self, '_file_sizes'):
                old_size = self._file_sizes.get(str(path))
                if old_size and old_size != stat.st_size:
                    size_change = stat.st_size - old_size
                    if size_change > 0:
                        reasons.append(f"File size increased by {size_change} bytes")
                    else:
                        reasons.append(f"File size decreased by {-size_change} bytes")
                self._file_sizes[str(path)] = stat.st_size
            else:
                self._file_sizes = {str(path): stat.st_size}

        except Exception as e:
            logger.debug(f"Error determining trigger: {e}")

        if reasons:
            return f"{event_type.capitalize()} - {', '.join(reasons)}"
        else:
            return f"{event_type.capitalize()} - Unknown trigger"

    def _is_credit_app(self, file_path: str) -> bool:
        """Check if file is specifically a credit application PDF"""
        path = Path(file_path)
        if path.suffix.lower() != '.pdf':
            return False

        # Skip files in history folders
        if 'history' in path.parts:
            return False

        file_name = path.name.lower()

        # Only these patterns indicate actual credit applications
        credit_app_patterns = ['credit app', 'credit application']

        for pattern in credit_app_patterns:
            if pattern in file_name:
                return True

        return False
    
    def _queue_for_processing(self, file_path: str, trigger_reason: str = None):
        """Queue a file for processing after it stabilizes"""
        # If file is already queued, just update the time (debounce)
        if file_path in self.pending_files:
            logger.info(f"File already queued, updating timestamp: {file_path}")
            self.pending_files[file_path]['time'] = time.time()
            if trigger_reason:
                self.pending_files[file_path]['trigger'] = trigger_reason
        else:
            self.pending_files[file_path] = {
                'time': time.time(),
                'type': 'credit_app',
                'trigger': trigger_reason or 'Manual processing'
            }
        # Process will happen in process_pending_files()
    
    def _queue_for_attachment(self, file_path: str, lead_id: int):
        """Queue a file for attachment after it stabilizes"""
        self.pending_files[file_path] = {'time': time.time(), 'type': 'attachment', 'lead_id': lead_id}
        # Process will happen in process_pending_files()
    
    def process_pending_files(self):
        """Process files that have stabilized (no changes for 60 seconds)"""
        current_time = time.time()
        files_to_process = []
        
        for file_path, queue_info in list(self.pending_files.items()):
            queue_time = queue_info.get('time', queue_info) if isinstance(queue_info, dict) else queue_info
            if current_time - queue_time > 60:  # 60 seconds stable (increased from 30)
                files_to_process.append((file_path, queue_info))
                del self.pending_files[file_path]
        
        for file_path, queue_info in files_to_process:
            if isinstance(queue_info, dict):
                if queue_info.get('type') == 'credit_app':
                    trigger_reason = queue_info.get('trigger', 'Unknown trigger')
                    self._process_file(file_path, trigger_reason)
                elif queue_info.get('type') == 'attachment':
                    self._process_attachment(file_path, queue_info.get('lead_id'))
            else:
                # Backward compatibility for old format
                self._process_file(file_path)
    
    def _process_attachment(self, file_path: str, lead_id: int):
        """Process a file for attachment to an existing lead"""
        try:
            path = Path(file_path)
            if not path.exists():
                logger.warning(f"File no longer exists for attachment: {file_path}")
                return
            
            logger.info(f"Processing attachment: {file_path} -> lead {lead_id}")
            
            # Attempt to attach file
            success = self._attach_file_to_existing_lead(file_path, lead_id)
            
            if success:
                logger.info(f"Successfully attached {path.name} to lead {lead_id}")
            else:
                # Don't warn for expected failures (non-attachable files like those in history folders)
                logger.debug(f"Failed to attach {path.name} to lead {lead_id}")
                
        except Exception as e:
            logger.error(f"Error processing attachment {file_path}: {e}")
    
    def _process_file(self, file_path: str, trigger_reason: str = "Manual processing"):
        """Process a credit application PDF"""
        try:
            path = Path(file_path)
            if not path.exists():
                logger.warning(f"File no longer exists: {file_path}")
                return

            logger.info(f"Processing file - Trigger: {trigger_reason}")
            
            # Check for marker file FIRST - indicates API already created lead
            folder_path = path.parent
            marker_files = list(folder_path.glob(".odoo_lead_*"))
            marker_lead_id = None
            if marker_files:
                # Extract lead ID from marker file
                marker_file = marker_files[0]
                lead_id_str = marker_file.name.replace(".odoo_lead_", "")
                try:
                    marker_lead_id = int(lead_id_str)
                    logger.info(f"Found marker file indicating lead {marker_lead_id} already created by API")
                except ValueError:
                    logger.warning(f"Invalid marker file format: {marker_file.name}")
            
            # Check if already processed (but allow reprocessing if marker file exists)
            mtime = path.stat().st_mtime
            if self.tracker.is_processed(file_path, mtime) and not marker_lead_id:
                logger.info(f"File already processed: {file_path}")
                return
            
            logger.info(f"Processing: {file_path}")
            
            # Extract data from PDF
            data = self.extractor.extract_from_pdf(file_path)
            if not data:
                logger.warning(f"No data extracted from: {file_path}")
                return
            
            logger.info(f"Extracted {len(data)} fields from PDF")
            
            # Get customer name from file path or data
            customer_name = self._get_customer_name(file_path, data)
            
            # Check if this folder already has a lead associated with it (prioritize marker file)
            folder_path = str(Path(file_path).parent)
            existing_lead_id = marker_lead_id or self.tracker.get_lead_id_for_folder(folder_path)

            if existing_lead_id:
                logger.info(f"Folder already has lead #{existing_lead_id}, updating instead of creating new")
                # Update the existing lead with new data
                self.odoo.update_lead(existing_lead_id, data)

                # Attach the updated PDF
                archive_old = self.config.get('two_way_sync', {}).get('archive_on_update', True)
                self.odoo.attach_file_to_lead(existing_lead_id, file_path, archive_old=archive_old, trigger_reason=trigger_reason)

                # Update processed tracker with existing lead
                self.tracker.mark_processed(file_path, mtime, existing_lead_id)

                # Scan for any additional files to attach
                self._attach_existing_folder_files(file_path, existing_lead_id)

                logger.info(f"Updated existing lead {existing_lead_id} with new PDF data")
                return
            
            # Search for existing lead
            phone = data.get('mobile_phone', data.get('phone', ''))
            lead = None
            leads = []
            
            if phone:
                leads = self.odoo.search_leads_by_phone(phone)
                
                # Smart lead selection: prioritize "Online App" tagged leads
                online_app_leads = [l for l in leads if "Online App" in l.get('tags', [])]
                
                if online_app_leads:
                    lead = online_app_leads[0]  # Newest first (already sorted by create_date desc)
                    logger.info(f"Found lead with 'Online App' tag: {lead['name']} (ID: {lead['id']})")
                elif leads:
                    lead = leads[0]  # Fallback to newest lead
                    logger.info(f"Found lead: {lead['name']} (ID: {lead['id']})")
            
            # If no lead found by phone, try searching by customer name
            if not lead and customer_name and customer_name != "Unknown Customer":
                logger.info(f"No lead found by phone, searching by name: {customer_name}")
                try:
                    # Search for leads with matching customer name
                    name_leads = self.odoo.models.execute_kw(
                        self.odoo.database, self.odoo.uid, self.odoo.password,
                        'crm.lead', 'search',
                        [[('contact_name', 'ilike', customer_name)]],
                        {'limit': 10, 'order': 'create_date desc'}
                    )
                    
                    if name_leads:
                        # Get lead details
                        leads_data = self.odoo.models.execute_kw(
                            self.odoo.database, self.odoo.uid, self.odoo.password,
                            'crm.lead', 'read', [name_leads],
                            {'fields': ['name', 'contact_name', 'partner_name', 'tag_ids', 'create_date']}
                        )
                        
                        if leads_data:
                            lead = leads_data[0]
                            logger.info(f"Found lead by name: {lead['name']} (ID: {lead['id']})")
                except Exception as e:
                    logger.error(f"Error searching by name: {e}")
            
            # Create or update lead
            if lead:
                lead_id = lead['id']
                self.odoo.update_lead(lead_id, data)
            else:
                logger.info("No matching lead found, creating new lead")
                lead_id = self.odoo.create_lead(data)
                
                if lead_id:
                    # Add tag for manually created lead using config
                    manual_tag = self.config.get('tags', {}).get('manual_entry', 'Walk In')
                    self.odoo.add_tag_to_lead(lead_id, manual_tag)
            
            if lead_id:
                # Attach PDF to lead
                archive_old = self.config.get('two_way_sync', {}).get('archive_on_update', True)
                self.odoo.attach_file_to_lead(lead_id, file_path, archive_old=archive_old)
                
                # Add synced tag using config
                synced_tag = self.config.get('tags', {}).get('synced', 'synced')
                self.odoo.add_tag_to_lead(lead_id, synced_tag)
                
                # Create JSON data file
                json_path = path.with_suffix('.json')
                with open(json_path, 'w') as f:
                    json.dump(data, f, indent=2)
                
                # Attach JSON to lead
                self.odoo.attach_file_to_lead(lead_id, str(json_path))
                
                # Mark as processed
                self.tracker.mark_processed(file_path, mtime, lead_id)
                
                # Scan folder for existing files to attach
                self._attach_existing_folder_files(file_path, lead_id)
                
                logger.info(f"Successfully processed and synced to lead {lead_id}")
            else:
                logger.error("Failed to create/update lead")
            
        except Exception as e:
            logger.error(f"Error processing {file_path}: {e}")
    
    def _attach_existing_folder_files(self, credit_app_path: str, lead_id: int):
        """Scan folder for existing files and attach them to the lead"""
        try:
            folder_path = Path(credit_app_path).parent
            logger.info(f"Scanning folder for existing files to attach: {folder_path}")
            
            attached_count = 0
            skipped_count = 0
            
            for file_path in folder_path.iterdir():
                if file_path.is_file():
                    # Skip the credit app itself and JSON files we created
                    # But DO NOT skip DCS Credit App PDFs
                    file_name_lower = file_path.name.lower()
                    if file_name_lower.endswith('.json'):
                        continue
                    # Skip credit app files UNLESS they start with "DCS"
                    if 'credit app' in file_name_lower and not file_name_lower.startswith('dcs '):
                        continue
                    
                    # Check if file is attachable
                    if self._is_attachable_file(file_path):
                        # Check if already attached
                        if not self._is_file_already_attached(lead_id, file_path.name):
                            success = self._attach_file_to_existing_lead(str(file_path), lead_id)
                            if success:
                                attached_count += 1
                                logger.info(f"Attached existing file: {file_path.name}")
                            else:
                                skipped_count += 1
                        else:
                            skipped_count += 1
                            logger.debug(f"File already attached: {file_path.name}")
                    else:
                        skipped_count += 1
                        logger.debug(f"File not attachable: {file_path.name}")
            
            if attached_count > 0:
                logger.info(f"Attached {attached_count} existing files from folder to lead {lead_id}")
            if skipped_count > 0:
                logger.debug(f"Skipped {skipped_count} files (already attached or not attachable)")
                
        except Exception as e:
            logger.error(f"Error scanning folder for existing files: {e}")
    
    def _get_customer_name(self, file_path: str, data: Dict) -> str:
        """Get customer name from file path or extracted data"""
        # Try to get from file path first (e.g., "John Smith Credit App.pdf")
        path = Path(file_path)
        name_from_path = path.stem.replace(' Credit App', '').strip()
        
        if name_from_path:
            return name_from_path
        
        # Fallback to extracted data
        first = data.get('first_name', '')
        last = data.get('last_name', '')
        return f"{first} {last}".strip() or "Unknown Customer"


def load_config():
    """Load configuration files"""
    # Load Odoo config
    odoo_config = {}
    if Path(ODOO_CONFIG_FILE).exists():
        with open(ODOO_CONFIG_FILE, 'r') as f:
            odoo_config = json.load(f)
    else:
        logger.warning("Odoo config not found, using defaults")
        odoo_config = {
            'url': 'http://localhost:8069',
            'database': 'odoo',
            'username': 'admin',
            'password': 'admin'
        }
    
    # Load or create watcher config
    watcher_config = {}
    config_path = Path(WATCHER_CONFIG_FILE)

    if config_path.exists():
        with open(config_path, 'r') as f:
            watcher_config = json.load(f)
    else:
        # Create default config
        watcher_config = {
            'watch_directory': 'current_month_only',  # Special value indicating current month watching
            'file_pattern': '*Credit App.pdf',
            'stabilization_time': 30,
            'tags': {
                'manual_entry': 'Walk In',
                'synced': 'synced'
            },
            'lead_stage': None,  # Set to stage name to auto-assign leads to specific stage
            'set_contact_name': True,  # Whether to set contact_name/partner_name fields
            'two_way_sync': {
                'enabled': False,  # Disabled by default
                'mode': 'hybrid',  # 'polling', 'on_demand', or 'hybrid'
                'polling_interval': 300,  # seconds (5 minutes)
                'sync_fields': ['all'],  # or specific field list
                'archive_history': True,
                'history_retention': 10,  # keep last 10 versions
                'archive_on_update': True  # Archive old PDFs when updating
            }
        }
        with open(config_path, 'w') as f:
            json.dump(watcher_config, f, indent=2)
        logger.info(f"Created default config: {config_path}")
    
    return odoo_config, watcher_config



def should_process_file(file_path):
    """Determine if a PDF file should be processed based on filename patterns"""
    file_name = Path(file_path).name.lower()
    
    # Skip version files
    if 'versions' in str(file_path).lower():
        return False
    
    # Process these types of files:
    process_patterns = [
        'credit app',           # Credit applications
        'dl scan',             # Driver's license scans  
        'drivers license',     # Driver's license
        'license scan',        # License scans
        'insurance',           # Insurance cards
        'pay stub',            # Pay stubs
        'paystub',             # Pay stubs (no space)
        'bank statement',      # Bank statements
        'w2',                  # W2 forms
        'tax return',          # Tax returns
        'proof of income',     # Income verification
        'employment',          # Employment verification
        'residence',           # Proof of residence
        'utility bill',        # Utility bills
    ]
    
    # Check if filename contains any of our target patterns
    for pattern in process_patterns:
        if pattern in file_name:
            logger.info(f"File matches pattern '{pattern}': {file_name}")
            return True
    
    # Default: don't process
    return False

def get_current_month_folder():
    """Get the current month's customer docs folder"""
    now = datetime.now()
    year = now.year
    month_name = now.strftime("%B")  # Full month name (e.g., "August")
    month_folder = Path(CUSTOMER_DOCS_BASE) / str(year) / f"{now.month:02d} - {month_name}"
    
    return month_folder

def main():
    """Main service loop"""
    logger.info("Starting File System -> Odoo Sync Service")
    
    # Load configurations
    odoo_config, watcher_config = load_config()
    
    # Initialize components
    odoo_client = OdooClient(odoo_config)
    if not odoo_client.connect():
        logger.error("Failed to connect to Odoo, exiting")
        sys.exit(1)
    
    tracker = ProcessedFileTracker()
    extractor = PDFDataExtractor()
    
    # Set up file system observer
    event_handler = CreditAppFileHandler(odoo_client, tracker, extractor, watcher_config)
    observer = Observer()
    
    # Watch only current month's folder
    current_month_folder = get_current_month_folder()
    
    # Create the folder if it doesn't exist
    if not current_month_folder.exists():
        logger.info(f"Creating current month folder: {current_month_folder}")
        current_month_folder.mkdir(parents=True, exist_ok=True)
    
    observer.schedule(event_handler, str(current_month_folder), recursive=True)
    observer.start()
    
    logger.info(f"Watching current month folder: {current_month_folder}")
    
    # Log currently monitored folders
    monitored_folders = tracker.get_monitored_folders()
    if monitored_folders:
        logger.info(f"Currently monitoring {len(monitored_folders)} customer folders for additional files:")
        for folder_path, info in monitored_folders.items():
            lead_id = info.get('lead_id')
            added_at = info.get('added_at', 'unknown')
            logger.info(f"  - {folder_path} (Lead #{lead_id}, added {added_at[:10]})")
    else:
        logger.info("No customer folders currently monitored for additional files")
    
    logger.info("Service is running. Press Ctrl+C to stop.")
    
    try:
        last_month_check = datetime.now().month
        
        while True:
            # Process pending files every 5 seconds
            event_handler.process_pending_files()
            
            # Check if month changed (restart watcher on new month)
            current_month = datetime.now().month
            if current_month != last_month_check:
                logger.info("Month changed, updating watch folder...")
                observer.stop()
                observer.join()
                
                # Start watching new month folder
                new_month_folder = get_current_month_folder()
                if not new_month_folder.exists():
                    logger.info(f"Creating new month folder: {new_month_folder}")
                    new_month_folder.mkdir(parents=True, exist_ok=True)
                
                observer = Observer()
                observer.schedule(event_handler, str(new_month_folder), recursive=True)
                observer.start()
                
                logger.info(f"Now watching: {new_month_folder}")
                last_month_check = current_month
            
            time.sleep(5)
            
    except KeyboardInterrupt:
        logger.info("Stopping service...")
        observer.stop()
    
    observer.join()
    logger.info("Service stopped")


if __name__ == "__main__":
    main()