nextgenpdf/pdf_renamer.py

import os
import tempfile
import zipfile
import threading
import uuid
import json
from datetime import datetime
from functools import wraps
from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory, flash, get_flashed_messages, jsonify
from werkzeug.utils import secure_filename
from werkzeug.middleware.proxy_fix import ProxyFix
from PyPDF2 import PdfReader
from authlib.integrations.flask_client import OAuth

app = Flask(__name__)

# Trust proxy headers (since we're behind Nginx)
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)

app.config['UPLOAD_FOLDER'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'processed')
app.config['ARCHIVE_FOLDER'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'archives')
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
os.makedirs(app.config['ARCHIVE_FOLDER'], exist_ok=True)
app.config['MAX_CONTENT_LENGTH'] = 500 * 1024 * 1024  # 500MB limit
app.secret_key = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production')

# Secure session cookie settings
app.config['SESSION_COOKIE_SECURE'] = True  # Only send cookie over HTTPS
app.config['SESSION_COOKIE_HTTPONLY'] = True  # Prevent JavaScript access
app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'  # CSRF protection
app.config['PERMANENT_SESSION_LIFETIME'] = 3600  # 1 hour session timeout

# Entra ID Configuration
TENANT_ID = os.getenv('TENANT_ID', 'your-tenant-id')
CLIENT_ID = os.getenv('CLIENT_ID', 'your-client-id')
CLIENT_SECRET = os.getenv('CLIENT_SECRET', 'your-client-secret')

print(f"DEBUG: TENANT_ID = {TENANT_ID}")
print(f"DEBUG: CLIENT_ID = {CLIENT_ID}")
print(f"DEBUG: CLIENT_SECRET = {'*' * len(CLIENT_SECRET) if CLIENT_SECRET else 'None'}")

# Admin Configuration
ADMIN_USERNAME = os.getenv('ADMIN_USERNAME', 'trbcit')
ADMIN_PASSWORD = os.getenv('ADMIN_PASSWORD', 'atdrmbicn')

# Initialize OAuth
oauth = OAuth(app)

# Register Microsoft OAuth provider with proper configuration
try:
    microsoft = oauth.register(
        name='microsoft',
        client_id=CLIENT_ID,
        client_secret=CLIENT_SECRET,
        server_metadata_url=f'https://login.microsoftonline.com/{TENANT_ID}/v2.0/.well-known/openid-configuration',
        client_kwargs={
            'scope': 'openid email profile',
            'token_endpoint_auth_method': 'client_secret_post'
        }
    )
    print("DEBUG: Microsoft OAuth provider registered successfully")
except Exception as e:
    print(f"ERROR: Failed to register Microsoft OAuth provider: {e}")
    microsoft = None

# Job persistence
JOBS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'jobs.json')
job_queue = {}
job_lock = threading.Lock()

def _save_jobs_unlocked():
    """Internal: Save jobs without acquiring lock (assumes lock is already held)"""
    try:
        # Convert datetime objects to strings for JSON serialization
        jobs_to_save = {}
        for job_id, job in job_queue.items():
            jobs_to_save[job_id] = job.copy()
            # Convert any datetime objects if needed
            if 'created_at' in jobs_to_save[job_id]:
                jobs_to_save[job_id]['created_at'] = str(jobs_to_save[job_id]['created_at'])
        
        with open(JOBS_FILE, 'w') as f:
            json.dump(jobs_to_save, f, indent=2)
    except Exception as e:
        print(f"ERROR: Failed to save jobs: {e}")

def save_jobs():
    """Save jobs to persistent storage"""
    with job_lock:
        _save_jobs_unlocked()

def load_jobs():
    """Load jobs from persistent storage"""
    global job_queue
    try:
        if os.path.exists(JOBS_FILE):
            with open(JOBS_FILE, 'r') as f:
                loaded_jobs = json.load(f)
                
                # Only load jobs where archive files still exist
                valid_jobs = {}
                skipped_count = 0
                
                for job_id, job_data in loaded_jobs.items():
                    # Only keep completed jobs with existing archive files
                    if job_data.get('status') == 'completed':
                        zip_path = job_data.get('zip_path', '')
                        if zip_path and os.path.exists(zip_path):
                            valid_jobs[job_id] = job_data
                        else:
                            skipped_count += 1
                            print(f"DEBUG: Skipping job {job_id} - archive file missing: {zip_path}")
                    # Skip jobs that were in progress (they failed during restart)
                    elif job_data.get('status') in ['pending', 'processing']:
                        skipped_count += 1
                        print(f"DEBUG: Skipping incomplete job {job_id} - status: {job_data.get('status')}")
                    # Keep error state jobs for visibility
                    elif job_data.get('status') == 'error':
                        valid_jobs[job_id] = job_data
                
                job_queue = valid_jobs
                print(f"DEBUG: Loaded {len(job_queue)} jobs from persistent storage ({skipped_count} skipped)")
                
                # Save the cleaned up job list back to disk
                if skipped_count > 0:
                    save_jobs()
        else:
            print("DEBUG: No existing jobs file found, starting fresh")
    except Exception as e:
        print(f"ERROR: Failed to load jobs: {e}")
        job_queue = {}

# Load jobs on startup
load_jobs()

# Job cleanup settings
JOB_RETENTION_DAYS = 7
SAVE_BATCH_SIZE = 5  # Only save after this many updates
save_counter = 0

def update_job_status(job_id, status, **kwargs):
    """Update job status and persist changes"""
    with job_lock:
        if job_id in job_queue:
            job_queue[job_id]['status'] = status
            job_queue[job_id].update(kwargs)
            job_queue[job_id]['updated_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            
            # Only save for important status changes
            if status in ['completed', 'error', 'processing']:
                _save_jobs_unlocked()  # Use unlocked version since we already hold the lock

def cleanup_old_jobs():
    """Remove jobs older than JOB_RETENTION_DAYS"""
    from datetime import timedelta
    cutoff = datetime.now() - timedelta(days=JOB_RETENTION_DAYS)
    
    with job_lock:
        jobs_to_remove = []
        for job_id, job in list(job_queue.items()):
            if job.get('status') == 'completed':
                try:
                    created_at = datetime.strptime(job.get('created_at', ''), '%Y-%m-%d %H:%M:%S')
                    if created_at < cutoff:
                        # Clean up associated files
                        if job.get('zip_path') and os.path.exists(job['zip_path']):
                            try:
                                os.remove(job['zip_path'])
                            except Exception as e:
                                print(f"Error removing ZIP file: {e}")
                        
                        # Clean up uploaded files
                        for file_info in job.get('files', []):
                            if 'path' in file_info and os.path.exists(file_info['path']):
                                try:
                                    os.remove(file_info['path'])
                                except Exception as e:
                                    print(f"Error removing file: {e}")
                        
                        jobs_to_remove.append(job_id)
                except (ValueError, TypeError) as e:
                    print(f"Error parsing date for job {job_id}: {e}")
        
        for job_id in jobs_to_remove:
            del job_queue[job_id]
            print(f"Cleaned up old job: {job_id}")
        
        if jobs_to_remove:
            _save_jobs_unlocked()  # Use unlocked version since we already hold the lock
            print(f"Removed {len(jobs_to_remove)} old jobs")

def save_jobs_batched():
    """Save jobs to disk with batching to reduce I/O"""
    global save_counter
    save_counter += 1
    
    # Save immediately for critical operations, otherwise batch
    if save_counter >= SAVE_BATCH_SIZE:
        save_jobs()
        save_counter = 0

def analyze_failed_extractions(log_path):
    """
    Analyze failed extraction log to identify new patterns.
    Returns suggestions for new regex patterns.
    """
    import re
    from collections import defaultdict

    if not os.path.exists(log_path):
        return "No failed extraction log found."

    patterns_found = defaultdict(int)

    try:
        with open(log_path, 'r', encoding='utf-8') as f:
            content = f.read()

        # Split into individual failed extractions
        failed_entries = content.split('-' * 50)

        for entry in failed_entries:
            if 'Text content:' in entry:
                text_part = entry.split('Text content:')[1].strip()
                lines = text_part.split('\n')

                # Analyze line patterns
                for i, line in enumerate(lines):
                    line = line.strip()
                    if not line:
                        continue

                    # Look for potential patterns
                    # Gender + something
                    if re.match(r'^(Female|Male)', line):
                        patterns_found['gender_start'] += 1

                    # School + something
                    if re.search(r'(?:High|School|Academy|Middle|Elementary)', line):
                        patterns_found['school_in_line'] += 1

                    # Grade + something
                    if re.match(r'^\d{1,2}', line):
                        patterns_found['grade_start'] += 1

                    # Names (capitalized words)
                    name_words = re.findall(r'\b([A-Z][a-z]+)\b', line)
                    if len(name_words) >= 2:
                        patterns_found['multiple_names'] += 1

                    # Check for concatenated patterns
                    if re.match(r'^[A-Z][a-z]+[A-Z][a-z]+', line):
                        patterns_found['concatenated_names'] += 1

    except Exception as e:
        return f"Error analyzing log: {str(e)}"

    # Generate suggestions
    suggestions = []
    if patterns_found.get('gender_start', 0) > 0:
        suggestions.append("Found gender at start of lines - may need new gender-first patterns")
    if patterns_found.get('school_in_line', 0) > 0:
        suggestions.append("Found school names in lines - may need school-based patterns")
    if patterns_found.get('grade_start', 0) > 0:
        suggestions.append("Found grades at start of lines - may need grade-first patterns")
    if patterns_found.get('multiple_names', 0) > 0:
        suggestions.append("Found multiple capitalized words - may need better name parsing")
    if patterns_found.get('concatenated_names', 0) > 0:
        suggestions.append("Found concatenated names - may need better concatenation handling")

    if not suggestions:
        suggestions.append("No obvious patterns detected - may need manual inspection")

def map_pattern_1(match1, match2):
    return {
        'gender': match1.group(1),
        'school': match1.group(2).strip(),
        'last_name': match1.group(3),
        'grade': match2.group(1),
        'first_name': match2.group(2)
    }

def map_pattern_2(match1, match2, match3):
    return {
        'first_name': match1.group(1),
        'gender': match2.group(1),
        'school': match2.group(2).strip(),
        'last_name': match2.group(3),
        'grade': match3.group(1)
    }

def map_pattern_3(match1, match2, match3):
    return {
        'gender': match1.group(1),
        'first_name': match1.group(2),
        'school': match2.group(1).strip(),
        'last_name': match2.group(2),
        'grade': match3.group(1)
    }

def map_pattern_4(match1, match2, match3):
    return {
        'school': match1.group(1).strip(),
        'grade': match1.group(2),
        'gender': match2.group(1),
        'first_name': match2.group(2),
        'last_name': match3.group(1)
    }

def map_pattern_5(match1, match2, match3):
    return {
        'first_name': match1.group(1),
        'gender': match2.group(1),
        'grade': match2.group(2),
        'school': match3.group(1).strip(),
        'last_name': match3.group(2)
    }

def map_pattern_6(match1, match2, match3):
    return {
        'first_name': match1.group(1),
        'school': match2.group(1).strip(),
        'grade': match2.group(2),
        'last_name': match2.group(3),
        'gender': match3.group(1)
    }

def map_pattern_7(match1, match2):
    return {
        'gender': match1.group(1),
        'first_name': match1.group(2),
        'school': match2.group(1).strip(),
        'grade': match2.group(2),
        'last_name': match2.group(3)
    }

def map_pattern_8(match1, match2, match3):
    return {
        'school': match1.group(1).strip(),
        'first_name': match1.group(2),
        'last_name': match2.group(1),
        'gender': match3.group(1),
        'grade': match3.group(2)
    }

def map_pattern_9(match1, match2, match3, match4):
    return {
        'first_name': match1.group(2),
        'last_name': match1.group(1),
        'grade': match2.group(1),
        'school': match3.group(1).strip(),
        'gender': match4.group(1)
    }

def map_pattern_10(match1, match2):
    return {
        'gender': match1.group(1),
        'school': match1.group(2).strip(),
        'last_name': match1.group(3),
        'grade': match2.group(1),
        'first_name': match2.group(2)
    }

ALLOWED_EXTENSIONS = {'pdf', 'zip'}
ALLOWED_MIME_TYPES = {
    'pdf': 'application/pdf',
    'zip': 'application/zip'
}

# Global jobs storage (in production, use database)
job_queue = {}

def login_required(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        print(f"DEBUG login_required: session = {dict(session)}")
        if 'user' not in session and 'admin' not in session:
            print("DEBUG: No user or admin in session, redirecting to login")
            return redirect(url_for('login'))
        print(f"DEBUG: User authenticated: {session.get('user', session.get('admin'))}")
        return f(*args, **kwargs)
    return decorated_function

def allowed_file(filename, file_stream=None):
    """
    Validate file based on extension and optionally MIME type.
    
    Args:
        filename: The name of the file
        file_stream: Optional file stream to check MIME type
    
    Returns:
        bool: True if file is allowed, False otherwise
    """
    # Check extension
    if '.' not in filename:
        return False
    
    ext = filename.rsplit('.', 1)[1].lower()
    if ext not in ALLOWED_EXTENSIONS:
        return False
    
    # Additional MIME type validation if file stream provided
    if file_stream:
        try:
            import magic
            file_stream.seek(0)
            mime_type = magic.from_buffer(file_stream.read(2048), mime=True)
            file_stream.seek(0)
            
            expected_mime = ALLOWED_MIME_TYPES.get(ext)
            if expected_mime and mime_type != expected_mime:
                print(f"MIME type mismatch: expected {expected_mime}, got {mime_type}")
                return False
        except ImportError:
            # python-magic not installed, skip MIME check
            pass
        except Exception as e:
            print(f"Error checking MIME type: {e}")
            return False
    
    return True

def extract_names_from_pdf(pdf_path):
    reader = None
    try:
        reader = PdfReader(pdf_path)

        # All PDFs are flattened, so only try text extraction from first page

        if len(reader.pages) > 0:
            first_page_text = reader.pages[0].extract_text()
            extracted_data = extract_student_info_from_text(first_page_text)

            if extracted_data and all(extracted_data.values()):
                return {
                    'filename': f"{extracted_data['last_name']}_{extracted_data['first_name']}_{extracted_data['grade']}.pdf",
                    'last_name': extracted_data['last_name'],
                    'first_name': extracted_data['first_name'],
                    'grade': extracted_data['grade'],
                    'school': extracted_data['school'],
                    'gender': extracted_data['gender']
                }
            # If extraction failed, include raw text for logging
            return {
                'error': 'Could not extract student information from PDF. Text extraction failed.',
                'raw_text': first_page_text or ''
            }
        # If we get here, we couldn't extract the data or text
        return {'error': 'Could not extract student information from PDF. No text found.'}

    except Exception as e:
        return {'error': f'PDF read error: {str(e)}'}
    finally:
        # Properly close PDF to prevent resource leaks
        if reader is not None:
            try:
                reader.stream.close()
            except:
                pass

def extract_student_info_from_text(text):
    import re
    extracted = {
        'first_name': '',
        'last_name': '',
        'grade': '',
        'school': '',
        'gender': ''
    }
    lines = text.split('\n')
    
    # --- Custom patterns for failed extractions ---
    # Pattern: GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
    for i in range(len(lines) - 1):
        m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', lines[i])
        m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', lines[i+1])
        if m1 and m2:
            extracted['gender'] = m1.group(1)
            extracted['school'] = m1.group(2).strip()
            extracted['last_name'] = m1.group(3)
            extracted['grade'] = m2.group(1)
            extracted['first_name'] = m2.group(2)
            break

    # Pattern: ConcatenatedNames, Grade, School, Gender (e.g., MaynardJenny, 11, Liberty Christian Academy, Female)
    for i in range(len(lines) - 3):
        m1 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i])
        m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
        m3 = lines[i+2].strip()
        m4 = re.match(r'^(Female|Male)$', lines[i+3])
        if m1 and m2 and m3 and m4:
            extracted['last_name'] = m1.group(1)
            extracted['first_name'] = m1.group(2)
            extracted['grade'] = m2.group(1)
            extracted['school'] = m3
            extracted['gender'] = m4.group(1)
            break

    # Pattern: SchoolFirstName, LastName, GenderGrade (e.g., Liberty Christian AcademySally, Sue, Female9)
    for i in range(len(lines) - 2):
        m1 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i])
        m2 = re.match(r'^([A-Z][a-z]+)$', lines[i+1])
        m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2])
        if m1 and m2 and m3:
            extracted['school'] = m1.group(1).strip()
            extracted['first_name'] = m1.group(2)
            extracted['last_name'] = m2.group(1)
            extracted['gender'] = m3.group(1)
            extracted['grade'] = m3.group(2)
            break

    # Enhanced preprocessing: aggressively split lines with multiple fields stuck together
    processed_lines = []
    skip_headers = [
        'student first name:', 'student last name:', 'student grade:',
        'student school:', 'details', 'gender:', 'docusign envelope id:',
        'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian'
    ]
    valid_schools = [
        'Amherst', 'Appomattox', 'Brookville', 'Dunbar', 'E.C. Glass', 'Forest', 'Heritage', 'Home School',
        'Jefferson Forest', 'Liberty Christian Academy', 'Linkhorne', 'Monelison', 'Rustburg', 'Sandusky',
        'Temple Christian', 'Timberlake Christian'
    ]
    school_suffixes = [' High', ' Middle', ' Academy', ' Elementary', ' School', ' Christian']
    all_school_names = sorted([base + suffix for base in valid_schools for suffix in [''] + school_suffixes], key=len, reverse=True)
    
    for line in lines:
        orig_line = line.strip()
        # Ignore header/title/label lines
        if not orig_line or any(h in orig_line.lower() for h in skip_headers):
            continue
        # Aggressive split: if line starts with a valid school name (with suffix) and is immediately followed by a capitalized name or digit, split
        split_done = False
        for school_name in all_school_names:
            if orig_line.startswith(school_name):
                rest = orig_line[len(school_name):]
                if rest and (re.match(r'^[A-Z]', rest) or re.match(r'^\d', rest)):
                    processed_lines.append(school_name)
                    processed_lines.append(rest)
                    split_done = True
                    break
        if split_done:
            continue
        # --- Always split off leading gender ---
        m_gender = re.match(r'^(Female|Male)(.+)', orig_line)
        if m_gender:
            processed_lines.append(m_gender.group(1))
            orig_line = m_gender.group(2).strip()
        # --- Split at last school suffix if followed by capitalized name or number ---
        last_pos = -1
        last_suffix = ''
        for suffix in school_suffixes:
            idx = orig_line.rfind(suffix)
            if idx != -1 and idx + len(suffix) > last_pos:
                last_pos = idx + len(suffix)
                last_suffix = suffix
        if last_pos != -1 and last_pos < len(orig_line):
            # Always split at the last suffix: everything after is not part of the school name
            processed_lines.append(orig_line[:last_pos].strip())
            trailing = orig_line[last_pos:].strip()
            if trailing:
                # Split all words (including lowercase, apostrophes, spaces, etc.) as separate lines
                for part in re.findall(r"[\w'\-]+", trailing):
                    processed_lines.append(part)
            continue
        # Pattern: Gender + School + LastName (e.g., 'FemaleBrookville HighMouse')
        m = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)$', orig_line)
        if m:
            processed_lines.append(m.group(1))
            processed_lines.append(m.group(2).strip())
            processed_lines.append(m.group(3))
            continue
        # Pattern: Grade + FirstName (e.g., '11Mickey')
        m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', orig_line)
        if m2:
            processed_lines.append(m2.group(1))
            processed_lines.append(m2.group(2))
            continue
        # Pattern: Two concatenated names (e.g., 'MaynardJenny')
        m3 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
        if m3 and orig_line not in processed_lines:
            processed_lines.append(m3.group(1))
            processed_lines.append(m3.group(2))
            continue
        # Pattern: Gender + FirstName (e.g., 'MaleJoshua')
        m7 = re.match(r'^(Female|Male)([A-Z][a-z]+)$', orig_line)
        if m7:
            processed_lines.append(m7.group(1))
            processed_lines.append(m7.group(2))
            continue
        # Pattern: Gender + School + FirstName + LastName + Grade (all concatenated)
        m4 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)(\d{1,2}|K)$', orig_line)
        if m4:
            processed_lines.append(m4.group(1))
            processed_lines.append(m4.group(2).strip())
            processed_lines.append(m4.group(3))
            processed_lines.append(m4.group(4))
            processed_lines.append(m4.group(5))
            continue
        # Pattern: Gender + School + FirstName + LastName (no grade)
        m5 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
        if m5:
            processed_lines.append(m5.group(1))
            processed_lines.append(m5.group(2).strip())
            processed_lines.append(m5.group(3))
            processed_lines.append(m4.group(4))
            continue
        # Pattern: Grade + Gender + School + FirstName + LastName (e.g., '10FemaleHomeSchoolAbigailDober')
        m6 = re.match(r'^(\d{1,2}|K)(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
        if m6:
            processed_lines.append(m6.group(1))
            processed_lines.append(m6.group(2))
            processed_lines.append(m6.group(3).strip())
            processed_lines.append(m6.group(4))
            processed_lines.append(m6.group(5))
            continue
        # Otherwise, keep line as is
        processed_lines.append(orig_line)
    # Remove empty lines
    lines = [l for l in processed_lines if l.strip()]

    # First pass: Look for individual fields that appear on separate lines
    for line in lines:
        line = line.strip()
        if not line:
            continue

        # Look for grade (numbers, including K-12)
        grade_match = re.match(r'^(?:Grade\s*)?(\d{1,2}|K|k|\d+(?:st|nd|rd|th))$', line)
        if grade_match:
            grade = grade_match.group(1).upper()
            if grade in ['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']:
                extracted['grade'] = grade

        # Look for gender (case insensitive, handle common variations)
        if re.match(r'^(Male|Female|M|F|Boy|Girl)$', line, re.IGNORECASE):
            gender = line.capitalize()
            if gender in ['Male', 'Female']:
                extracted['gender'] = gender

        # Look for school names (more flexible pattern)
        school_match = re.match(r'^(.+?)(?:\s+(?:High|School|Academy|Middle|Elementary|Christian|Catholic|Public|Private))$', line, re.IGNORECASE)
        if school_match and len(school_match.group(1)) > 3:  # Avoid false positives
            extracted['school'] = line

        # If line is all uppercase and not a known field, treat as possible school
        if line.isupper() and len(line) > 5 and not any(x in line for x in ['STUDENT', 'NAME', 'GRADE', 'GENDER', 'DETAILS', 'DOCUSIGN', 'ENVELOPE', 'ID']):
            extracted['school'] = line.title()

    # Second pass: Try various concatenation patterns
    patterns = [
        # Pattern 1: Gender + School + LastName, Grade + FirstName
        {
            'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
            'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$',
            'map': lambda m1, m2: {
                'gender': m1.group(1),
                'school': m1.group(2).strip(),
                'last_name': m1.group(3),
                'grade': m2.group(1),
                'first_name': m2.group(2)
            }
        },
        # Pattern 2: FirstName, Gender + School + LastName, Grade
        {
            'regex1': r'^([A-Z][a-z]+)$',
            'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
            'regex3': r'^(\d{1,2}|K)$',
            'map': lambda m1, m2, m3: {
                'first_name': m1.group(1),
                'gender': m2.group(1),
                'school': m2.group(2).strip(),
                'last_name': m2.group(3),
                'grade': m3.group(1)
            }
        },
        # Pattern 3: Gender + FirstName, School + LastName, Grade
        {
            'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
            'regex2': r'^(.+?)([A-Z][a-z]+)$',
            'regex3': r'^(\d{1,2}|K)$',
            'map': lambda m1, m2, m3: {
                'gender': m1.group(1),
                'first_name': m1.group(2),
                'school': m2.group(1).strip(),
                'last_name': m2.group(2),
                'grade': m3.group(1)
            }
        },
        # Pattern 4: School + Grade, Gender + FirstName, LastName
        {
            'regex1': r'^(.+?)(\d{1,2}|K)$',
            'regex2': r'^(Female|Male)([A-Z][a-z]+)$',
            'regex3': r'^([A-Z][a-z]+)$',
            'map': lambda m1, m2, m3: {
                'school': m1.group(1).strip(),
                'grade': m1.group(2),
                'gender': m2.group(1),
                'first_name': m2.group(2),
                'last_name': m3.group(1)
            }
        },
        # Pattern 5: FirstName, Gender + Grade, School + LastName
        {
            'regex1': r'^([A-Z][a-z]+)$',
            'regex2': r'^(Female|Male)(\d{1,2}|K)$',
            'regex3': r'^(.+?)([A-Z][a-z]+)$',
            'map': lambda m1, m2, m3: {
                'first_name': m1.group(1),
                'gender': m2.group(1),
                'grade': m2.group(2),
                'school': m3.group(1).strip(),
                'last_name': m3.group(2)
            }
        },
        # Pattern 6: FirstName, School + Grade + LastName, Gender
        {
            'regex1': r'^([A-Z][a-z]+)$',
            'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
            'regex3': r'^(Female|Male)$',
            'map': lambda m1, m2, m3: {
                'first_name': m1.group(1),
                'school': m2.group(1).strip(),
                'grade': m2.group(2),
                'last_name': m2.group(3),
                'gender': m3.group(1)
            }
        },
        # Pattern 7: Gender + FirstName, School + Grade + LastName
        {
            'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
            'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
            'map': lambda m1, m2: {
                'gender': m1.group(1),
                'first_name': m1.group(2),
                'school': m2.group(1).strip(),
                'grade': m2.group(2),
                'last_name': m2.group(3)
            }
        },
    ]

    for pattern in patterns:
        if 'regex3' in pattern:
            for i in range(len(lines) - 2):
                m1 = re.match(pattern['regex1'], lines[i])
                m2 = re.match(pattern['regex2'], lines[i+1])
                m3 = re.match(pattern['regex3'], lines[i+2])
                if m1 and m2 and m3:
                    mapped = pattern['map'](m1, m2, m3)
                    extracted.update(mapped)
                    break
        else:
            for i in range(len(lines) - 1):
                m1 = re.match(pattern['regex1'], lines[i])
                m2 = re.match(pattern['regex2'], lines[i+1])
                if m1 and m2:
                    mapped = pattern['map'](m1, m2)
                    extracted.update(mapped)
                    break
        if all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
            break

    # Post-processing and validation
    required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
    if all(extracted.get(field) for field in required_fields):
        # Clean up school name: remove any trailing capitalized names or numbers
        extracted['school'] = re.sub(r'([A-Z][a-z]+|\d+)$', '', extracted['school']).strip()
        # Remove any names from school if they match first/last name
        for n in [extracted['first_name'], extracted['last_name']]:
            if n:
                extracted['school'] = re.sub(rf'\b{re.escape(n)}\b', '', extracted['school']).strip()
        # Final check: if school is just a number or a name, discard it
        if re.fullmatch(r'(\d{1,2}|K)', extracted['school']) or re.fullmatch(r'([A-Z][a-z]+|[A-Z\-]+|[A-Z][a-z]+\s[A-Z][a-z]+)+', extracted['school']):
            extracted['school'] = ''
        extracted['school'] = extracted['school'].strip()
        return extracted

    # Fallback: Try to extract fields from any order in the lines (for highly concatenated or out-of-order data)
    forbidden_names = set(['School', 'Home', 'High', 'Middle', 'Academy', 'Christian', 'Temple', 'Rustburg', 'Liberty', 'Forest', 'Heritage', 'Amherst', 'Appomattox', 'Dunbar', 'Glass', 'Jefferson', 'Linkhorne', 'Monelison', 'Sandusky', 'Timberlake'])
    gender_words = set(['Male', 'Female'])
    grade_words = set([str(i) for i in range(1,13)] + ['K'])
    school_name = extracted['school'].lower()
    names = []
    for line in lines:
        for word in line.split():
            w_lower = word.lower()
            # Exclude if word matches or contains any part of the school name
            if (re.match(r'^[A-Z][a-z]+$', word)
                and word not in forbidden_names
                and word not in gender_words
                and word not in grade_words
                and school_name not in w_lower
                and w_lower not in school_name):
                names.append(word)
    # Require two distinct names
    if len(names) >= 2 and names[0] != names[1]:
        if not extracted['first_name']:
            extracted['first_name'] = names[0]
        if not extracted['last_name']:
            extracted['last_name'] = names[1]
    else:
        return None
    # Look for grade
    for line in lines:
        m = re.search(r'(\d{1,2}|K)', line)
        if m and not extracted['grade']:
            extracted['grade'] = m.group(1)
    # Look for gender
    for line in lines:
        if 'female' in line.lower() and not extracted['gender']:
            extracted['gender'] = 'Female'
        elif 'male' in line.lower() and not extracted['gender']:
            extracted['gender'] = 'Male'
    # Look for school (any line with 'school', 'academy', 'high', etc.)
    for line in lines:
        if any(x in line.lower() for x in ['school', 'academy', 'high', 'middle', 'elementary', 'christian', 'catholic', 'public', 'private']):
            extracted['school'] = line.strip()
            break
    if all(extracted.get(field) for field in required_fields):
        extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
        return extracted

    return None

def process_job(job_id, files):
    temp_extract_dirs = []
    try:
        print("=== DEBUG: process_job STARTED ===")
        with job_lock:
            job_queue[job_id]['status'] = 'processing'
            job_queue[job_id]['progress'] = 'Starting job processing...'
        save_jobs()  # Persist initial progress
        # Create base Episode directory for this job
        episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'Episode')
        os.makedirs(episode_dir, exist_ok=True)
        processed_files = []
        skipped_dir = os.path.join(episode_dir, 'Skipped')
        os.makedirs(skipped_dir, exist_ok=True)
        skipped_log_path = os.path.join(skipped_dir, 'skipped_files.log')
        job_queue[job_id]['progress'] = 'Created output directory structure'
        
        # Analyze ZIP contents before processing
        zip_contents = {}
        job_queue[job_id]['progress'] = 'Analyzing uploaded files...'
        for file_info in files:
            if file_info['name'].lower().endswith('.zip'):
                try:
                    with zipfile.ZipFile(file_info['path'], 'r') as zip_ref:
                        pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')]
                        zip_contents[file_info['name']] = pdf_files
                except Exception as e:
                    zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"]
        
        job_queue[job_id]['zip_contents'] = zip_contents
        
        total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \
                      sum(len(zip_contents.get(file_info['name'], [])) 
                          for file_info in files if file_info['name'].lower().endswith('.zip'))
        
        job_queue[job_id]['progress'] = f'Found {total_files} files to process'
        save_jobs()  # Persist file count
        
        processed_count = 0
        
        for file_info in files:
            filepath = file_info['path']
            filename = file_info['name']
            
            if filename.lower().endswith('.zip'):
                try:
                    job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
                    print(f"DEBUG: Starting ZIP extraction for {filename}")
                    # Extract ZIP
                    extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
                    os.makedirs(extract_dir, exist_ok=True)
                    temp_extract_dirs.append(extract_dir)
                    print(f"DEBUG: Created extract directory: {extract_dir}")
                    with zipfile.ZipFile(filepath, 'r') as zip_ref:
                        file_count = len(zip_ref.namelist())
                        print(f"DEBUG: ZIP contains {file_count} files: {zip_ref.namelist()}")
                        zip_ref.extractall(extract_dir)
                    job_queue[job_id]['progress'] = f'Extracted {file_count} files from {filename}'
                    save_jobs()  # Persist ZIP extraction progress
                    print(f"DEBUG: ZIP extraction completed, progress updated: {job_queue[job_id]['progress']}")
                    
                    # Process PDFs in extract_dir
                    pdf_files_found = []
                    for root, dirs, files_in_zip in os.walk(extract_dir):
                        for f in files_in_zip:
                            if f.lower().endswith('.pdf'):
                                pdf_files_found.append(f)
                                pdf_path = os.path.join(root, f)
                                job_queue[job_id]['progress'] = f'Processing PDF: {f}'
                                save_jobs()  # Persist current file progress
                                print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
                                print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
                                # Skip MacOS resource fork files (._*)
                                if f.startswith('._'):
                                    print(f"DEBUG: Skipping MacOS resource fork file: {f}")
                                    continue
                                pdf_info = extract_names_from_pdf(pdf_path)
                                print(f"DEBUG: PDF extraction result: {pdf_info}")
                                if 'error' in pdf_info:
                                    # Move skipped file to Skipped/
                                    skipped_pdf_path = os.path.join(skipped_dir, f)
                                    try:
                                        os.rename(pdf_path, skipped_pdf_path)
                                    except Exception as move_err:
                                        print(f"WARNING: Could not move skipped PDF {f}: {move_err}")
                                    # Log skip reason and raw text if available
                                    with open(skipped_log_path, 'a', encoding='utf-8') as slog:
                                        slog.write(f"File: {f}\nReason: {pdf_info['error']}\n")
                                        if 'raw_text' in pdf_info:
                                            slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
                                        slog.write("-"*50 + "\n")
                                    # Store reason and raw_text for web GUI
                                    processed_files.append(('skipped', f, pdf_info['error'], pdf_info.get('raw_text', '')))
                                    job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
                                    print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
                                else:
                                    job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
                                    print(f"DEBUG: Successfully extracted data for {pdf_info['first_name']} {pdf_info['last_name']}")
                                    # Create folder structure: Episode/School/Grade/Gender/
                                    school_dir = os.path.join(episode_dir, pdf_info['school'])
                                    grade_dir = os.path.join(school_dir, pdf_info['grade'])
                                    gender_dir = os.path.join(grade_dir, pdf_info['gender'])
                                    print(f"DEBUG: Creating directories: {school_dir} -> {grade_dir} -> {gender_dir}")
                                    os.makedirs(gender_dir, exist_ok=True)
                                    print(f"DEBUG: Directories created successfully")
                                    
                                    # Move file to appropriate folder with new name
                                    new_path = os.path.join(gender_dir, pdf_info['filename'])
                                    print(f"DEBUG: Attempting to move {pdf_path} to {new_path}")
                                    counter = 1
                                    while os.path.exists(new_path):
                                        base, ext = os.path.splitext(pdf_info['filename'])
                                        new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
                                        counter += 1
                                        print(f"DEBUG: File exists, trying {new_path}")
                                    print(f"DEBUG: Moving file to {new_path}")
                                    os.rename(pdf_path, new_path)
                                    processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
                                    processed_count += 1
                                    job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
                                    # Batch saves - only every 5 files
                                    if processed_count % 5 == 0:
                                        save_jobs()
                                    print(f"DEBUG: Organized file {processed_count}/{total_files}")
                    
                    print(f"DEBUG: Found {len(pdf_files_found)} PDF files in ZIP: {pdf_files_found}")
                    
                except Exception as e:
                    processed_files.append(('skipped', filename, f'ZIP processing error: {str(e)}'))
                    job_queue[job_id]['progress'] = f'Failed to process ZIP {filename}: {str(e)}'
                    print(f"ZIP processing error for {filename}: {str(e)}")
                    continue
            else:
                # Single PDF
                try:
                    job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
                    save_jobs()  # Persist current file progress
                    # Skip MacOS resource fork files (._*)
                    if filename.startswith('._'):
                        print(f"DEBUG: Skipping MacOS resource fork file: {filename}")
                        continue
                    pdf_info = extract_names_from_pdf(filepath)
                    if 'error' in pdf_info:
                        # Move skipped file to Skipped/
                        skipped_pdf_path = os.path.join(skipped_dir, filename)
                        try:
                            os.rename(filepath, skipped_pdf_path)
                        except Exception as move_err:
                            print(f"WARNING: Could not move skipped PDF {filename}: {move_err}")
                        # Log skip reason and raw text if available
                        with open(skipped_log_path, 'a', encoding='utf-8') as slog:
                            slog.write(f"File: {filename}\nReason: {pdf_info['error']}\n")
                            if 'raw_text' in pdf_info:
                                slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
                            slog.write("-"*50 + "\n")
                        processed_files.append(('skipped', filename, pdf_info['error'], pdf_info.get('raw_text', '')))
                        job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
                    else:
                        job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
                        # Create folder structure: Episode/School/Grade/Gender/
                        school_dir = os.path.join(episode_dir, pdf_info['school'])
                        grade_dir = os.path.join(school_dir, pdf_info['grade'])
                        gender_dir = os.path.join(grade_dir, pdf_info['gender'])
                        os.makedirs(gender_dir, exist_ok=True)
                        
                        # Move file to appropriate folder with new name
                        new_filepath = os.path.join(gender_dir, pdf_info['filename'])
                        counter = 1
                        while os.path.exists(new_filepath):
                            base, ext = os.path.splitext(pdf_info['filename'])
                            new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
                            counter += 1
                        os.rename(filepath, new_filepath)
                        processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
                        processed_count += 1
                        job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
                        # Batch saves - only every 5 files
                        if processed_count % 5 == 0:
                            save_jobs()
                except Exception as e:
                    processed_files.append(('skipped', filename, f'PDF processing error: {str(e)}'))
                    job_queue[job_id]['progress'] = f'Failed to process PDF {filename}: {str(e)}'
                    print(f"PDF processing error for {filename}: {str(e)}")
                    continue
        
        job_queue[job_id]['progress'] = 'Creating ZIP archive...'
        # Create ZIP of the entire Episode directory structure
        # Always create ZIP, including Skipped/ folder if present
        try:
            zip_output = os.path.join(app.config['ARCHIVE_FOLDER'], f'renamed_{job_id}.zip')
            print(f"DEBUG: Creating ZIP at {zip_output}")
            with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
                for root, dirs, files_in_dir in os.walk(episode_dir):
                    for file in files_in_dir:
                        if file.startswith('.') or file.startswith('._'):
                            continue  # Ignore dotfiles and MacOS resource forks
                        file_path = os.path.join(root, file)
                        arcname = os.path.relpath(file_path, episode_dir)
                        print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
                        zip_out.write(file_path, arcname)
            job_queue[job_id]['zip_path'] = zip_output
            job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
            job_queue[job_id]['progress'] = f'Created ZIP archive with {len([p for p in processed_files if p[0] == 'success'])} organized files and {len([p for p in processed_files if p[0] == 'skipped'])} skipped files'
            save_jobs()  # Persist completion progress
            print(f"DEBUG: ZIP creation completed successfully")
            # Clean up Episode directory after successful ZIP creation
            try:
                import shutil
                shutil.rmtree(episode_dir)
                print(f"DEBUG: Cleaned up Episode directory: {episode_dir}")
            except Exception as cleanup_error:
                print(f"WARNING: Failed to clean up Episode directory: {cleanup_error}")
            # Clean up original uploaded files
            for file_info in files:
                try:
                    if os.path.exists(file_info['path']):
                        os.remove(file_info['path'])
                        print(f"DEBUG: Removed uploaded file: {file_info['path']}")
                except Exception as file_cleanup_error:
                    print(f"WARNING: Failed to remove uploaded file: {file_cleanup_error}")
        except Exception as e:
            job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
            job_queue[job_id]['status'] = 'error'
            save_jobs()  # Persist error status
            print(f"DEBUG: ZIP creation failed: {str(e)}")
        
        with job_lock:
            job_queue[job_id]['status'] = 'completed'
            job_queue[job_id]['processed_files'] = processed_files
            # Store skipped as (filename, reason, raw_text)
            job_queue[job_id]['skipped'] = [(p[1], p[2], p[3]) for p in processed_files if p[0] == 'skipped']
        save_jobs()  # Persist completed job
        print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")

    except Exception as e:
        with job_lock:
            job_queue[job_id]['status'] = 'error'
            job_queue[job_id]['progress'] = f'Job failed with error: {str(e)}'
        save_jobs()  # Persist error status
        print(f"Job {job_id} failed: {str(e)}")  # Debug logging
    finally:
        # Clean up temporary directories
        for temp_dir in temp_extract_dirs:
            if os.path.exists(temp_dir):
                try:
                    import shutil
                    shutil.rmtree(temp_dir)
                    print(f"DEBUG: Cleaned up temporary directory: {temp_dir}")
                except Exception as cleanup_error:
                    print(f"ERROR: Failed to clean up {temp_dir}: {cleanup_error}")

@app.route('/login')
def login():
    return render_template_string('''
    <!DOCTYPE html>
    <html>
    <head>
        <title>Login</title>
        <style>
            body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
            .login-container { background-color: #fff; padding: 30px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); display: inline-block; margin: 20px; }
            h1 { color: #333; }
            .login-options { display: flex; justify-content: space-around; flex-wrap: wrap; }
            .login-option { margin: 10px; }
            a { color: #007bff; text-decoration: none; padding: 15px 30px; background-color: #007bff; color: white; border-radius: 5px; display: inline-block; }
            a:hover { background-color: #0056b3; }
            form { margin-top: 20px; }
            input[type="text"], input[type="password"] { padding: 10px; margin: 5px; width: 200px; border: 1px solid #ccc; border-radius: 5px; }
            input[type="submit"] { background-color: #28a745; color: white; border: none; padding: 10px 20px; border-radius: 5px; cursor: pointer; }
            input[type="submit"]:hover { background-color: #218838; }
        </style>
    </head>
    <body>
        <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
        <h1>PDF Renamer Login</h1>
        <div class="login-options">
            <div class="login-option">
                <h2>Microsoft Entra ID</h2>
                <a href="/entra-login">Login with Microsoft</a>
            </div>
            <div class="login-option">
                <h2>Admin Login</h2>
                <form method="post" action="/admin-login">
                    <input type="text" name="username" placeholder="Username" required><br>
                    <input type="password" name="password" placeholder="Password" required><br>
                    <input type="submit" value="Admin Login">
                </form>
            </div>
        </div>
    </body>
    </html>
    ''')

@app.route('/entra-login')
def entra_login():
    # Check if Microsoft OAuth provider is available
    if microsoft is None:
        flash('Microsoft Entra ID authentication is not configured.', 'error')
        return redirect(url_for('login'))
    
    # Check if Microsoft Entra ID is configured
    if TENANT_ID in ['your-tenant-id', 'YOUR_MICROSOFT_ENTRA_TENANT_ID'] or \
       CLIENT_ID in ['your-client-id', 'YOUR_MICROSOFT_ENTRA_CLIENT_ID'] or \
       CLIENT_SECRET in ['your-client-secret', 'YOUR_MICROSOFT_ENTRA_CLIENT_SECRET']:
        flash('Microsoft Entra ID is not properly configured. Please check your environment variables.', 'error')
        return redirect(url_for('login'))

    try:
        redirect_uri = url_for('auth', _external=True)
        print(f"DEBUG: Initiating OAuth with redirect_uri: {redirect_uri}")
        return microsoft.authorize_redirect(redirect_uri)
    except Exception as e:
        print(f"ERROR: Failed to initiate OAuth: {e}")
        import traceback
        print(f"Traceback: {traceback.format_exc()}")
        flash('Failed to initiate Microsoft authentication. Please try again or use admin login.', 'error')
        return redirect(url_for('login'))

@app.route('/admin-login', methods=['POST'])
def admin_login():
    username = request.form.get('username')
    password = request.form.get('password')
    if username == ADMIN_USERNAME and password == ADMIN_PASSWORD:
        session['admin'] = True
        return redirect(url_for('upload_files'))
    else:
        return render_template_string('''
        <!DOCTYPE html>
        <html>
        <head>
            <title>Login Failed</title>
            <style>
                body { font-family: Arial, sans-serif; background-color: #f4f4f4; text-align: center; padding: 50px; }
                .error { color: #dc3545; background-color: #f8d7da; padding: 20px; border-radius: 5px; border: 1px solid #f5c6cb; }
                a { color: #007bff; text-decoration: none; }
            </style>
        </head>
        <body>
            <div class="error">
                <h1>Login Failed</h1>
                <p>Invalid username or password.</p>
                <a href="/login">Try Again</a>
            </div>
        </body>
        </html>
        ''')

@app.route('/auth')
def auth():
    try:
        print("DEBUG: Starting OAuth callback processing")
        token = microsoft.authorize_access_token()
        print(f"DEBUG: Got token: {token}")
        
        # Get user info from the token's userinfo (already parsed by Authlib)
        user_info = token.get('userinfo')
        
        if not user_info:
            print("ERROR: No userinfo in token")
            flash('Failed to get user information from Microsoft.', 'error')
            return redirect(url_for('login'))
        
        print(f"DEBUG: Parsed user info: {user_info}")
        
        # Store user information in session
        session['user'] = {
            'name': user_info.get('name', 'Unknown'),
            'email': user_info.get('email') or user_info.get('preferred_username', 'unknown@example.com'),
            'oid': user_info.get('oid'),
        }
        session.permanent = True
        
        print(f"DEBUG: User logged in successfully: {session['user']}")
        return redirect(url_for('upload_files'))
        
    except Exception as e:
        print(f"OAuth error: {e}")  # Log the error for debugging
        import traceback
        print(f"Full traceback: {traceback.format_exc()}")
        flash('Authentication failed. Please try again.', 'error')
        return redirect(url_for('login'))

@app.route('/logout')
@login_required
def logout():
    session.pop('user', None)
    session.pop('admin', None)
    return redirect(url_for('login'))

@app.route('/', methods=['GET', 'POST'])
@login_required
def upload_files():
    if request.method == 'POST':
        files = request.files.getlist('files')
        if not files or all(not f.filename for f in files):
            flash('No files selected.', 'error')
            return redirect(url_for('upload_files'))
        
        # Create job ID using timestamp format: YYYY-MM-DD_HH-MM-SS
        job_id = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
        job_queue[job_id] = {
            'status': 'queued',
            'files': [],
            'zip_path': None,
            'zip_name': None,
            'processed_files': [],
            'skipped': [],
            'created_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
            'job_id': job_id  # Store the job ID for reference
        }
        
        file_list = []
        for file in files:
            if file and not file.filename.startswith('.') and allowed_file(file.filename):
                filename = secure_filename(file.filename)
                filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
                file.save(filepath)
                file_list.append({'name': filename, 'path': filepath})
                print(f"Uploaded file: {filename} -> {filepath}")  # Debug logging
        
        if not file_list:
            flash('No valid files uploaded. Only PDF and ZIP files are allowed.', 'error')
            job_queue.pop(job_id, None)  # Remove empty job
            return redirect(url_for('upload_files'))
        
        job_queue[job_id]['files'] = file_list
        print(f"Created job {job_id} with {len(file_list)} files")  # Debug logging
        
        # Save job to persistent storage
        save_jobs()
        
        # Start processing in background
        def process_with_context():
            with app.app_context():
                process_job(job_id, file_list)
        
        thread = threading.Thread(target=process_with_context)
        thread.daemon = True
        thread.start()
        
        flash(f'Job created successfully. Processing {len(file_list)} files...', 'success')
        return redirect(url_for('jobs'))
    
    return render_template_string('''
    <!DOCTYPE html>
    <html>
    <head>
        <title>NextGenPDF - Automated PDF Organization</title>
        <style>
            body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
            img { margin-bottom: 20px; }
            h1 { color: #333; margin-bottom: 10px; }
            .subtitle { color: #666; font-size: 18px; margin-bottom: 30px; }
            form { background-color: #fff; padding: 20px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); display: inline-block; }
            input[type="file"] { margin: 10px 0; }
            input[type="submit"] { background-color: #28a745; color: white; border: none; padding: 10px 20px; border-radius: 5px; cursor: pointer; }
            input[type="submit"]:hover { background-color: #218838; }
            .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
            .logout:hover { background-color: #c82333; }
            .view-jobs-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 10px 0; }
            .view-jobs-btn:hover { background-color: #0056b3; }
            .info-box { background-color: #e7f3ff; border-left: 5px solid #007bff; padding: 20px; margin: 20px auto; max-width: 800px; text-align: left; border-radius: 5px; box-shadow: 0 2px 5px rgba(0,0,0,0.1); }
            .info-box h2 { color: #007bff; margin-top: 0; font-size: 24px; }
            .info-box p { color: #333; line-height: 1.6; margin: 10px 0; }
            .info-box ul { color: #333; line-height: 1.8; margin: 10px 0; padding-left: 20px; }
            .info-box li { margin: 5px 0; }
            .feature-icon { color: #28a745; font-weight: bold; margin-right: 5px; }
        </style>
    </head>
    <body>
        <a href="/logout" class="logout">Logout</a>
        <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
        <h1>NextGenPDF</h1>
        <p class="subtitle">Automated PDF Organization & Renaming System</p>
        
        <div class="info-box">
            <h2>📄 What Does NextGenPDF Do?</h2>
            <p><strong>NextGenPDF</strong> automatically extracts student information from PDF registration forms and organizes them into a structured folder system.</p>
            
            <p><strong>How it works:</strong></p>
            <ul>
                <li><span class="feature-icon">✓</span> Upload individual PDFs or ZIP files containing multiple PDFs</li>
                <li><span class="feature-icon">✓</span> Extracts student name, school, grade, and gender from each PDF</li>
                <li><span class="feature-icon">✓</span> Renames files to: <code>LastName_FirstName_Grade.pdf</code></li>
                <li><span class="feature-icon">✓</span> Organizes into folders: <code>Episode/School/Grade/Gender/</code></li>
                <li><span class="feature-icon">✓</span> Creates a downloadable ZIP archive with the organized structure</li>
                <li><span class="feature-icon">✓</span> Provides real-time progress updates during processing</li>
            </ul>
            
            <p><strong>Perfect for:</strong> Managing registration forms, organizing student documents, and automating file management workflows.</p>
        </div>
        
        <a href="/jobs" class="view-jobs-btn">View Jobs</a>
        <br><br>
        <form method="post" enctype="multipart/form-data">
            <input type="file" name="files" multiple accept=".pdf,.zip">
            <br><br>
            <input type="submit" value="Upload & Process Files">
        </form>
    </body>
    </html>
    ''')

@app.route('/jobs')
@login_required
def jobs():
    return render_template_string('''
    <!DOCTYPE html>
    <html>
    <head>
        <title>Jobs</title>
        <style>
            body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
            img { margin-bottom: 20px; }
            h1 { color: #333; }
            .job { background-color: #fff; margin: 10px auto; padding: 15px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); max-width: 800px; text-align: left; }
            .status { padding: 5px 10px; border-radius: 5px; color: white; font-weight: bold; }
            .status.queued { background-color: #ffc107; }
            .status.processing { background-color: #17a2b8; }
            .status.completed { background-color: #28a745; }
            .status.error { background-color: #dc3545; }
            .progress-text { color: #17a2b8; font-style: italic; }
            .download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; }
            .download-btn:hover { background-color: #0056b3; }
            .delete-btn { background-color: #dc3545; color: white; border: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; cursor: pointer; }
            .delete-btn:hover { background-color: #c82333; }
            .flash { padding: 10px; margin: 10px 0; border-radius: 5px; text-align: center; }
            .flash.success { background-color: #d4edda; color: #155724; border: 1px solid #c3e6cb; }
            .flash.error { background-color: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
            .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
            .logout:hover { background-color: #c82333; }
            .nav-links { margin-bottom: 20px; }
            .nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
            .nav-links a:hover { background-color: #e9ecef; }
            .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
            .nav-btn:hover { background-color: #0056b3; }
            .zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
            .zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
            .zip-toggle:hover { background-color: #dee2e6; }
            .zip-details { display: none; margin-top: 10px; }
            .skipped-section { background-color: #fff3cd; border: 1px solid #ffeaa7; border-radius: 5px; padding: 10px; margin: 10px 0; }
            .skipped-toggle { cursor: pointer; background-color: #ffeaa7; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #ffc107; font-weight: bold; color: #856404; }
            .skipped-toggle:hover { background-color: #ffc107; }
            .skipped-details { display: none; margin-top: 10px; }
            ul { list-style-type: none; padding: 0; }
            li { background-color: #ffffff; margin: 2px 0; padding: 5px; border-radius: 3px; }
        </style>
    </head>
    <body>
        <a href="/logout" class="logout">Logout</a>
        <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
        <h1>Processing Jobs</h1>
        <div class="nav-links" style="margin-bottom: 20px;">
            <a href="/" class="nav-btn">← Upload Files</a>
            <a href="/completed-jobs" class="nav-btn">View Completed Jobs Archive →</a>
        </div>
        {% with messages = get_flashed_messages(with_categories=true) %}
        {% if messages %}
        {% for category, message in messages %}
        <div class="flash {{ category }}">{{ message }}</div>
        {% endfor %}
        {% endif %}
        {% endwith %}
        {% for job_id, job in jobs.items() %}
        <div class="job">
            <h3>Job {{ job_id[:8] }}</h3>
            <p><strong>Status:</strong> <span class="status {{ job.status }}">{{ job.status.title() }}</span></p>
            {% if job.status == 'processing' and job.progress %}
            <p><strong>Progress:</strong> <span class="progress-text">{{ job.progress }}</span></p>
            {% endif %}
            <p><strong>Files:</strong> {{ job.files|length }}</p>
            {% if job.zip_contents %}
            <p><strong>ZIP Contents:</strong></p>
            {% for zip_name, pdf_files in job.zip_contents.items() %}
            <div class="zip-content">
                <div class="zip-toggle" onclick="toggleZipDetails(this)">
                    <strong>{{ zip_name }}</strong> ({{ pdf_files|length }} files) ▶
                </div>
                <div class="zip-details">
                    <ul>
                    {% for pdf_file in pdf_files %}
                        <li>{{ pdf_file }}</li>
                    {% endfor %}
                    </ul>
                </div>
            </div>
            {% endfor %}
            {% endif %}
            {% if job.status == 'completed' %}
                {% if job.zip_name %}
                <p><a href="/download/{{ job.zip_name }}" class="download-btn">Download ZIP</a>
                <form method="post" action="/delete/{{ job_id }}" style="display: inline;">
                    <button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
                </form></p>
                {% else %}
                <p><em>No files were successfully processed to create a ZIP archive.</em>
                <form method="post" action="/delete/{{ job_id }}" style="display: inline;">
                    <button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
                </form></p>
                {% endif %}
                {% if job.skipped %}
                <div class="skipped-section">
                    <div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
                        ⚠️ Skipped Files ({{ job.skipped|length }}) ▶
                    </div>
                    <div class="skipped-details">
                        <ul>
                        {% for skip_info in job.skipped %}
                            <li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}</li>
                        {% endfor %}
                        </ul>
                    </div>
                </div>
                {% endif %}
            {% endif %}
        </div>
        {% endfor %}
        {% if not jobs %}
        <p>No jobs yet. <a href="/">Upload some files</a> to get started.</p>
        {% endif %}
        <script>
            // Track previous job statuses to detect completion
            let previousJobStatuses = {};
            
            // Collapsible section functions
            function toggleZipDetails(element) {
                const details = element.nextElementSibling;
                const isVisible = details.style.display === 'block';
                details.style.display = isVisible ? 'none' : 'block';
                element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
            }
            
            function toggleSkippedDetails(element) {
                const details = element.nextElementSibling;
                const isVisible = details.style.display === 'block';
                details.style.display = isVisible ? 'none' : 'block';
                element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
            }
            
            function updateJobs() {
                console.log('[updateJobs] Starting update...');
                fetch('/api/jobs', { credentials: 'same-origin' })
                    .then(response => {
                        console.log('[updateJobs] Response received:', response.status);
                        if (!response.ok) {
                            throw new Error(`HTTP ${response.status}`);
                        }
                        return response.json();
                    })
                    .then(data => {
                        console.log('[updateJobs] Job data received:', data);
                        const jobs = Object.entries(data);
                        console.log('[updateJobs] Number of jobs:', jobs.length);
                        
                        // Check for job completions
                        let hasNewCompletion = false;
                        if (jobs.length > 0) {
                            jobs.forEach(([jobId, job]) => {
                                console.log(`[updateJobs] Job ${jobId}: status=${job.status}, progress=${job.progress || 'none'}`);
                                
                                // Check if job just completed
                                if (job.status === 'completed' && previousJobStatuses[jobId] === 'processing') {
                                    hasNewCompletion = true;
                                    console.log(`[updateJobs] Job ${jobId} just completed!`);
                                }
                                
                                // Update previous status
                                previousJobStatuses[jobId] = job.status;
                            });
                        }
                        
                        // Redirect to completed jobs if a job just finished
                        if (hasNewCompletion) {
                            console.log('[updateJobs] Redirecting to completed jobs page...');
                            setTimeout(() => {
                                window.location.href = '/completed-jobs';
                            }, 500); // Small delay to show completion briefly
                            return; // Stop processing to avoid UI updates during redirect
                        }
                        
                        console.log('[updateJobs] Starting DOM manipulation...');
                        // Find the container where jobs are rendered
                        const body = document.body;
                        const existingJobs = body.querySelectorAll('.job');
                        console.log(`[updateJobs] Found ${existingJobs.length} existing job elements`);
                        
                        // Remove existing job elements
                        existingJobs.forEach(job => {
                            console.log('[updateJobs] Removing job element');
                            job.remove();
                        });
                        
                        // Remove existing "No jobs yet" messages
                        const noJobsMsgs = body.querySelectorAll('p');
                        noJobsMsgs.forEach(p => {
                            if (p.textContent.includes('No jobs yet')) {
                                console.log('[updateJobs] Removing "no jobs" message');
                                p.remove();
                            }
                        });
                        
                        // Find where to insert new jobs (after nav-links div)
                        const navLinks = body.querySelector('.nav-links');
                        const scriptTag = body.querySelector('script');
                        console.log('[updateJobs] navLinks found:', !!navLinks, 'scriptTag found:', !!scriptTag);
                        
                        // Add updated job elements
                        const jobEntries = Object.entries(data);
                        console.log(`[updateJobs] Rendering ${jobEntries.length} jobs`);
                        
                        if (jobEntries.length > 0) {
                            jobEntries.forEach(([jobId, job]) => {
                                console.log(`[updateJobs] Creating element for job ${jobId}`);
                                try {
                                const jobDiv = document.createElement('div');
                                jobDiv.className = 'job';
                                
                                // Build the HTML step by step to avoid syntax errors
                                let jobHTML = `
                                    <h3>Job ${jobId.substring(0, 8)}</h3>
                                    <p><strong>Status:</strong> <span class="status ${job.status}">${job.status.charAt(0).toUpperCase() + job.status.slice(1)}</span></p>`;
                                
                                // Add progress if processing
                                if (job.status === 'processing' && job.progress) {
                                    jobHTML += `<p><strong>Progress:</strong> <span class="progress-text">${job.progress}</span></p>`;
                                }
                                
                                jobHTML += `<p><strong>Files:</strong> ${job.files ? job.files.length : 0}</p>`;
                                
                                // Add ZIP contents if available
                                if (job.zip_contents) {
                                    jobHTML += `<p><strong>ZIP Contents:</strong></p>`;
                                    Object.entries(job.zip_contents).forEach(([zipName, pdfFiles]) => {
                                        jobHTML += `
                                            <div class="zip-content">
                                                <div class="zip-toggle" onclick="toggleZipDetails(this)">
                                                    <strong>${zipName}</strong> (${pdfFiles.length} files) ▶
                                                </div>
                                                <div class="zip-details">
                                                    <ul>
                                                        ${pdfFiles.map(pdfFile => `<li>${pdfFile}</li>`).join('')}
                                                    </ul>
                                                </div>
                                            </div>`;
                                    });
                                }
                                
                                // Add download/delete buttons if completed
                                if (job.status === 'completed') {
                                    if (job.zip_name) {
                                        jobHTML += `
                                            <p><a href="/download/${job.zip_name}" class="download-btn">Download ZIP</a>
                                            <form method="post" action="/delete/${jobId}" style="display: inline;">
                                                <button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
                                            </form></p>`;
                                    } else {
                                        jobHTML += `
                                            <p><em>No files were successfully processed to create a ZIP archive.</em>
                                            <form method="post" action="/delete/${jobId}" style="display: inline;">
                                                <button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
                                            </form></p>`;
                                    }
                                    
                                    // Add skipped files section if any
                                    if (job.skipped && job.skipped.length > 0) {
                                        jobHTML += `
                                            <div class="skipped-section">
                                                <div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
                                                    ⚠️ Skipped Files (${job.skipped.length}) ▶
                                                </div>
                                                <div class="skipped-details">
                                                    <ul>
                                                        ${job.skipped.map(skipInfo => `<li><strong>${skipInfo[0]}</strong>: ${skipInfo[1]}</li>`).join('')}
                                                    </ul>
                                                </div>
                                            </div>`;
                                    }
                                }
                                
                                jobDiv.innerHTML = jobHTML;
                                console.log(`[updateJobs] HTML created for job ${jobId}`);
                                
                                // Insert job before the script tag
                                if (scriptTag) {
                                    body.insertBefore(jobDiv, scriptTag);
                                } else {
                                    body.appendChild(jobDiv);
                                }
                                console.log(`[updateJobs] Job ${jobId} element inserted into DOM`);
                                } catch (err) {
                                    console.error(`[updateJobs] Error creating job element for ${jobId}:`, err);
                                }
                            });
                        } else {
                            console.log('[updateJobs] No jobs to render, showing "no jobs" message');
                            const noJobsMsg = document.createElement('p');
                            noJobsMsg.innerHTML = 'No jobs yet. <a href="/">Upload some files</a> to get started.';
                            if (scriptTag) {
                                body.insertBefore(noJobsMsg, scriptTag);
                            } else {
                                body.appendChild(noJobsMsg);
                            }
                        }
                        console.log('[updateJobs] Update complete!');
                    })
                    .catch(function(error) {
                        console.error('[updateJobs] Error updating jobs:', error);
                        if (error.stack) {
                            console.error('[updateJobs] Error stack:', error.stack);
                        }
                    });
            }
            
            console.log('[Init] Setting up job update interval...');
            // Update jobs every 1 second
            const updateInterval = setInterval(updateJobs, 1000);
            console.log('[Init] Update interval set, ID:', updateInterval);
            
            // Initialize previous job statuses
            console.log('[Init] Running initial job update...');
            updateJobs(); // Run once immediately to populate initial statuses
        </script>
    </body>
    </html>
    ''', jobs=job_queue)

@app.route('/api/jobs')
@login_required
def api_jobs():
    return jsonify(job_queue)

@app.route('/download/<filename>')
@login_required
def download(filename):
    """Download a completed job's ZIP file with path traversal protection"""
    # Prevent path traversal attacks
    if '..' in filename or '/' in filename or '\\' in filename:
        return "Invalid filename", 400
    
    # Security check: only allow downloading files that belong to completed jobs
    with job_lock:
        for job_id, job in job_queue.items():
            if job.get('status') == 'completed' and job.get('zip_name') == filename:
                # Use ARCHIVE_FOLDER instead of UPLOAD_FOLDER
                file_path = os.path.join(app.config['ARCHIVE_FOLDER'], filename)
                # Verify the resolved path is within ARCHIVE_FOLDER
                resolved_path = os.path.abspath(file_path)
                archive_folder = os.path.abspath(app.config['ARCHIVE_FOLDER'])
                
                if not resolved_path.startswith(archive_folder):
                    return "Access denied", 403
                
                if os.path.exists(resolved_path):
                    return send_from_directory(app.config['ARCHIVE_FOLDER'], filename, as_attachment=True)
    
    return "File not found or access denied", 404

@app.route('/delete/<job_id>', methods=['POST'])
@login_required
def delete_job(job_id):
    import sys
    print(f"DEBUG: delete_job called for job_id: {job_id}", file=sys.stderr, flush=True)
    print(f"DEBUG: delete_job called for job_id: {job_id}", flush=True)
    
    try:
        print(f"DEBUG: Acquiring job_lock", file=sys.stderr, flush=True)
        with job_lock:
            print(f"DEBUG: Lock acquired, checking if job exists", file=sys.stderr, flush=True)
            if job_id in job_queue:
                job = job_queue[job_id]
                print(f"DEBUG: Job found with status: {job.get('status')}", file=sys.stderr, flush=True)
                
                # Only allow deletion of completed jobs
                if job.get('status') == 'completed':
                    print(f"DEBUG: Job is completed, proceeding with deletion", file=sys.stderr, flush=True)
                    
                    # Clean up the ZIP file from archives folder
                    if job.get('zip_name'):
                        print(f"DEBUG: ARCHIVE_FOLDER config: {app.config.get('ARCHIVE_FOLDER')}", file=sys.stderr, flush=True)
                        zip_path = os.path.join(app.config['ARCHIVE_FOLDER'], job['zip_name'])
                        print(f"DEBUG: Attempting to delete ZIP at: {zip_path}", file=sys.stderr, flush=True)
                        if os.path.exists(zip_path):
                            try:
                                os.remove(zip_path)
                                print(f"DEBUG: Deleted archive file: {zip_path}", file=sys.stderr, flush=True)
                            except Exception as e:
                                print(f"ERROR: Failed to delete archive: {e}", file=sys.stderr, flush=True)
                                import traceback
                                traceback.print_exc(file=sys.stderr)
                        else:
                            print(f"DEBUG: ZIP file does not exist: {zip_path}", file=sys.stderr, flush=True)
                    
                    # Clean up any remaining uploaded files
                    print(f"DEBUG: Cleaning up {len(job.get('files', []))} uploaded files", file=sys.stderr, flush=True)
                    for file_info in job.get('files', []):
                        if 'path' in file_info and os.path.exists(file_info['path']):
                            try:
                                os.remove(file_info['path'])
                                print(f"DEBUG: Deleted uploaded file: {file_info['path']}", file=sys.stderr, flush=True)
                            except Exception as e:
                                print(f"ERROR: Failed to delete uploaded file: {e}", file=sys.stderr, flush=True)
                    
                    # Remove the job from the queue
                    print(f"DEBUG: Removing job from queue", file=sys.stderr, flush=True)
                    del job_queue[job_id]
                    print(f"DEBUG: Saving jobs to disk", file=sys.stderr, flush=True)
                    _save_jobs_unlocked()  # Use unlocked version since we already hold the lock
                    print(f"DEBUG: Job deletion complete", file=sys.stderr, flush=True)
                else:
                    print(f"DEBUG: Job status is not completed: {job.get('status')}", file=sys.stderr, flush=True)
                    return "Only completed jobs can be deleted", 400
            else:
                print(f"DEBUG: Job not found in queue", file=sys.stderr, flush=True)
                return "Job not found", 404
    except Exception as e:
        print(f"ERROR: Exception in delete_job: {e}", file=sys.stderr, flush=True)
        print(f"ERROR: Exception in delete_job: {e}", flush=True)
        import traceback
        traceback.print_exc(file=sys.stderr)
        traceback.print_exc()
        return f"Error deleting job: {str(e)}", 500
    
    print(f"DEBUG: Redirecting to /jobs", file=sys.stderr, flush=True)
    try:
        return redirect('/jobs')
    except Exception as e:
        print(f"ERROR: Redirect failed: {e}", file=sys.stderr, flush=True)
        return "Job deleted but redirect failed", 200

@app.route('/completed-jobs')
@login_required
def completed_jobs():
    # Show ALL jobs with their statuses for debugging
    all_jobs = job_queue
    return render_template_string('''
    <!DOCTYPE html>
    <html>
    <head>
        <title>All Jobs Status</title>
        <style>
            body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
            img { margin-bottom: 20px; }
            h1 { color: #333; }
            .job { background-color: #fff; margin: 10px auto; padding: 15px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); max-width: 1000px; text-align: left; }
            .status { padding: 5px 10px; border-radius: 5px; color: white; font-weight: bold; }
            .status.completed { background-color: #28a745; }
            .status.processing { background-color: #17a2b8; }
            .status.error { background-color: #dc3545; }
            .progress-text { color: #17a2b8; font-style: italic; margin-left: 10px; }
            .download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin: 5px; display: inline-block; }
            .download-btn:hover { background-color: #0056b3; }
            .delete-btn { background-color: #dc3545; color: white; border: none; padding: 8px 16px; border-radius: 5px; margin: 5px; cursor: pointer; }
            .delete-btn:hover { background-color: #c82333; }
            .flash { padding: 10px; margin: 10px 0; border-radius: 5px; text-align: center; }
            .flash.success { background-color: #d4edda; color: #155724; border: 1px solid #c3e6cb; }
            .flash.error { background-color: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
            .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
            .logout:hover { background-color: #c82333; }
            .nav-links { margin-bottom: 20px; }
            .nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
            .nav-links a:hover { background-color: #e9ecef; }
            .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
            .nav-btn:hover { background-color: #0056b3; }
            .job-stats { background-color: #e9ecef; padding: 10px; border-radius: 5px; margin-bottom: 15px; }
            .file-count { color: #17a2b8; font-weight: bold; }
            .zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
            .zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
            .zip-toggle:hover { background-color: #dee2e6; }
            .zip-details { display: none; margin-top: 10px; }
            .skipped-section { background-color: #fff3cd; border: 1px solid #ffeaa7; border-radius: 5px; padding: 10px; margin: 10px 0; }
            .skipped-toggle { cursor: pointer; background-color: #ffeaa7; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #ffc107; font-weight: bold; color: #856404; }
            .skipped-toggle:hover { background-color: #ffc107; }
            .skipped-details { display: none; margin-top: 10px; }
            ul { list-style-type: none; padding: 0; }
            li { background-color: #ffffff; margin: 2px 0; padding: 5px; border-radius: 3px; }
        </style>
    </head>
    <body>
        <a href="/logout" class="logout">Logout</a>
        <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
        <h1>All Jobs Status</h1>
        <div class="nav-links">
            <a href="/" class="nav-btn">← Upload Files</a>
            <a href="/jobs" class="nav-btn">View All Jobs →</a>
        </div>
        {% with messages = get_flashed_messages(with_categories=true) %}
        {% if messages %}
        {% for category, message in messages %}
        <div class="flash {{ category }}">{{ message }}</div>
        {% endfor %}
        {% endif %}
        {% endwith %}
        {% for job_id, job in all_jobs.items() %}
        <div class="job">
            <h3>Job {{ job_id[:8] }} - {{ job.created_at if job.created_at else 'Unknown Date' }}</h3>
            <div class="job-stats">
                <span class="status {{ job.status }}">{{ job.status.title() }}</span>
                <span class="file-count">{{ job.processed_files|selectattr('0', 'equalto', 'success')|list|length }} files processed</span>
                {% if job.progress %}
                <span class="progress-text">Progress: {{ job.progress }}</span>
                {% endif %}
                {% if job.skipped %}
                <span style="color: #856404; font-weight: bold;">{{ job.skipped|length }} files skipped</span>
                {% endif %}
            </div>
            
            {% if job.zip_contents %}
            <p><strong>Original ZIP Contents:</strong></p>
            {% for zip_name, pdf_files in job.zip_contents.items() %}
            <div class="zip-content">
                <div class="zip-toggle" onclick="toggleZipDetails(this)">
                    <strong>{{ zip_name }}</strong> ({{ pdf_files|length }} files) ▶
                </div>
                <div class="zip-details">
                    <ul>
                    {% for pdf_file in pdf_files %}
                        <li>{{ pdf_file }}</li>
                    {% endfor %}
                    </ul>
                </div>
            </div>
            {% endfor %}
            {% endif %}
            
            {% if job.zip_name %}
            <p><strong>Download:</strong> <a href="/download/{{ job.zip_name }}" class="download-btn">📁 Download Organized ZIP ({{ job.zip_name }})</a></p>
            {% endif %}
            
            {% if job.skipped %}
            <div class="skipped-section">
                <div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
                    ⚠️ Skipped Files ({{ job.skipped|length }}) ▶
                </div>
                <div class="skipped-details">
                    <ul>
                    {% for skip_info in job.skipped %}
                        <li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}
                        {% if skip_info|length > 2 and skip_info[2] %}
                            <br><details><summary>Extracted Text</summary><pre style="white-space: pre-wrap; background: #f8f9fa; border: 1px solid #eee; padding: 5px;">{{ skip_info[2] }}</pre></details>
                        {% endif %}
                        </li>
                    {% endfor %}
                    </ul>
                </div>
            </div>
            {% endif %}
            
            <form method="post" action="/delete/{{ job_id }}" style="display: inline;">
                <button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to permanently delete this completed job and its files?')">🗑️ Delete Job</button>
            </form>
        </div>
        {% endfor %}
        {% if not jobs %}
        <p>No completed jobs yet. <a href="/">Upload some files</a> to get started.</p>
        {% endif %}
        <script>
            // Collapsible section functions
            function toggleZipDetails(element) {
                const details = element.nextElementSibling;
                const isVisible = details.style.display === 'block';
                details.style.display = isVisible ? 'none' : 'block';
                element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
            }
            
            function toggleSkippedDetails(element) {
                const details = element.nextElementSibling;
                const isVisible = details.style.display === 'block';
                details.style.display = isVisible ? 'none' : 'block';
                element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
            }
        </script>
    </body>
    </html>
    ''', all_jobs=all_jobs)

if __name__ == '__main__':
    app.run(debug=True, port=3000)