Nav updates

pdf_renamer.py

@@ -432,6 +432,7 @@ def extract_student_info_from_text(text):
 
 def process_job(job_id, files):
     try:
+        print("=== DEBUG: process_job STARTED ===")
         job_queue[job_id]['status'] = 'processing'
         job_queue[job_id]['progress'] = 'Starting job processing...'
         processed_files = []
@@ -444,6 +445,7 @@ def process_job(job_id, files):
         
         # Analyze ZIP contents before processing
         zip_contents = {}
+        job_queue[job_id]['progress'] = 'Analyzing uploaded files...'
         for file_info in files:
             if file_info['name'].lower().endswith('.zip'):
                 try:
@@ -459,6 +461,8 @@ def process_job(job_id, files):
                       sum(len(zip_contents.get(file_info['name'], [])) 
                           for file_info in files if file_info['name'].lower().endswith('.zip'))
         
+        job_queue[job_id]['progress'] = f'Found {total_files} files to process'
+        
         processed_count = 0
         
         for file_info in files:
@@ -466,46 +470,74 @@ def process_job(job_id, files):
             filename = file_info['name']
             
             if filename.lower().endswith('.zip'):
+                try:
                     job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
+                    print(f"DEBUG: Starting ZIP extraction for {filename}")
                     # Extract ZIP
                     extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
                     os.makedirs(extract_dir, exist_ok=True)
                     temp_extract_dirs.append(extract_dir)
+                    print(f"DEBUG: Created extract directory: {extract_dir}")
                     with zipfile.ZipFile(filepath, 'r') as zip_ref:
+                        file_count = len(zip_ref.namelist())
+                        print(f"DEBUG: ZIP contains {file_count} files: {zip_ref.namelist()}")
                         zip_ref.extractall(extract_dir)
-                job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}'
+                    job_queue[job_id]['progress'] = f'Extracted {file_count} files from {filename}'
+                    print(f"DEBUG: ZIP extraction completed, progress updated: {job_queue[job_id]['progress']}")
                     
                     # Process PDFs in extract_dir
+                    pdf_files_found = []
                     for root, dirs, files_in_zip in os.walk(extract_dir):
                         for f in files_in_zip:
                             if f.lower().endswith('.pdf'):
+                                pdf_files_found.append(f)
                                 pdf_path = os.path.join(root, f)
                                 job_queue[job_id]['progress'] = f'Processing PDF: {f}'
+                                print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
+                                print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
                                 pdf_info = extract_names_from_pdf(pdf_path)
+                                print(f"DEBUG: PDF extraction result: {pdf_info}")
                                 if 'error' in pdf_info:
                                     processed_files.append(('skipped', f, pdf_info['error']))
                                     job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
+                                    print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
                                 else:
                                     job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
+                                    print(f"DEBUG: Successfully extracted data for {pdf_info['first_name']} {pdf_info['last_name']}")
                                     # Create folder structure: Episode/School/Grade/Gender/
                                     school_dir = os.path.join(episode_dir, pdf_info['school'])
                                     grade_dir = os.path.join(school_dir, pdf_info['grade'])
                                     gender_dir = os.path.join(grade_dir, pdf_info['gender'])
+                                    print(f"DEBUG: Creating directories: {school_dir} -> {grade_dir} -> {gender_dir}")
                                     os.makedirs(gender_dir, exist_ok=True)
+                                    print(f"DEBUG: Directories created successfully")
                                     
                                     # Move file to appropriate folder with new name
                                     new_path = os.path.join(gender_dir, pdf_info['filename'])
+                                    print(f"DEBUG: Attempting to move {pdf_path} to {new_path}")
                                     counter = 1
                                     while os.path.exists(new_path):
                                         base, ext = os.path.splitext(pdf_info['filename'])
                                         new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
                                         counter += 1
+                                        print(f"DEBUG: File exists, trying {new_path}")
+                                    print(f"DEBUG: Moving file to {new_path}")
                                     os.rename(pdf_path, new_path)
                                     processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
                                     processed_count += 1
                                     job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+                                    print(f"DEBUG: Organized file {processed_count}/{total_files}")
+                    
+                    print(f"DEBUG: Found {len(pdf_files_found)} PDF files in ZIP: {pdf_files_found}")
+                    
+                except Exception as e:
+                    processed_files.append(('skipped', filename, f'ZIP processing error: {str(e)}'))
+                    job_queue[job_id]['progress'] = f'Failed to process ZIP {filename}: {str(e)}'
+                    print(f"ZIP processing error for {filename}: {str(e)}")
+                    continue
             else:
                 # Single PDF
+                try:
                     job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
                     pdf_info = extract_names_from_pdf(filepath)
                     if 'error' in pdf_info:
@@ -530,31 +562,43 @@ def process_job(job_id, files):
                         processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
                         processed_count += 1
                         job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+                except Exception as e:
+                    processed_files.append(('skipped', filename, f'PDF processing error: {str(e)}'))
+                    job_queue[job_id]['progress'] = f'Failed to process PDF {filename}: {str(e)}'
+                    print(f"PDF processing error for {filename}: {str(e)}")
+                    continue
         
         job_queue[job_id]['progress'] = 'Creating ZIP archive...'
         # Create ZIP of the entire Episode directory structure
         successful_files = [p for p in processed_files if p[0] == 'success']
+        print(f"DEBUG: Starting ZIP creation, successful_files: {len(successful_files)}")
         if successful_files:
             try:
                 zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip')
+                print(f"DEBUG: Creating ZIP at {zip_output}")
                 with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
                     for root, dirs, files_in_dir in os.walk(episode_dir):
                         for file in files_in_dir:
                             file_path = os.path.join(root, file)
                             arcname = os.path.relpath(file_path, episode_dir)
+                            print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
                             zip_out.write(file_path, arcname)
                 job_queue[job_id]['zip_path'] = zip_output
                 job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
                 job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
+                print(f"DEBUG: ZIP creation completed successfully")
             except Exception as e:
                 job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
                 job_queue[job_id]['status'] = 'error'
+                print(f"DEBUG: ZIP creation failed: {str(e)}")
         else:
             job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
+            print(f"DEBUG: No successful files to archive")
         
         job_queue[job_id]['status'] = 'completed'
         job_queue[job_id]['processed_files'] = processed_files
         job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
+        print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")
 
     except Exception as e:
         job_queue[job_id]['status'] = 'error'
@@ -689,7 +733,11 @@ def upload_files():
         print(f"Created job {job_id} with {len(file_list)} files")  # Debug logging
         
         # Start processing in background
-        thread = threading.Thread(target=process_job, args=(job_id, file_list))
+        def process_with_context():
+            with app.app_context():
+                process_job(job_id, file_list)
+        
+        thread = threading.Thread(target=process_with_context)
         thread.daemon = True
         thread.start()
         
@@ -761,6 +809,8 @@ def jobs():
             .nav-links { margin-bottom: 20px; }
             .nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
             .nav-links a:hover { background-color: #e9ecef; }
+            .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
+            .nav-btn:hover { background-color: #0056b3; }
             .zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
             .zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
             .zip-toggle:hover { background-color: #dee2e6; }
@@ -778,8 +828,8 @@ def jobs():
         <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
         <h1>Processing Jobs</h1>
         <div class="nav-links" style="margin-bottom: 20px;">
-            <a href="/">← Upload Files</a>
+            <a href="/" class="nav-btn">← Upload Files</a>
-            <a href="/completed-jobs">View Completed Jobs Archive →</a>
+            <a href="/completed-jobs" class="nav-btn">View Completed Jobs Archive →</a>
         </div>
         {% with messages = get_flashed_messages(with_categories=true) %}
         {% if messages %}
@@ -850,6 +900,13 @@ def jobs():
                 fetch('/api/jobs', { credentials: 'same-origin' })
                     .then(response => response.json())
                     .then(data => {
+                        console.log('Job update received:', data);
+                        const jobs = Object.entries(data);
+                        if (jobs.length > 0) {
+                            jobs.forEach(([jobId, job]) => {
+                                console.log(`Job ${jobId}: status=${job.status}, progress=${job.progress || 'none'}`);
+                            });
+                        }
                         const jobsContainer = document.querySelector('h1').parentNode;
                         const existingJobs = jobsContainer.querySelectorAll('.job');
                         const existingNoJobsMsg = jobsContainer.querySelectorAll('p');
@@ -928,8 +985,8 @@ def jobs():
                     .catch(error => console.error('Error updating jobs:', error));
             }
             
-            // Update jobs every 2 seconds
+            // Update jobs every 1 second
-            setInterval(updateJobs, 2000);
+            setInterval(updateJobs, 1000);
             
             // Collapsible section functions
             function toggleZipDetails(element) {
@@ -1015,6 +1072,8 @@ def completed_jobs():
             .nav-links { margin-bottom: 20px; }
             .nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
             .nav-links a:hover { background-color: #e9ecef; }
+            .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
+            .nav-btn:hover { background-color: #0056b3; }
             .job-stats { background-color: #e9ecef; padding: 10px; border-radius: 5px; margin-bottom: 15px; }
             .file-count { color: #17a2b8; font-weight: bold; }
             .zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
@@ -1034,8 +1093,8 @@ def completed_jobs():
         <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
         <h1>Completed Jobs Archive</h1>
         <div class="nav-links">
-            <a href="/">← Upload Files</a>
+            <a href="/" class="nav-btn">← Upload Files</a>
-            <a href="/jobs">View All Jobs →</a>
+            <a href="/jobs" class="nav-btn">View All Jobs →</a>
         </div>
         {% with messages = get_flashed_messages(with_categories=true) %}
         {% if messages %}

test_job_processing.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""
+Simple test script to verify job processing works
+"""
+import sys
+import os
+sys.path.append(os.path.dirname(__file__))
+
+from pdf_renamer import app, process_job, job_queue
+import tempfile
+import uuid
+
+# Create app context for testing
+with app.app_context():
+    # Create a test job with dummy files
+    job_id = str(uuid.uuid4())
+    job_queue[job_id] = {
+        'status': 'queued',
+        'files': [],
+        'zip_path': None,
+        'zip_name': None,
+        'processed_files': [],
+        'skipped': []
+    }
+
+    # Create some dummy file entries (they don't need to exist for this test)
+    file_list = [
+        {'name': 'test1.pdf', 'path': '/tmp/test1.pdf'},
+        {'name': 'test2.pdf', 'path': '/tmp/test2.pdf'}
+    ]
+
+    job_queue[job_id]['files'] = file_list
+
+    print(f"Testing job processing for job_id: {job_id}")
+    print("Calling process_job...")
+
+    # Call process_job directly
+    process_job(job_id, file_list)
+
+    print("process_job completed")
+    print(f"Final job status: {job_queue[job_id]['status']}")

test_upload.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""
+Test the upload and job processing flow
+"""
+import sys
+import os
+sys.path.append(os.path.dirname(__file__))
+
+from pdf_renamer import app, job_queue
+import tempfile
+import uuid
+from werkzeug.datastructures import FileStorage
+import io
+
+# Create a simple PDF-like content (not a real PDF, but enough for testing)
+pdf_content = b"%PDF-1.4\n1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj\n2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj\n3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n/Contents 4 0 R\n>>\nendobj\n4 0 obj\n<<\n/Length 44\n>>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF) Tj\nET\nendstream\nendobj\nxref\n0 5\n0000000000 65535 f\n0000000009 00000 n\n0000000058 00000 n\n0000000115 00000 n\n0000000200 00000 n\ntrailer\n<<\n/Size 5\n/Root 1 0 R\n>>\nstartxref\n284\n%%EOF"
+
+with app.test_client() as client:
+    with app.app_context():
+        # Create a test file
+        file_data = io.BytesIO(pdf_content)
+        file_data.filename = 'test.pdf'
+
+        # Simulate file upload
+        response = client.post('/', data={
+            'files': (file_data, 'test.pdf')
+        }, content_type='multipart/form-data')
+
+        print(f"Upload response status: {response.status_code}")
+        print(f"Response data: {response.get_data(as_text=True)}")
+
+        # Check if any jobs were created
+        print(f"Current jobs: {list(job_queue.keys())}")
+        for job_id, job in job_queue.items():
+            print(f"Job {job_id}: status={job.get('status')}, files={len(job.get('files', []))}")