Add detailed progress tracking for job processing

pdf_renamer.py

@@ -74,31 +74,45 @@ def extract_names_from_pdf(pdf_path):
 
 def process_job(job_id, files):
     job_queue[job_id]['status'] = 'processing'
+    job_queue[job_id]['progress'] = 'Starting job processing...'
     processed_files = []
     temp_extract_dirs = []
     
     # Create base Episode directory for this job
     episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'Episode_{job_id}')
     os.makedirs(episode_dir, exist_ok=True)
+    job_queue[job_id]['progress'] = 'Created output directory structure'
+    
+    total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \
+                  sum(len([f for f in zipfile.ZipFile(file_info['path']).namelist() 
+                          if f.lower().endswith('.pdf')]) 
+                      for file_info in files if file_info['name'].lower().endswith('.zip'))
+    
+    processed_count = 0
     
     for file_info in files:
         filepath = file_info['path']
         filename = file_info['name']
         
         if filename.lower().endswith('.zip'):
+            job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
             # Extract ZIP
             extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
             os.makedirs(extract_dir, exist_ok=True)
             temp_extract_dirs.append(extract_dir)
             with zipfile.ZipFile(filepath, 'r') as zip_ref:
                 zip_ref.extractall(extract_dir)
+            job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}'
+            
             # Process PDFs in extract_dir
             for root, dirs, files_in_zip in os.walk(extract_dir):
                 for f in files_in_zip:
                     if f.lower().endswith('.pdf'):
                         pdf_path = os.path.join(root, f)
+                        job_queue[job_id]['progress'] = f'Processing PDF: {f}'
                         pdf_info = extract_names_from_pdf(pdf_path)
                         if pdf_info:
+                            job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
                             # Create folder structure: Episode/School/Grade/
                             school_dir = os.path.join(episode_dir, pdf_info['school'])
                             grade_dir = os.path.join(school_dir, pdf_info['grade'])
@@ -113,12 +127,17 @@ def process_job(job_id, files):
                                 counter += 1
                             os.rename(pdf_path, new_path)
                             processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
+                            processed_count += 1
+                            job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
                         else:
                             processed_files.append(('skipped', f, None))
+                            job_queue[job_id]['progress'] = f'Skipped {f} - missing required fields'
         else:
             # Single PDF
+            job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
             pdf_info = extract_names_from_pdf(filepath)
             if pdf_info:
+                job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
                 # Create folder structure: Episode/School/Grade/
                 school_dir = os.path.join(episode_dir, pdf_info['school'])
                 grade_dir = os.path.join(school_dir, pdf_info['grade'])
@@ -133,9 +152,13 @@ def process_job(job_id, files):
                     counter += 1
                 os.rename(filepath, new_filepath)
                 processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
+                processed_count += 1
+                job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
             else:
                 processed_files.append(('skipped', filename, None))
+                job_queue[job_id]['progress'] = f'Skipped {filename} - missing required fields'
     
+    job_queue[job_id]['progress'] = 'Creating ZIP archive...'
     # Create ZIP of the entire Episode directory structure
     successful_files = [p for p in processed_files if p[0] == 'success']
     if successful_files:
@@ -148,6 +171,7 @@ def process_job(job_id, files):
                     zip_out.write(file_path, arcname)
         job_queue[job_id]['zip_path'] = zip_output
         job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
+        job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
     
     job_queue[job_id]['status'] = 'completed'
     job_queue[job_id]['processed_files'] = processed_files
@@ -329,6 +353,7 @@ def jobs():
             .status.queued { background-color: #ffc107; }
             .status.processing { background-color: #17a2b8; }
             .status.completed { background-color: #28a745; }
+            .progress-text { color: #17a2b8; font-style: italic; }
             .download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; }
             .download-btn:hover { background-color: #0056b3; }
             .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
@@ -345,6 +370,9 @@ def jobs():
         <div class="job">
             <h3>Job {{ job_id[:8] }}</h3>
             <p><strong>Status:</strong> <span class="status {{ job.status }}">{{ job.status.title() }}</span></p>
+            {% if job.status == 'processing' and job.progress %}
+            <p><strong>Progress:</strong> <span class="progress-text">{{ job.progress }}</span></p>
+            {% endif %}
             <p><strong>Files:</strong> {{ job.files|length }}</p>
             {% if job.status == 'completed' %}
                 {% if job.zip_name %}