Nav updates

pdf_renamer.py

@@ -432,6 +432,7 @@ def extract_student_info_from_text(text):
 
 def process_job(job_id, files):
     try:
+        print("=== DEBUG: process_job STARTED ===")
         job_queue[job_id]['status'] = 'processing'
         job_queue[job_id]['progress'] = 'Starting job processing...'
         processed_files = []
@@ -444,6 +445,7 @@ def process_job(job_id, files):
         
         # Analyze ZIP contents before processing
         zip_contents = {}
+        job_queue[job_id]['progress'] = 'Analyzing uploaded files...'
         for file_info in files:
             if file_info['name'].lower().endswith('.zip'):
                 try:
@@ -459,6 +461,8 @@ def process_job(job_id, files):
                       sum(len(zip_contents.get(file_info['name'], [])) 
                           for file_info in files if file_info['name'].lower().endswith('.zip'))
         
+        job_queue[job_id]['progress'] = f'Found {total_files} files to process'
+        
         processed_count = 0
         
         for file_info in files:
@@ -466,95 +470,135 @@ def process_job(job_id, files):
             filename = file_info['name']
             
             if filename.lower().endswith('.zip'):
-                job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
-                # Extract ZIP
-                extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
-                os.makedirs(extract_dir, exist_ok=True)
-                temp_extract_dirs.append(extract_dir)
-                with zipfile.ZipFile(filepath, 'r') as zip_ref:
-                    zip_ref.extractall(extract_dir)
-                job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}'
-                
-                # Process PDFs in extract_dir
-                for root, dirs, files_in_zip in os.walk(extract_dir):
-                    for f in files_in_zip:
-                        if f.lower().endswith('.pdf'):
-                            pdf_path = os.path.join(root, f)
-                            job_queue[job_id]['progress'] = f'Processing PDF: {f}'
-                            pdf_info = extract_names_from_pdf(pdf_path)
-                            if 'error' in pdf_info:
-                                processed_files.append(('skipped', f, pdf_info['error']))
-                                job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
-                            else:
-                                job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
-                                # Create folder structure: Episode/School/Grade/Gender/
-                                school_dir = os.path.join(episode_dir, pdf_info['school'])
-                                grade_dir = os.path.join(school_dir, pdf_info['grade'])
-                                gender_dir = os.path.join(grade_dir, pdf_info['gender'])
-                                os.makedirs(gender_dir, exist_ok=True)
-                                
-                                # Move file to appropriate folder with new name
-                                new_path = os.path.join(gender_dir, pdf_info['filename'])
-                                counter = 1
-                                while os.path.exists(new_path):
-                                    base, ext = os.path.splitext(pdf_info['filename'])
-                                    new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
-                                    counter += 1
-                                os.rename(pdf_path, new_path)
-                                processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
-                                processed_count += 1
-                                job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+                try:
+                    job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
+                    print(f"DEBUG: Starting ZIP extraction for {filename}")
+                    # Extract ZIP
+                    extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
+                    os.makedirs(extract_dir, exist_ok=True)
+                    temp_extract_dirs.append(extract_dir)
+                    print(f"DEBUG: Created extract directory: {extract_dir}")
+                    with zipfile.ZipFile(filepath, 'r') as zip_ref:
+                        file_count = len(zip_ref.namelist())
+                        print(f"DEBUG: ZIP contains {file_count} files: {zip_ref.namelist()}")
+                        zip_ref.extractall(extract_dir)
+                    job_queue[job_id]['progress'] = f'Extracted {file_count} files from {filename}'
+                    print(f"DEBUG: ZIP extraction completed, progress updated: {job_queue[job_id]['progress']}")
+                    
+                    # Process PDFs in extract_dir
+                    pdf_files_found = []
+                    for root, dirs, files_in_zip in os.walk(extract_dir):
+                        for f in files_in_zip:
+                            if f.lower().endswith('.pdf'):
+                                pdf_files_found.append(f)
+                                pdf_path = os.path.join(root, f)
+                                job_queue[job_id]['progress'] = f'Processing PDF: {f}'
+                                print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
+                                print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
+                                pdf_info = extract_names_from_pdf(pdf_path)
+                                print(f"DEBUG: PDF extraction result: {pdf_info}")
+                                if 'error' in pdf_info:
+                                    processed_files.append(('skipped', f, pdf_info['error']))
+                                    job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
+                                    print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
+                                else:
+                                    job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
+                                    print(f"DEBUG: Successfully extracted data for {pdf_info['first_name']} {pdf_info['last_name']}")
+                                    # Create folder structure: Episode/School/Grade/Gender/
+                                    school_dir = os.path.join(episode_dir, pdf_info['school'])
+                                    grade_dir = os.path.join(school_dir, pdf_info['grade'])
+                                    gender_dir = os.path.join(grade_dir, pdf_info['gender'])
+                                    print(f"DEBUG: Creating directories: {school_dir} -> {grade_dir} -> {gender_dir}")
+                                    os.makedirs(gender_dir, exist_ok=True)
+                                    print(f"DEBUG: Directories created successfully")
+                                    
+                                    # Move file to appropriate folder with new name
+                                    new_path = os.path.join(gender_dir, pdf_info['filename'])
+                                    print(f"DEBUG: Attempting to move {pdf_path} to {new_path}")
+                                    counter = 1
+                                    while os.path.exists(new_path):
+                                        base, ext = os.path.splitext(pdf_info['filename'])
+                                        new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
+                                        counter += 1
+                                        print(f"DEBUG: File exists, trying {new_path}")
+                                    print(f"DEBUG: Moving file to {new_path}")
+                                    os.rename(pdf_path, new_path)
+                                    processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
+                                    processed_count += 1
+                                    job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+                                    print(f"DEBUG: Organized file {processed_count}/{total_files}")
+                    
+                    print(f"DEBUG: Found {len(pdf_files_found)} PDF files in ZIP: {pdf_files_found}")
+                    
+                except Exception as e:
+                    processed_files.append(('skipped', filename, f'ZIP processing error: {str(e)}'))
+                    job_queue[job_id]['progress'] = f'Failed to process ZIP {filename}: {str(e)}'
+                    print(f"ZIP processing error for {filename}: {str(e)}")
+                    continue
             else:
                 # Single PDF
-                job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
-                pdf_info = extract_names_from_pdf(filepath)
-                if 'error' in pdf_info:
-                    processed_files.append(('skipped', filename, pdf_info['error']))
-                    job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
-                else:
-                    job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
-                    # Create folder structure: Episode/School/Grade/Gender/
-                    school_dir = os.path.join(episode_dir, pdf_info['school'])
-                    grade_dir = os.path.join(school_dir, pdf_info['grade'])
-                    gender_dir = os.path.join(grade_dir, pdf_info['gender'])
-                    os.makedirs(gender_dir, exist_ok=True)
-                    
-                    # Move file to appropriate folder with new name
-                    new_filepath = os.path.join(gender_dir, pdf_info['filename'])
-                    counter = 1
-                    while os.path.exists(new_filepath):
-                        base, ext = os.path.splitext(pdf_info['filename'])
-                        new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
-                        counter += 1
-                    os.rename(filepath, new_filepath)
-                    processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
-                    processed_count += 1
-                    job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+                try:
+                    job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
+                    pdf_info = extract_names_from_pdf(filepath)
+                    if 'error' in pdf_info:
+                        processed_files.append(('skipped', filename, pdf_info['error']))
+                        job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
+                    else:
+                        job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
+                        # Create folder structure: Episode/School/Grade/Gender/
+                        school_dir = os.path.join(episode_dir, pdf_info['school'])
+                        grade_dir = os.path.join(school_dir, pdf_info['grade'])
+                        gender_dir = os.path.join(grade_dir, pdf_info['gender'])
+                        os.makedirs(gender_dir, exist_ok=True)
+                        
+                        # Move file to appropriate folder with new name
+                        new_filepath = os.path.join(gender_dir, pdf_info['filename'])
+                        counter = 1
+                        while os.path.exists(new_filepath):
+                            base, ext = os.path.splitext(pdf_info['filename'])
+                            new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
+                            counter += 1
+                        os.rename(filepath, new_filepath)
+                        processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
+                        processed_count += 1
+                        job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+                except Exception as e:
+                    processed_files.append(('skipped', filename, f'PDF processing error: {str(e)}'))
+                    job_queue[job_id]['progress'] = f'Failed to process PDF {filename}: {str(e)}'
+                    print(f"PDF processing error for {filename}: {str(e)}")
+                    continue
         
         job_queue[job_id]['progress'] = 'Creating ZIP archive...'
         # Create ZIP of the entire Episode directory structure
         successful_files = [p for p in processed_files if p[0] == 'success']
+        print(f"DEBUG: Starting ZIP creation, successful_files: {len(successful_files)}")
         if successful_files:
             try:
                 zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip')
+                print(f"DEBUG: Creating ZIP at {zip_output}")
                 with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
                     for root, dirs, files_in_dir in os.walk(episode_dir):
                         for file in files_in_dir:
                             file_path = os.path.join(root, file)
                             arcname = os.path.relpath(file_path, episode_dir)
+                            print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
                             zip_out.write(file_path, arcname)
                 job_queue[job_id]['zip_path'] = zip_output
                 job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
                 job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
+                print(f"DEBUG: ZIP creation completed successfully")
             except Exception as e:
                 job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
                 job_queue[job_id]['status'] = 'error'
+                print(f"DEBUG: ZIP creation failed: {str(e)}")
         else:
             job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
+            print(f"DEBUG: No successful files to archive")
         
         job_queue[job_id]['status'] = 'completed'
         job_queue[job_id]['processed_files'] = processed_files
         job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
+        print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")
 
     except Exception as e:
         job_queue[job_id]['status'] = 'error'
@@ -689,7 +733,11 @@ def upload_files():
         print(f"Created job {job_id} with {len(file_list)} files")  # Debug logging
         
         # Start processing in background
-        thread = threading.Thread(target=process_job, args=(job_id, file_list))
+        def process_with_context():
+            with app.app_context():
+                process_job(job_id, file_list)
+        
+        thread = threading.Thread(target=process_with_context)
         thread.daemon = True
         thread.start()
         
@@ -761,6 +809,8 @@ def jobs():
             .nav-links { margin-bottom: 20px; }
             .nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
             .nav-links a:hover { background-color: #e9ecef; }
+            .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
+            .nav-btn:hover { background-color: #0056b3; }
             .zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
             .zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
             .zip-toggle:hover { background-color: #dee2e6; }
@@ -778,8 +828,8 @@ def jobs():
         <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
         <h1>Processing Jobs</h1>
         <div class="nav-links" style="margin-bottom: 20px;">
-            <a href="/">← Upload Files</a>
-            <a href="/completed-jobs">View Completed Jobs Archive →</a>
+            <a href="/" class="nav-btn">← Upload Files</a>
+            <a href="/completed-jobs" class="nav-btn">View Completed Jobs Archive →</a>
         </div>
         {% with messages = get_flashed_messages(with_categories=true) %}
         {% if messages %}
@@ -850,6 +900,13 @@ def jobs():
                 fetch('/api/jobs', { credentials: 'same-origin' })
                     .then(response => response.json())
                     .then(data => {
+                        console.log('Job update received:', data);
+                        const jobs = Object.entries(data);
+                        if (jobs.length > 0) {
+                            jobs.forEach(([jobId, job]) => {
+                                console.log(`Job ${jobId}: status=${job.status}, progress=${job.progress || 'none'}`);
+                            });
+                        }
                         const jobsContainer = document.querySelector('h1').parentNode;
                         const existingJobs = jobsContainer.querySelectorAll('.job');
                         const existingNoJobsMsg = jobsContainer.querySelectorAll('p');
@@ -928,8 +985,8 @@ def jobs():
                     .catch(error => console.error('Error updating jobs:', error));
             }
             
-            // Update jobs every 2 seconds
-            setInterval(updateJobs, 2000);
+            // Update jobs every 1 second
+            setInterval(updateJobs, 1000);
             
             // Collapsible section functions
             function toggleZipDetails(element) {
@@ -1015,6 +1072,8 @@ def completed_jobs():
             .nav-links { margin-bottom: 20px; }
             .nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
             .nav-links a:hover { background-color: #e9ecef; }
+            .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
+            .nav-btn:hover { background-color: #0056b3; }
             .job-stats { background-color: #e9ecef; padding: 10px; border-radius: 5px; margin-bottom: 15px; }
             .file-count { color: #17a2b8; font-weight: bold; }
             .zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
@@ -1034,8 +1093,8 @@ def completed_jobs():
         <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
         <h1>Completed Jobs Archive</h1>
         <div class="nav-links">
-            <a href="/">← Upload Files</a>
-            <a href="/jobs">View All Jobs →</a>
+            <a href="/" class="nav-btn">← Upload Files</a>
+            <a href="/jobs" class="nav-btn">View All Jobs →</a>
         </div>
         {% with messages = get_flashed_messages(with_categories=true) %}
         {% if messages %}