Add comprehensive diagnostics and fix processing issues

- Add startup diagnostics showing directory permissions and PDF support
- Enhanced process_pdfs_async with detailed logging at every step
- Check if PyMuPDF is available and fail gracefully if not
- Scan and log all PDF files found in upload directory
- Improved error handling during file processing
- Sequential processing with progress updates every 10 files
- Clear error messages when no PDFs found or processing fails
- This will show exactly where processing stops on remote server

app.py

@@ -53,6 +53,15 @@ except Exception as e:
 # Load environment variables
 load_dotenv()
 
+# Startup diagnostics
+print("=" * 60)
+print("APPLICATION STARTUP DIAGNOSTICS")
+print("=" * 60)
+print(f"Working directory: {os.getcwd()}")
+print(f"App directory: {os.path.dirname(os.path.abspath(__file__))}")
+print(f"PDF Support (PyMuPDF): {PDF_SUPPORT}")
+print("=" * 60)
+
 @app.context_processor
 def inject_logo():
     return {'logo_b64': LOGO_B64}
@@ -273,6 +282,27 @@ JOBS_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'jobs.json')
 RESULTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'results'))
 os.makedirs(RESULTS_DIR, exist_ok=True)
 
+# Check directory permissions at startup
+app_dir = os.path.dirname(os.path.abspath(__file__))
+uploads_dir = os.path.join(app_dir, 'uploads')
+sessions_dir = os.path.join(app_dir, 'sessions')
+
+print(f"\nDirectory Checks:")
+print(f"  Uploads: {uploads_dir}")
+print(f"    Exists: {os.path.exists(uploads_dir)}")
+print(f"    Writable: {os.access(uploads_dir, os.W_OK) if os.path.exists(uploads_dir) else 'N/A'}")
+print(f"  Sessions: {sessions_dir}")
+print(f"    Exists: {os.path.exists(sessions_dir)}")
+print(f"    Writable: {os.access(sessions_dir, os.W_OK) if os.path.exists(sessions_dir) else 'N/A'}")
+print(f"  Results: {RESULTS_DIR}")
+print(f"    Exists: {os.path.exists(RESULTS_DIR)}")
+print(f"    Writable: {os.access(RESULTS_DIR, os.W_OK) if os.path.exists(RESULTS_DIR) else 'N/A'}")
+print(f"  Jobs file: {JOBS_FILE}")
+print(f"    Exists: {os.path.exists(JOBS_FILE)}")
+if os.path.exists(JOBS_FILE):
+    print(f"    Writable: {os.access(JOBS_FILE, os.W_OK)}")
+print("=" * 60)
+
 processing_complete = False
 current_output_dir = None
 
@@ -705,13 +735,41 @@ def process_pdfs(pdfdir, outputfolder, job_id=None):
     current_output_dir = outputfolder
 
 def process_pdfs_async(pdfdir, outputfolder, job_id):
-    print(f"process_pdfs_async called with job_id={job_id}, pdfdir={pdfdir}, outputfolder={outputfolder}")
+    print("=" * 60)
+    print(f"PROCESS_PDFS_ASYNC CALLED")
+    print(f"  Job ID: {job_id}")
+    print(f"  PDF Dir: {pdfdir}")
+    print(f"  Output Folder: {outputfolder}")
+    print(f"  PDF Dir Exists: {os.path.exists(pdfdir)}")
+    if os.path.exists(pdfdir):
+        print(f"  PDF Dir Writable: {os.access(pdfdir, os.W_OK)}")
+        print(f"  PDF Dir Contents: {os.listdir(pdfdir)}")
+    print(f"  Output Dir Exists: {os.path.exists(outputfolder)}")
+    if os.path.exists(outputfolder):
+        print(f"  Output Dir Writable: {os.access(outputfolder, os.W_OK)}")
+    print(f"  PDF Support Available: {PDF_SUPPORT}")
+    print(f"  Timestamp: {datetime.now()}")
+    print("=" * 60)
+    
     global processing_complete
-    print(f"Processing started for job {job_id} at {datetime.now()}")
+    
+    if not PDF_SUPPORT:
+        error_msg = "PyMuPDF (fitz) is not installed - PDF processing cannot continue"
+        print(f"ERROR: {error_msg}")
+        jobs = load_jobs()
+        if job_id in jobs:
+            jobs[job_id]['status'] = 'failed'
+            jobs[job_id]['error'] = error_msg
+            jobs[job_id]['message'] = error_msg
+            save_jobs(jobs)
+        return
+    
     try:
         jobs = load_jobs()
+        print(f"Jobs loaded: {len(jobs)} total, job {job_id} exists: {job_id in jobs}")
+        
         if job_id not in jobs:
-            print(f"Job {job_id} not found in jobs")
+            print(f"ERROR: Job {job_id} not found in jobs")
             return
         
         status_file = 'status.txt'
@@ -728,14 +786,32 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
         
         addresses_dir = os.path.join(outputfolder, 'Addresses')
         missing_dir = os.path.join(outputfolder, 'Missing_Addresses')
-        os.makedirs(addresses_dir)
+        os.makedirs(addresses_dir, exist_ok=True)
-        os.makedirs(missing_dir)
+        os.makedirs(missing_dir, exist_ok=True)
+        print(f"Created output directories: {addresses_dir}, {missing_dir}")
         
+        # Scan for PDF files
         pdf_files = []
+        print(f"Scanning directory: {pdfdir}")
         for root, dirs, files in os.walk(pdfdir):
+            print(f"  Checking directory: {root} with {len(files)} files")
             for f in files:
                 if f.endswith('.pdf'):
-                    pdf_files.append(os.path.join(root, f))
+                    pdf_path = os.path.join(root, f)
+                    pdf_files.append(pdf_path)
+                    print(f"    Found PDF: {f}")
+        
+        print(f"Total PDFs found: {len(pdf_files)}")
+        
+        if len(pdf_files) == 0:
+            error_msg = f"No PDF files found in {pdfdir}"
+            print(f"ERROR: {error_msg}")
+            jobs = load_jobs()
+            jobs[job_id]['status'] = 'failed'
+            jobs[job_id]['error'] = error_msg
+            jobs[job_id]['message'] = error_msg
+            save_jobs(jobs)
+            return
         
         msg = f"Step 1/4: Found {len(pdf_files)} PDF files\n"
         with open(status_file, 'a') as status_f:
@@ -764,13 +840,34 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
         time.sleep(0.1)
         
         # Parallel processing
-        print(f"Processing {len(pdf_files)} files sequentially")
+        print(f"Processing {len(pdf_files)} files sequentially (avoiding multiprocessing issues)")
         results = []
-        for pdf_file in pdf_files:
+        for i, pdf_file in enumerate(pdf_files, 1):
-            result = process_single_pdf(pdf_file)
+            print(f"Processing file {i}/{len(pdf_files)}: {os.path.basename(pdf_file)}")
-            results.append(result)
+            try:
+                result = process_single_pdf(pdf_file)
+                results.append(result)
+                print(f"  Result type: {result['type']}")
+            except Exception as e:
+                print(f"  ERROR processing {pdf_file}: {e}")
+                results.append({
+                    'type': 'error',
+                    'filename': os.path.basename(pdf_file),
+                    'error': str(e),
+                    'filepath': pdf_file
+                })
+            
+            # Update progress periodically
+            if i % 10 == 0 or i == len(pdf_files):
+                progress_percent = 20 + int((i / len(pdf_files)) * 60)  # 20-80%
+                jobs = load_jobs()
+                jobs[job_id]['message'] = f"Processing file {i}/{len(pdf_files)}"
+                jobs[job_id]['progress'] = progress_percent
+                save_jobs(jobs)
+                print(f"  Progress: {progress_percent}%")
         
         # Process results sequentially
+        print("Processing results and sorting files...")
         with open(os.path.join(addresses_dir, 'Addresses.csv'), 'w') as addr_csv:
             addr_csv.write("File,User,Street,City,State,Zip\n")
         
@@ -779,49 +876,23 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
         
         page_groups = {}
         for i, result in enumerate(results, 1):
-            progress_percent = 20 + int((i / len(results)) * 60)  # 20-80% range for processing
-            
-            msg = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}\n"
-            with open(status_file, 'a') as status_f:
-                status_f.write(msg)
-            print(msg.strip())
-            
-            # Update job status periodically
-            if i % 100 == 0 or i == len(results):  # Update every 100 files or at the end
-                jobs[job_id]['message'] = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}"
-                jobs[job_id]['progress'] = progress_percent
-                save_jobs(jobs)
-            
             if result['type'] == 'address':
                 with open(os.path.join(addresses_dir, 'Addresses.csv'), 'a') as addr_csv:
                     addr_csv.write(f"{result['filename']},{result['user']},{result['address']},{result['city']},{result['state']},{result['zip']}\n")
                 shutil.copy(result['filepath'], os.path.join(addresses_dir, result['filename']))
-                msg = f"  -> Moved to Addresses\n"
+                if 'pages' in result:
-                with open(status_file, 'a') as status_f:
+                    pages = result['pages']
-                    status_f.write(msg)
+                    if pages not in page_groups:
-                print(msg.strip())
+                        page_groups[pages] = []
+                    page_groups[pages].append(result['filepath'])
             elif result['type'] == 'missing':
                 with open(os.path.join(missing_dir, 'Missing_Addresses.csv'), 'a') as miss_csv:
                     miss_csv.write(f"{result['filename']}\t{result['user']}\n")
                 shutil.copy(result['filepath'], os.path.join(missing_dir, result['filename']))
-                msg = f"  -> Moved to Missing_Addresses\n"
-                with open(status_file, 'a') as status_f:
-                    status_f.write(msg)
-                print(msg.strip())
             elif result['type'] == 'error':
-                msg = f"  -> Error: {result['error']}\n"
+                print(f"  Error with {result['filename']}: {result.get('error', 'Unknown error')}")
-                with open(status_file, 'a') as status_f:
+        
-                    status_f.write(msg)
+        print(f"Results sorted. Found {len(page_groups)} page count groups for merging")
-                print(msg.strip())
-            
-            # Group for merging
-            if 'pages' in result:
-                pages = result['pages']
-                if pages not in page_groups:
-                    page_groups[pages] = []
-                page_groups[pages].append(result['filepath'])
-            
-            time.sleep(0.01)  # Small delay for status updates
         
         msg = "Step 2/4: Sorting complete. Starting merging...\n"
         with open(status_file, 'a') as status_f: