diff --git a/app.py b/app.py index e0a1a67..e2b24d5 100644 --- a/app.py +++ b/app.py @@ -53,6 +53,15 @@ except Exception as e: # Load environment variables load_dotenv() +# Startup diagnostics +print("=" * 60) +print("APPLICATION STARTUP DIAGNOSTICS") +print("=" * 60) +print(f"Working directory: {os.getcwd()}") +print(f"App directory: {os.path.dirname(os.path.abspath(__file__))}") +print(f"PDF Support (PyMuPDF): {PDF_SUPPORT}") +print("=" * 60) + @app.context_processor def inject_logo(): return {'logo_b64': LOGO_B64} @@ -273,6 +282,27 @@ JOBS_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'jobs.json') RESULTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'results')) os.makedirs(RESULTS_DIR, exist_ok=True) +# Check directory permissions at startup +app_dir = os.path.dirname(os.path.abspath(__file__)) +uploads_dir = os.path.join(app_dir, 'uploads') +sessions_dir = os.path.join(app_dir, 'sessions') + +print(f"\nDirectory Checks:") +print(f" Uploads: {uploads_dir}") +print(f" Exists: {os.path.exists(uploads_dir)}") +print(f" Writable: {os.access(uploads_dir, os.W_OK) if os.path.exists(uploads_dir) else 'N/A'}") +print(f" Sessions: {sessions_dir}") +print(f" Exists: {os.path.exists(sessions_dir)}") +print(f" Writable: {os.access(sessions_dir, os.W_OK) if os.path.exists(sessions_dir) else 'N/A'}") +print(f" Results: {RESULTS_DIR}") +print(f" Exists: {os.path.exists(RESULTS_DIR)}") +print(f" Writable: {os.access(RESULTS_DIR, os.W_OK) if os.path.exists(RESULTS_DIR) else 'N/A'}") +print(f" Jobs file: {JOBS_FILE}") +print(f" Exists: {os.path.exists(JOBS_FILE)}") +if os.path.exists(JOBS_FILE): + print(f" Writable: {os.access(JOBS_FILE, os.W_OK)}") +print("=" * 60) + processing_complete = False current_output_dir = None @@ -705,13 +735,41 @@ def process_pdfs(pdfdir, outputfolder, job_id=None): current_output_dir = outputfolder def process_pdfs_async(pdfdir, outputfolder, job_id): - print(f"process_pdfs_async called with job_id={job_id}, pdfdir={pdfdir}, outputfolder={outputfolder}") + print("=" * 60) + print(f"PROCESS_PDFS_ASYNC CALLED") + print(f" Job ID: {job_id}") + print(f" PDF Dir: {pdfdir}") + print(f" Output Folder: {outputfolder}") + print(f" PDF Dir Exists: {os.path.exists(pdfdir)}") + if os.path.exists(pdfdir): + print(f" PDF Dir Writable: {os.access(pdfdir, os.W_OK)}") + print(f" PDF Dir Contents: {os.listdir(pdfdir)}") + print(f" Output Dir Exists: {os.path.exists(outputfolder)}") + if os.path.exists(outputfolder): + print(f" Output Dir Writable: {os.access(outputfolder, os.W_OK)}") + print(f" PDF Support Available: {PDF_SUPPORT}") + print(f" Timestamp: {datetime.now()}") + print("=" * 60) + global processing_complete - print(f"Processing started for job {job_id} at {datetime.now()}") + + if not PDF_SUPPORT: + error_msg = "PyMuPDF (fitz) is not installed - PDF processing cannot continue" + print(f"ERROR: {error_msg}") + jobs = load_jobs() + if job_id in jobs: + jobs[job_id]['status'] = 'failed' + jobs[job_id]['error'] = error_msg + jobs[job_id]['message'] = error_msg + save_jobs(jobs) + return + try: jobs = load_jobs() + print(f"Jobs loaded: {len(jobs)} total, job {job_id} exists: {job_id in jobs}") + if job_id not in jobs: - print(f"Job {job_id} not found in jobs") + print(f"ERROR: Job {job_id} not found in jobs") return status_file = 'status.txt' @@ -728,14 +786,32 @@ def process_pdfs_async(pdfdir, outputfolder, job_id): addresses_dir = os.path.join(outputfolder, 'Addresses') missing_dir = os.path.join(outputfolder, 'Missing_Addresses') - os.makedirs(addresses_dir) - os.makedirs(missing_dir) + os.makedirs(addresses_dir, exist_ok=True) + os.makedirs(missing_dir, exist_ok=True) + print(f"Created output directories: {addresses_dir}, {missing_dir}") + # Scan for PDF files pdf_files = [] + print(f"Scanning directory: {pdfdir}") for root, dirs, files in os.walk(pdfdir): + print(f" Checking directory: {root} with {len(files)} files") for f in files: if f.endswith('.pdf'): - pdf_files.append(os.path.join(root, f)) + pdf_path = os.path.join(root, f) + pdf_files.append(pdf_path) + print(f" Found PDF: {f}") + + print(f"Total PDFs found: {len(pdf_files)}") + + if len(pdf_files) == 0: + error_msg = f"No PDF files found in {pdfdir}" + print(f"ERROR: {error_msg}") + jobs = load_jobs() + jobs[job_id]['status'] = 'failed' + jobs[job_id]['error'] = error_msg + jobs[job_id]['message'] = error_msg + save_jobs(jobs) + return msg = f"Step 1/4: Found {len(pdf_files)} PDF files\n" with open(status_file, 'a') as status_f: @@ -764,13 +840,34 @@ def process_pdfs_async(pdfdir, outputfolder, job_id): time.sleep(0.1) # Parallel processing - print(f"Processing {len(pdf_files)} files sequentially") + print(f"Processing {len(pdf_files)} files sequentially (avoiding multiprocessing issues)") results = [] - for pdf_file in pdf_files: - result = process_single_pdf(pdf_file) - results.append(result) + for i, pdf_file in enumerate(pdf_files, 1): + print(f"Processing file {i}/{len(pdf_files)}: {os.path.basename(pdf_file)}") + try: + result = process_single_pdf(pdf_file) + results.append(result) + print(f" Result type: {result['type']}") + except Exception as e: + print(f" ERROR processing {pdf_file}: {e}") + results.append({ + 'type': 'error', + 'filename': os.path.basename(pdf_file), + 'error': str(e), + 'filepath': pdf_file + }) + + # Update progress periodically + if i % 10 == 0 or i == len(pdf_files): + progress_percent = 20 + int((i / len(pdf_files)) * 60) # 20-80% + jobs = load_jobs() + jobs[job_id]['message'] = f"Processing file {i}/{len(pdf_files)}" + jobs[job_id]['progress'] = progress_percent + save_jobs(jobs) + print(f" Progress: {progress_percent}%") # Process results sequentially + print("Processing results and sorting files...") with open(os.path.join(addresses_dir, 'Addresses.csv'), 'w') as addr_csv: addr_csv.write("File,User,Street,City,State,Zip\n") @@ -779,49 +876,23 @@ def process_pdfs_async(pdfdir, outputfolder, job_id): page_groups = {} for i, result in enumerate(results, 1): - progress_percent = 20 + int((i / len(results)) * 60) # 20-80% range for processing - - msg = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}\n" - with open(status_file, 'a') as status_f: - status_f.write(msg) - print(msg.strip()) - - # Update job status periodically - if i % 100 == 0 or i == len(results): # Update every 100 files or at the end - jobs[job_id]['message'] = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}" - jobs[job_id]['progress'] = progress_percent - save_jobs(jobs) - if result['type'] == 'address': with open(os.path.join(addresses_dir, 'Addresses.csv'), 'a') as addr_csv: addr_csv.write(f"{result['filename']},{result['user']},{result['address']},{result['city']},{result['state']},{result['zip']}\n") shutil.copy(result['filepath'], os.path.join(addresses_dir, result['filename'])) - msg = f" -> Moved to Addresses\n" - with open(status_file, 'a') as status_f: - status_f.write(msg) - print(msg.strip()) + if 'pages' in result: + pages = result['pages'] + if pages not in page_groups: + page_groups[pages] = [] + page_groups[pages].append(result['filepath']) elif result['type'] == 'missing': with open(os.path.join(missing_dir, 'Missing_Addresses.csv'), 'a') as miss_csv: miss_csv.write(f"{result['filename']}\t{result['user']}\n") shutil.copy(result['filepath'], os.path.join(missing_dir, result['filename'])) - msg = f" -> Moved to Missing_Addresses\n" - with open(status_file, 'a') as status_f: - status_f.write(msg) - print(msg.strip()) elif result['type'] == 'error': - msg = f" -> Error: {result['error']}\n" - with open(status_file, 'a') as status_f: - status_f.write(msg) - print(msg.strip()) - - # Group for merging - if 'pages' in result: - pages = result['pages'] - if pages not in page_groups: - page_groups[pages] = [] - page_groups[pages].append(result['filepath']) - - time.sleep(0.01) # Small delay for status updates + print(f" Error with {result['filename']}: {result.get('error', 'Unknown error')}") + + print(f"Results sorted. Found {len(page_groups)} page count groups for merging") msg = "Step 2/4: Sorting complete. Starting merging...\n" with open(status_file, 'a') as status_f: