Add comprehensive diagnostics and fix processing issues
- Add startup diagnostics showing directory permissions and PDF support - Enhanced process_pdfs_async with detailed logging at every step - Check if PyMuPDF is available and fail gracefully if not - Scan and log all PDF files found in upload directory - Improved error handling during file processing - Sequential processing with progress updates every 10 files - Clear error messages when no PDFs found or processing fails - This will show exactly where processing stops on remote server
This commit is contained in:
nmemmert committed
1 parent
3215cd9032
commit
a6b751053a
1 file changed
+115
-44
@@ -53,6 +53,15 @@ except Exception as e:
|
||||
# Load environment variables
|
||||
load_dotenv()
|
||||
|
||||
# Startup diagnostics
|
||||
print("=" * 60)
|
||||
print("APPLICATION STARTUP DIAGNOSTICS")
|
||||
print("=" * 60)
|
||||
print(f"Working directory: {os.getcwd()}")
|
||||
print(f"App directory: {os.path.dirname(os.path.abspath(__file__))}")
|
||||
print(f"PDF Support (PyMuPDF): {PDF_SUPPORT}")
|
||||
print("=" * 60)
|
||||
|
||||
@app.context_processor
|
||||
def inject_logo():
|
||||
return {'logo_b64': LOGO_B64}
|
||||
@@ -273,6 +282,27 @@ JOBS_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'jobs.json')
|
||||
RESULTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'results'))
|
||||
os.makedirs(RESULTS_DIR, exist_ok=True)
|
||||
|
||||
# Check directory permissions at startup
|
||||
app_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
uploads_dir = os.path.join(app_dir, 'uploads')
|
||||
sessions_dir = os.path.join(app_dir, 'sessions')
|
||||
|
||||
print(f"\nDirectory Checks:")
|
||||
print(f" Uploads: {uploads_dir}")
|
||||
print(f" Exists: {os.path.exists(uploads_dir)}")
|
||||
print(f" Writable: {os.access(uploads_dir, os.W_OK) if os.path.exists(uploads_dir) else 'N/A'}")
|
||||
print(f" Sessions: {sessions_dir}")
|
||||
print(f" Exists: {os.path.exists(sessions_dir)}")
|
||||
print(f" Writable: {os.access(sessions_dir, os.W_OK) if os.path.exists(sessions_dir) else 'N/A'}")
|
||||
print(f" Results: {RESULTS_DIR}")
|
||||
print(f" Exists: {os.path.exists(RESULTS_DIR)}")
|
||||
print(f" Writable: {os.access(RESULTS_DIR, os.W_OK) if os.path.exists(RESULTS_DIR) else 'N/A'}")
|
||||
print(f" Jobs file: {JOBS_FILE}")
|
||||
print(f" Exists: {os.path.exists(JOBS_FILE)}")
|
||||
if os.path.exists(JOBS_FILE):
|
||||
print(f" Writable: {os.access(JOBS_FILE, os.W_OK)}")
|
||||
print("=" * 60)
|
||||
|
||||
processing_complete = False
|
||||
current_output_dir = None
|
||||
|
||||
@@ -705,13 +735,41 @@ def process_pdfs(pdfdir, outputfolder, job_id=None):
|
||||
current_output_dir = outputfolder
|
||||
|
||||
def process_pdfs_async(pdfdir, outputfolder, job_id):
|
||||
print(f"process_pdfs_async called with job_id={job_id}, pdfdir={pdfdir}, outputfolder={outputfolder}")
|
||||
print("=" * 60)
|
||||
print(f"PROCESS_PDFS_ASYNC CALLED")
|
||||
print(f" Job ID: {job_id}")
|
||||
print(f" PDF Dir: {pdfdir}")
|
||||
print(f" Output Folder: {outputfolder}")
|
||||
print(f" PDF Dir Exists: {os.path.exists(pdfdir)}")
|
||||
if os.path.exists(pdfdir):
|
||||
print(f" PDF Dir Writable: {os.access(pdfdir, os.W_OK)}")
|
||||
print(f" PDF Dir Contents: {os.listdir(pdfdir)}")
|
||||
print(f" Output Dir Exists: {os.path.exists(outputfolder)}")
|
||||
if os.path.exists(outputfolder):
|
||||
print(f" Output Dir Writable: {os.access(outputfolder, os.W_OK)}")
|
||||
print(f" PDF Support Available: {PDF_SUPPORT}")
|
||||
print(f" Timestamp: {datetime.now()}")
|
||||
print("=" * 60)
|
||||
|
||||
global processing_complete
|
||||
print(f"Processing started for job {job_id} at {datetime.now()}")
|
||||
|
||||
if not PDF_SUPPORT:
|
||||
error_msg = "PyMuPDF (fitz) is not installed - PDF processing cannot continue"
|
||||
print(f"ERROR: {error_msg}")
|
||||
jobs = load_jobs()
|
||||
if job_id in jobs:
|
||||
jobs[job_id]['status'] = 'failed'
|
||||
jobs[job_id]['error'] = error_msg
|
||||
jobs[job_id]['message'] = error_msg
|
||||
save_jobs(jobs)
|
||||
return
|
||||
|
||||
try:
|
||||
jobs = load_jobs()
|
||||
print(f"Jobs loaded: {len(jobs)} total, job {job_id} exists: {job_id in jobs}")
|
||||
|
||||
if job_id not in jobs:
|
||||
print(f"Job {job_id} not found in jobs")
|
||||
print(f"ERROR: Job {job_id} not found in jobs")
|
||||
return
|
||||
|
||||
status_file = 'status.txt'
|
||||
@@ -728,14 +786,32 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
||||
|
||||
addresses_dir = os.path.join(outputfolder, 'Addresses')
|
||||
missing_dir = os.path.join(outputfolder, 'Missing_Addresses')
|
||||
os.makedirs(addresses_dir)
|
||||
os.makedirs(missing_dir)
|
||||
os.makedirs(addresses_dir, exist_ok=True)
|
||||
os.makedirs(missing_dir, exist_ok=True)
|
||||
print(f"Created output directories: {addresses_dir}, {missing_dir}")
|
||||
|
||||
# Scan for PDF files
|
||||
pdf_files = []
|
||||
print(f"Scanning directory: {pdfdir}")
|
||||
for root, dirs, files in os.walk(pdfdir):
|
||||
print(f" Checking directory: {root} with {len(files)} files")
|
||||
for f in files:
|
||||
if f.endswith('.pdf'):
|
||||
pdf_files.append(os.path.join(root, f))
|
||||
pdf_path = os.path.join(root, f)
|
||||
pdf_files.append(pdf_path)
|
||||
print(f" Found PDF: {f}")
|
||||
|
||||
print(f"Total PDFs found: {len(pdf_files)}")
|
||||
|
||||
if len(pdf_files) == 0:
|
||||
error_msg = f"No PDF files found in {pdfdir}"
|
||||
print(f"ERROR: {error_msg}")
|
||||
jobs = load_jobs()
|
||||
jobs[job_id]['status'] = 'failed'
|
||||
jobs[job_id]['error'] = error_msg
|
||||
jobs[job_id]['message'] = error_msg
|
||||
save_jobs(jobs)
|
||||
return
|
||||
|
||||
msg = f"Step 1/4: Found {len(pdf_files)} PDF files\n"
|
||||
with open(status_file, 'a') as status_f:
|
||||
@@ -764,13 +840,34 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
||||
time.sleep(0.1)
|
||||
|
||||
# Parallel processing
|
||||
print(f"Processing {len(pdf_files)} files sequentially")
|
||||
print(f"Processing {len(pdf_files)} files sequentially (avoiding multiprocessing issues)")
|
||||
results = []
|
||||
for pdf_file in pdf_files:
|
||||
result = process_single_pdf(pdf_file)
|
||||
results.append(result)
|
||||
for i, pdf_file in enumerate(pdf_files, 1):
|
||||
print(f"Processing file {i}/{len(pdf_files)}: {os.path.basename(pdf_file)}")
|
||||
try:
|
||||
result = process_single_pdf(pdf_file)
|
||||
results.append(result)
|
||||
print(f" Result type: {result['type']}")
|
||||
except Exception as e:
|
||||
print(f" ERROR processing {pdf_file}: {e}")
|
||||
results.append({
|
||||
'type': 'error',
|
||||
'filename': os.path.basename(pdf_file),
|
||||
'error': str(e),
|
||||
'filepath': pdf_file
|
||||
})
|
||||
|
||||
# Update progress periodically
|
||||
if i % 10 == 0 or i == len(pdf_files):
|
||||
progress_percent = 20 + int((i / len(pdf_files)) * 60) # 20-80%
|
||||
jobs = load_jobs()
|
||||
jobs[job_id]['message'] = f"Processing file {i}/{len(pdf_files)}"
|
||||
jobs[job_id]['progress'] = progress_percent
|
||||
save_jobs(jobs)
|
||||
print(f" Progress: {progress_percent}%")
|
||||
|
||||
# Process results sequentially
|
||||
print("Processing results and sorting files...")
|
||||
with open(os.path.join(addresses_dir, 'Addresses.csv'), 'w') as addr_csv:
|
||||
addr_csv.write("File,User,Street,City,State,Zip\n")
|
||||
|
||||
@@ -779,49 +876,23 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
||||
|
||||
page_groups = {}
|
||||
for i, result in enumerate(results, 1):
|
||||
progress_percent = 20 + int((i / len(results)) * 60) # 20-80% range for processing
|
||||
|
||||
msg = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}\n"
|
||||
with open(status_file, 'a') as status_f:
|
||||
status_f.write(msg)
|
||||
print(msg.strip())
|
||||
|
||||
# Update job status periodically
|
||||
if i % 100 == 0 or i == len(results): # Update every 100 files or at the end
|
||||
jobs[job_id]['message'] = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}"
|
||||
jobs[job_id]['progress'] = progress_percent
|
||||
save_jobs(jobs)
|
||||
|
||||
if result['type'] == 'address':
|
||||
with open(os.path.join(addresses_dir, 'Addresses.csv'), 'a') as addr_csv:
|
||||
addr_csv.write(f"{result['filename']},{result['user']},{result['address']},{result['city']},{result['state']},{result['zip']}\n")
|
||||
shutil.copy(result['filepath'], os.path.join(addresses_dir, result['filename']))
|
||||
msg = f" -> Moved to Addresses\n"
|
||||
with open(status_file, 'a') as status_f:
|
||||
status_f.write(msg)
|
||||
print(msg.strip())
|
||||
if 'pages' in result:
|
||||
pages = result['pages']
|
||||
if pages not in page_groups:
|
||||
page_groups[pages] = []
|
||||
page_groups[pages].append(result['filepath'])
|
||||
elif result['type'] == 'missing':
|
||||
with open(os.path.join(missing_dir, 'Missing_Addresses.csv'), 'a') as miss_csv:
|
||||
miss_csv.write(f"{result['filename']}\t{result['user']}\n")
|
||||
shutil.copy(result['filepath'], os.path.join(missing_dir, result['filename']))
|
||||
msg = f" -> Moved to Missing_Addresses\n"
|
||||
with open(status_file, 'a') as status_f:
|
||||
status_f.write(msg)
|
||||
print(msg.strip())
|
||||
elif result['type'] == 'error':
|
||||
msg = f" -> Error: {result['error']}\n"
|
||||
with open(status_file, 'a') as status_f:
|
||||
status_f.write(msg)
|
||||
print(msg.strip())
|
||||
|
||||
# Group for merging
|
||||
if 'pages' in result:
|
||||
pages = result['pages']
|
||||
if pages not in page_groups:
|
||||
page_groups[pages] = []
|
||||
page_groups[pages].append(result['filepath'])
|
||||
|
||||
time.sleep(0.01) # Small delay for status updates
|
||||
print(f" Error with {result['filename']}: {result.get('error', 'Unknown error')}")
|
||||
|
||||
print(f"Results sorted. Found {len(page_groups)} page count groups for merging")
|
||||
|
||||
msg = "Step 2/4: Sorting complete. Starting merging...\n"
|
||||
with open(status_file, 'a') as status_f:
|
||||
|
||||
Reference in new issue
Block a user