diff --git a/134d5916-1c71-47da-a24c-0ead74f8f8e2.pdf b/134d5916-1c71-47da-a24c-0ead74f8f8e2.pdf
deleted file mode 100644
index d62e90e..0000000
Binary files a/134d5916-1c71-47da-a24c-0ead74f8f8e2.pdf and /dev/null differ
diff --git a/36154a41-79c0-4b26-a5e8-4d6b3b92dc2f.pdf b/36154a41-79c0-4b26-a5e8-4d6b3b92dc2f.pdf
deleted file mode 100644
index 3cf90c1..0000000
Binary files a/36154a41-79c0-4b26-a5e8-4d6b3b92dc2f.pdf and /dev/null differ
diff --git a/49ab90d0-790b-437c-96d3-9257209eab9f.pdf b/49ab90d0-790b-437c-96d3-9257209eab9f.pdf
deleted file mode 100644
index 76e3799..0000000
Binary files a/49ab90d0-790b-437c-96d3-9257209eab9f.pdf and /dev/null differ
diff --git a/4ba5ec74-f265-4b13-b0e0-fea815f16c27.pdf b/4ba5ec74-f265-4b13-b0e0-fea815f16c27.pdf
deleted file mode 100644
index 4e92927..0000000
Binary files a/4ba5ec74-f265-4b13-b0e0-fea815f16c27.pdf and /dev/null differ
diff --git a/6ea17a04-4600-45dd-b668-aa55155e03f9.pdf b/6ea17a04-4600-45dd-b668-aa55155e03f9.pdf
deleted file mode 100644
index ab447db..0000000
Binary files a/6ea17a04-4600-45dd-b668-aa55155e03f9.pdf and /dev/null differ
diff --git a/__pycache__/pdf_renamer.cpython-313.pyc b/__pycache__/pdf_renamer.cpython-313.pyc
index 9b03b23..48d88f8 100644
Binary files a/__pycache__/pdf_renamer.cpython-313.pyc and b/__pycache__/pdf_renamer.cpython-313.pyc differ
diff --git a/b3a891fc-eb29-4246-b587-e58b2f12918b (1).pdf b/b3a891fc-eb29-4246-b587-e58b2f12918b (1).pdf
deleted file mode 100644
index 37ebef3..0000000
Binary files a/b3a891fc-eb29-4246-b587-e58b2f12918b (1).pdf and /dev/null differ
diff --git a/cd194a62-e8a8-47c9-8cea-30d71d6febb7.pdf b/cd194a62-e8a8-47c9-8cea-30d71d6febb7.pdf
deleted file mode 100644
index 9ee3e80..0000000
Binary files a/cd194a62-e8a8-47c9-8cea-30d71d6febb7.pdf and /dev/null differ
diff --git a/d068c5e9-bc93-4d21-818f-2270492f35a5.pdf b/d068c5e9-bc93-4d21-818f-2270492f35a5.pdf
deleted file mode 100644
index 607bf1d..0000000
Binary files a/d068c5e9-bc93-4d21-818f-2270492f35a5.pdf and /dev/null differ
diff --git a/e0f2bef4-53ad-4fed-b906-478f23f51d1e.pdf b/e0f2bef4-53ad-4fed-b906-478f23f51d1e.pdf
deleted file mode 100644
index c540406..0000000
Binary files a/e0f2bef4-53ad-4fed-b906-478f23f51d1e.pdf and /dev/null differ
diff --git a/pdf_renamer.py b/pdf_renamer.py
index b3be657..d6b6007 100644
--- a/pdf_renamer.py
+++ b/pdf_renamer.py
@@ -432,6 +432,7 @@ def extract_student_info_from_text(text):
def process_job(job_id, files):
try:
+ print("=== DEBUG: process_job STARTED ===")
job_queue[job_id]['status'] = 'processing'
job_queue[job_id]['progress'] = 'Starting job processing...'
processed_files = []
@@ -444,6 +445,7 @@ def process_job(job_id, files):
# Analyze ZIP contents before processing
zip_contents = {}
+ job_queue[job_id]['progress'] = 'Analyzing uploaded files...'
for file_info in files:
if file_info['name'].lower().endswith('.zip'):
try:
@@ -459,6 +461,8 @@ def process_job(job_id, files):
sum(len(zip_contents.get(file_info['name'], []))
for file_info in files if file_info['name'].lower().endswith('.zip'))
+ job_queue[job_id]['progress'] = f'Found {total_files} files to process'
+
processed_count = 0
for file_info in files:
@@ -466,95 +470,135 @@ def process_job(job_id, files):
filename = file_info['name']
if filename.lower().endswith('.zip'):
- job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
- # Extract ZIP
- extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
- os.makedirs(extract_dir, exist_ok=True)
- temp_extract_dirs.append(extract_dir)
- with zipfile.ZipFile(filepath, 'r') as zip_ref:
- zip_ref.extractall(extract_dir)
- job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}'
-
- # Process PDFs in extract_dir
- for root, dirs, files_in_zip in os.walk(extract_dir):
- for f in files_in_zip:
- if f.lower().endswith('.pdf'):
- pdf_path = os.path.join(root, f)
- job_queue[job_id]['progress'] = f'Processing PDF: {f}'
- pdf_info = extract_names_from_pdf(pdf_path)
- if 'error' in pdf_info:
- processed_files.append(('skipped', f, pdf_info['error']))
- job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
- else:
- job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
- # Create folder structure: Episode/School/Grade/Gender/
- school_dir = os.path.join(episode_dir, pdf_info['school'])
- grade_dir = os.path.join(school_dir, pdf_info['grade'])
- gender_dir = os.path.join(grade_dir, pdf_info['gender'])
- os.makedirs(gender_dir, exist_ok=True)
-
- # Move file to appropriate folder with new name
- new_path = os.path.join(gender_dir, pdf_info['filename'])
- counter = 1
- while os.path.exists(new_path):
- base, ext = os.path.splitext(pdf_info['filename'])
- new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
- counter += 1
- os.rename(pdf_path, new_path)
- processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
- processed_count += 1
- job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+ try:
+ job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
+ print(f"DEBUG: Starting ZIP extraction for {filename}")
+ # Extract ZIP
+ extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
+ os.makedirs(extract_dir, exist_ok=True)
+ temp_extract_dirs.append(extract_dir)
+ print(f"DEBUG: Created extract directory: {extract_dir}")
+ with zipfile.ZipFile(filepath, 'r') as zip_ref:
+ file_count = len(zip_ref.namelist())
+ print(f"DEBUG: ZIP contains {file_count} files: {zip_ref.namelist()}")
+ zip_ref.extractall(extract_dir)
+ job_queue[job_id]['progress'] = f'Extracted {file_count} files from {filename}'
+ print(f"DEBUG: ZIP extraction completed, progress updated: {job_queue[job_id]['progress']}")
+
+ # Process PDFs in extract_dir
+ pdf_files_found = []
+ for root, dirs, files_in_zip in os.walk(extract_dir):
+ for f in files_in_zip:
+ if f.lower().endswith('.pdf'):
+ pdf_files_found.append(f)
+ pdf_path = os.path.join(root, f)
+ job_queue[job_id]['progress'] = f'Processing PDF: {f}'
+ print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
+ print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
+ pdf_info = extract_names_from_pdf(pdf_path)
+ print(f"DEBUG: PDF extraction result: {pdf_info}")
+ if 'error' in pdf_info:
+ processed_files.append(('skipped', f, pdf_info['error']))
+ job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
+ print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
+ else:
+ job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
+ print(f"DEBUG: Successfully extracted data for {pdf_info['first_name']} {pdf_info['last_name']}")
+ # Create folder structure: Episode/School/Grade/Gender/
+ school_dir = os.path.join(episode_dir, pdf_info['school'])
+ grade_dir = os.path.join(school_dir, pdf_info['grade'])
+ gender_dir = os.path.join(grade_dir, pdf_info['gender'])
+ print(f"DEBUG: Creating directories: {school_dir} -> {grade_dir} -> {gender_dir}")
+ os.makedirs(gender_dir, exist_ok=True)
+ print(f"DEBUG: Directories created successfully")
+
+ # Move file to appropriate folder with new name
+ new_path = os.path.join(gender_dir, pdf_info['filename'])
+ print(f"DEBUG: Attempting to move {pdf_path} to {new_path}")
+ counter = 1
+ while os.path.exists(new_path):
+ base, ext = os.path.splitext(pdf_info['filename'])
+ new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
+ counter += 1
+ print(f"DEBUG: File exists, trying {new_path}")
+ print(f"DEBUG: Moving file to {new_path}")
+ os.rename(pdf_path, new_path)
+ processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
+ processed_count += 1
+ job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+ print(f"DEBUG: Organized file {processed_count}/{total_files}")
+
+ print(f"DEBUG: Found {len(pdf_files_found)} PDF files in ZIP: {pdf_files_found}")
+
+ except Exception as e:
+ processed_files.append(('skipped', filename, f'ZIP processing error: {str(e)}'))
+ job_queue[job_id]['progress'] = f'Failed to process ZIP {filename}: {str(e)}'
+ print(f"ZIP processing error for {filename}: {str(e)}")
+ continue
else:
# Single PDF
- job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
- pdf_info = extract_names_from_pdf(filepath)
- if 'error' in pdf_info:
- processed_files.append(('skipped', filename, pdf_info['error']))
- job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
- else:
- job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
- # Create folder structure: Episode/School/Grade/Gender/
- school_dir = os.path.join(episode_dir, pdf_info['school'])
- grade_dir = os.path.join(school_dir, pdf_info['grade'])
- gender_dir = os.path.join(grade_dir, pdf_info['gender'])
- os.makedirs(gender_dir, exist_ok=True)
-
- # Move file to appropriate folder with new name
- new_filepath = os.path.join(gender_dir, pdf_info['filename'])
- counter = 1
- while os.path.exists(new_filepath):
- base, ext = os.path.splitext(pdf_info['filename'])
- new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
- counter += 1
- os.rename(filepath, new_filepath)
- processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
- processed_count += 1
- job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+ try:
+ job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
+ pdf_info = extract_names_from_pdf(filepath)
+ if 'error' in pdf_info:
+ processed_files.append(('skipped', filename, pdf_info['error']))
+ job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
+ else:
+ job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
+ # Create folder structure: Episode/School/Grade/Gender/
+ school_dir = os.path.join(episode_dir, pdf_info['school'])
+ grade_dir = os.path.join(school_dir, pdf_info['grade'])
+ gender_dir = os.path.join(grade_dir, pdf_info['gender'])
+ os.makedirs(gender_dir, exist_ok=True)
+
+ # Move file to appropriate folder with new name
+ new_filepath = os.path.join(gender_dir, pdf_info['filename'])
+ counter = 1
+ while os.path.exists(new_filepath):
+ base, ext = os.path.splitext(pdf_info['filename'])
+ new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
+ counter += 1
+ os.rename(filepath, new_filepath)
+ processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
+ processed_count += 1
+ job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+ except Exception as e:
+ processed_files.append(('skipped', filename, f'PDF processing error: {str(e)}'))
+ job_queue[job_id]['progress'] = f'Failed to process PDF {filename}: {str(e)}'
+ print(f"PDF processing error for {filename}: {str(e)}")
+ continue
job_queue[job_id]['progress'] = 'Creating ZIP archive...'
# Create ZIP of the entire Episode directory structure
successful_files = [p for p in processed_files if p[0] == 'success']
+ print(f"DEBUG: Starting ZIP creation, successful_files: {len(successful_files)}")
if successful_files:
try:
zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip')
+ print(f"DEBUG: Creating ZIP at {zip_output}")
with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
for root, dirs, files_in_dir in os.walk(episode_dir):
for file in files_in_dir:
file_path = os.path.join(root, file)
arcname = os.path.relpath(file_path, episode_dir)
+ print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
zip_out.write(file_path, arcname)
job_queue[job_id]['zip_path'] = zip_output
job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
+ print(f"DEBUG: ZIP creation completed successfully")
except Exception as e:
job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
job_queue[job_id]['status'] = 'error'
+ print(f"DEBUG: ZIP creation failed: {str(e)}")
else:
job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
+ print(f"DEBUG: No successful files to archive")
job_queue[job_id]['status'] = 'completed'
job_queue[job_id]['processed_files'] = processed_files
job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
+ print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")
except Exception as e:
job_queue[job_id]['status'] = 'error'
@@ -689,7 +733,11 @@ def upload_files():
print(f"Created job {job_id} with {len(file_list)} files") # Debug logging
# Start processing in background
- thread = threading.Thread(target=process_job, args=(job_id, file_list))
+ def process_with_context():
+ with app.app_context():
+ process_job(job_id, file_list)
+
+ thread = threading.Thread(target=process_with_context)
thread.daemon = True
thread.start()
@@ -761,6 +809,8 @@ def jobs():
.nav-links { margin-bottom: 20px; }
.nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
.nav-links a:hover { background-color: #e9ecef; }
+ .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
+ .nav-btn:hover { background-color: #0056b3; }
.zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
.zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
.zip-toggle:hover { background-color: #dee2e6; }
@@ -778,8 +828,8 @@ def jobs():
Processing Jobs
{% with messages = get_flashed_messages(with_categories=true) %}
{% if messages %}
@@ -850,6 +900,13 @@ def jobs():
fetch('/api/jobs', { credentials: 'same-origin' })
.then(response => response.json())
.then(data => {
+ console.log('Job update received:', data);
+ const jobs = Object.entries(data);
+ if (jobs.length > 0) {
+ jobs.forEach(([jobId, job]) => {
+ console.log(`Job ${jobId}: status=${job.status}, progress=${job.progress || 'none'}`);
+ });
+ }
const jobsContainer = document.querySelector('h1').parentNode;
const existingJobs = jobsContainer.querySelectorAll('.job');
const existingNoJobsMsg = jobsContainer.querySelectorAll('p');
@@ -928,8 +985,8 @@ def jobs():
.catch(error => console.error('Error updating jobs:', error));
}
- // Update jobs every 2 seconds
- setInterval(updateJobs, 2000);
+ // Update jobs every 1 second
+ setInterval(updateJobs, 1000);
// Collapsible section functions
function toggleZipDetails(element) {
@@ -1015,6 +1072,8 @@ def completed_jobs():
.nav-links { margin-bottom: 20px; }
.nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
.nav-links a:hover { background-color: #e9ecef; }
+ .nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
+ .nav-btn:hover { background-color: #0056b3; }
.job-stats { background-color: #e9ecef; padding: 10px; border-radius: 5px; margin-bottom: 15px; }
.file-count { color: #17a2b8; font-weight: bold; }
.zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
@@ -1034,8 +1093,8 @@ def completed_jobs():
Completed Jobs Archive
{% with messages = get_flashed_messages(with_categories=true) %}
{% if messages %}
diff --git a/test_job_processing.py b/test_job_processing.py
new file mode 100644
index 0000000..5c34c17
--- /dev/null
+++ b/test_job_processing.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""
+Simple test script to verify job processing works
+"""
+import sys
+import os
+sys.path.append(os.path.dirname(__file__))
+
+from pdf_renamer import app, process_job, job_queue
+import tempfile
+import uuid
+
+# Create app context for testing
+with app.app_context():
+ # Create a test job with dummy files
+ job_id = str(uuid.uuid4())
+ job_queue[job_id] = {
+ 'status': 'queued',
+ 'files': [],
+ 'zip_path': None,
+ 'zip_name': None,
+ 'processed_files': [],
+ 'skipped': []
+ }
+
+ # Create some dummy file entries (they don't need to exist for this test)
+ file_list = [
+ {'name': 'test1.pdf', 'path': '/tmp/test1.pdf'},
+ {'name': 'test2.pdf', 'path': '/tmp/test2.pdf'}
+ ]
+
+ job_queue[job_id]['files'] = file_list
+
+ print(f"Testing job processing for job_id: {job_id}")
+ print("Calling process_job...")
+
+ # Call process_job directly
+ process_job(job_id, file_list)
+
+ print("process_job completed")
+ print(f"Final job status: {job_queue[job_id]['status']}")
\ No newline at end of file
diff --git a/test_upload.py b/test_upload.py
new file mode 100644
index 0000000..07ad4ad
--- /dev/null
+++ b/test_upload.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""
+Test the upload and job processing flow
+"""
+import sys
+import os
+sys.path.append(os.path.dirname(__file__))
+
+from pdf_renamer import app, job_queue
+import tempfile
+import uuid
+from werkzeug.datastructures import FileStorage
+import io
+
+# Create a simple PDF-like content (not a real PDF, but enough for testing)
+pdf_content = b"%PDF-1.4\n1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj\n2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj\n3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n/Contents 4 0 R\n>>\nendobj\n4 0 obj\n<<\n/Length 44\n>>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF) Tj\nET\nendstream\nendobj\nxref\n0 5\n0000000000 65535 f\n0000000009 00000 n\n0000000058 00000 n\n0000000115 00000 n\n0000000200 00000 n\ntrailer\n<<\n/Size 5\n/Root 1 0 R\n>>\nstartxref\n284\n%%EOF"
+
+with app.test_client() as client:
+ with app.app_context():
+ # Create a test file
+ file_data = io.BytesIO(pdf_content)
+ file_data.filename = 'test.pdf'
+
+ # Simulate file upload
+ response = client.post('/', data={
+ 'files': (file_data, 'test.pdf')
+ }, content_type='multipart/form-data')
+
+ print(f"Upload response status: {response.status_code}")
+ print(f"Response data: {response.get_data(as_text=True)}")
+
+ # Check if any jobs were created
+ print(f"Current jobs: {list(job_queue.keys())}")
+ for job_id, job in job_queue.items():
+ print(f"Job {job_id}: status={job.get('status')}, files={len(job.get('files', []))}")
\ No newline at end of file