diff --git a/README.md b/README.md
index a0e2d77..ad3f719 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ A Python web application built with Flask that automatically renames PDF files b
- **Batch Processing**: Handle multiple PDFs or ZIP archives containing PDFs
- **Form Field Extraction**: Uses PyPDF2 to read PDF form fields
- **Intelligent Renaming**: Renames files to "LastName_FirstName.pdf" format
-- **Structured Organization**: Creates Episode/School/Grade folder hierarchy
+- **Structured Organization**: Creates Episode/School/Grade/Gender folder hierarchy
- **Duplicate Handling**: Automatically appends counters for duplicate names
- **ZIP Support**: Upload ZIP files and download processed results as ZIP with folder structure
- **Job Management**: Background processing with job status tracking and history
@@ -122,12 +122,14 @@ Users can choose their preferred login method on the login page.
## Important Notes
-- PDFs must contain fillable form fields named exactly: "Student First Name", "Student Last Name", "Student Grade", and "Student School"
-- All four fields must be present and filled for the PDF to be processed
-- Files without these fields or with extraction errors will be skipped
-- All processing happens server-side with temporary file storage
-- Maximum upload size is limited to 500MB
-- Output creates a structured folder hierarchy within the downloaded ZIP
+- **PDF Compatibility**: The system works with both fillable PDFs (containing form fields) and flattened PDFs (filled forms that have been saved)
+- **Required Information**: PDFs must contain student information for: "Student First Name", "Student Last Name", "Student Grade", "Student School", and "Gender"
+- **Field Extraction**: For fillable PDFs, looks for form fields with exact names. For flattened PDFs, extracts information from the text content of the first page
+- **Processing Requirements**: All five pieces of student information must be present for the PDF to be processed
+- **Error Handling**: Files without the required information or with extraction errors will be skipped with detailed error messages
+- **Security**: All processing happens server-side with temporary file storage
+- **Limits**: Maximum upload size is limited to 500MB
+- **Output**: Creates a structured folder hierarchy within the downloaded ZIP (Episode/School/Grade/Gender)
## Production Deployment
diff --git a/pdf_renamer.py b/pdf_renamer.py
index 5f3b448..c54db58 100644
--- a/pdf_renamer.py
+++ b/pdf_renamer.py
@@ -4,7 +4,7 @@ import zipfile
import threading
import uuid
from functools import wraps
-from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory
+from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory, flash, get_flashed_messages, jsonify
from werkzeug.utils import secure_filename
from PyPDF2 import PdfReader
from authlib.integrations.flask_client import OAuth
@@ -32,6 +32,168 @@ microsoft = oauth.register(
client_kwargs={'scope': 'openid profile email'}
)
+def analyze_failed_extractions(log_path):
+ """
+ Analyze failed extraction log to identify new patterns.
+ Returns suggestions for new regex patterns.
+ """
+ import re
+ from collections import defaultdict
+
+ if not os.path.exists(log_path):
+ return "No failed extraction log found."
+
+ patterns_found = defaultdict(int)
+
+ try:
+ with open(log_path, 'r', encoding='utf-8') as f:
+ content = f.read()
+
+ # Split into individual failed extractions
+ failed_entries = content.split('-' * 50)
+
+ for entry in failed_entries:
+ if 'Text content:' in entry:
+ text_part = entry.split('Text content:')[1].strip()
+ lines = text_part.split('\n')
+
+ # Analyze line patterns
+ for i, line in enumerate(lines):
+ line = line.strip()
+ if not line:
+ continue
+
+ # Look for potential patterns
+ # Gender + something
+ if re.match(r'^(Female|Male)', line):
+ patterns_found['gender_start'] += 1
+
+ # School + something
+ if re.search(r'(?:High|School|Academy|Middle|Elementary)', line):
+ patterns_found['school_in_line'] += 1
+
+ # Grade + something
+ if re.match(r'^\d{1,2}', line):
+ patterns_found['grade_start'] += 1
+
+ # Names (capitalized words)
+ name_words = re.findall(r'\b([A-Z][a-z]+)\b', line)
+ if len(name_words) >= 2:
+ patterns_found['multiple_names'] += 1
+
+ # Check for concatenated patterns
+ if re.match(r'^[A-Z][a-z]+[A-Z][a-z]+', line):
+ patterns_found['concatenated_names'] += 1
+
+ except Exception as e:
+ return f"Error analyzing log: {str(e)}"
+
+ # Generate suggestions
+ suggestions = []
+ if patterns_found.get('gender_start', 0) > 0:
+ suggestions.append("Found gender at start of lines - may need new gender-first patterns")
+ if patterns_found.get('school_in_line', 0) > 0:
+ suggestions.append("Found school names in lines - may need school-based patterns")
+ if patterns_found.get('grade_start', 0) > 0:
+ suggestions.append("Found grades at start of lines - may need grade-first patterns")
+ if patterns_found.get('multiple_names', 0) > 0:
+ suggestions.append("Found multiple capitalized words - may need better name parsing")
+ if patterns_found.get('concatenated_names', 0) > 0:
+ suggestions.append("Found concatenated names - may need better concatenation handling")
+
+ if not suggestions:
+ suggestions.append("No obvious patterns detected - may need manual inspection")
+
+def map_pattern_1(match1, match2):
+ return {
+ 'gender': match1.group(1),
+ 'school': match1.group(2).strip(),
+ 'last_name': match1.group(3),
+ 'grade': match2.group(1),
+ 'first_name': match2.group(2)
+ }
+
+def map_pattern_2(match1, match2, match3):
+ return {
+ 'first_name': match1.group(1),
+ 'gender': match2.group(1),
+ 'school': match2.group(2).strip(),
+ 'last_name': match2.group(3),
+ 'grade': match3.group(1)
+ }
+
+def map_pattern_3(match1, match2, match3):
+ return {
+ 'gender': match1.group(1),
+ 'first_name': match1.group(2),
+ 'school': match2.group(1).strip(),
+ 'last_name': match2.group(2),
+ 'grade': match3.group(1)
+ }
+
+def map_pattern_4(match1, match2, match3):
+ return {
+ 'school': match1.group(1).strip(),
+ 'grade': match1.group(2),
+ 'gender': match2.group(1),
+ 'first_name': match2.group(2),
+ 'last_name': match3.group(1)
+ }
+
+def map_pattern_5(match1, match2, match3):
+ return {
+ 'first_name': match1.group(1),
+ 'gender': match2.group(1),
+ 'grade': match2.group(2),
+ 'school': match3.group(1).strip(),
+ 'last_name': match3.group(2)
+ }
+
+def map_pattern_6(match1, match2, match3):
+ return {
+ 'first_name': match1.group(1),
+ 'school': match2.group(1).strip(),
+ 'grade': match2.group(2),
+ 'last_name': match2.group(3),
+ 'gender': match3.group(1)
+ }
+
+def map_pattern_7(match1, match2):
+ return {
+ 'gender': match1.group(1),
+ 'first_name': match1.group(2),
+ 'school': match2.group(1).strip(),
+ 'grade': match2.group(2),
+ 'last_name': match2.group(3)
+ }
+
+def map_pattern_8(match1, match2, match3):
+ return {
+ 'school': match1.group(1).strip(),
+ 'first_name': match1.group(2),
+ 'last_name': match2.group(1),
+ 'gender': match3.group(1),
+ 'grade': match3.group(2)
+ }
+
+def map_pattern_9(match1, match2, match3, match4):
+ return {
+ 'first_name': match1.group(2),
+ 'last_name': match1.group(1),
+ 'grade': match2.group(1),
+ 'school': match3.group(1).strip(),
+ 'gender': match4.group(1)
+ }
+
+def map_pattern_10(match1, match2):
+ return {
+ 'gender': match1.group(1),
+ 'school': match1.group(2).strip(),
+ 'last_name': match1.group(3),
+ 'grade': match2.group(1),
+ 'first_name': match2.group(2)
+ }
+
ALLOWED_EXTENSIONS = {'pdf', 'zip'}
# Global jobs storage (in production, use database)
@@ -51,162 +213,352 @@ def allowed_file(filename):
def extract_names_from_pdf(pdf_path):
try:
reader = PdfReader(pdf_path)
- fields = reader.get_fields()
-
- # Extract all required fields
- last_name = fields.get('Last Name', {}).get('/V', '').strip()
- first_name = fields.get('First Name', {}).get('/V', '').strip()
- grade = fields.get('Student Grade', {}).get('/V', '').strip()
- school = fields.get('Student School', {}).get('/V', '').strip()
-
- # Check for missing fields
- missing_fields = []
- if not last_name:
- missing_fields.append('Last Name')
- if not first_name:
- missing_fields.append('First Name')
- if not grade:
- missing_fields.append('Student Grade')
- if not school:
- missing_fields.append('Student School')
-
- if missing_fields:
- return {'error': f'Missing or empty fields: {", ".join(missing_fields)}'}
-
- return {
- 'filename': f"{last_name}_{first_name}.pdf",
- 'last_name': last_name,
- 'first_name': first_name,
- 'grade': grade,
- 'school': school
- }
+
+ # All PDFs are flattened, so only try text extraction from first page
+ if len(reader.pages) > 0:
+ first_page_text = reader.pages[0].extract_text()
+ extracted_data = extract_student_info_from_text(first_page_text)
+
+ if extracted_data and all(extracted_data.values()):
+ return {
+ 'filename': f"{extracted_data['last_name']}{extracted_data['first_name']}_{extracted_data['grade']}.pdf",
+ 'last_name': extracted_data['last_name'],
+ 'first_name': extracted_data['first_name'],
+ 'grade': extracted_data['grade'],
+ 'school': extracted_data['school'],
+ 'gender': extracted_data['gender']
+ }
+
+ # If we get here, we couldn't extract the data
+ # Log the failed extraction for debugging
+ import os
+ failed_log_path = os.path.join(os.path.dirname(pdf_path), 'failed_extractions.log')
+ try:
+ with open(failed_log_path, 'a', encoding='utf-8') as f:
+ f.write(f"Failed to extract from: {os.path.basename(pdf_path)}\n")
+ if len(reader.pages) > 0:
+ text = reader.pages[0].extract_text()
+ f.write(f"Text content:\n{text}\n")
+ f.write("-" * 50 + "\n")
+ except Exception as log_error:
+ pass # Don't let logging errors break the main flow
+
+ return {'error': 'Could not extract student information from PDF. Text extraction failed.'}
+
except Exception as e:
return {'error': f'PDF read error: {str(e)}'}
+def extract_student_info_from_text(text):
+ """
+ Extract student information from flattened PDF text.
+ This function handles multiple concatenation patterns and edge cases.
+ """
+ import re
+
+ extracted = {
+ 'first_name': '',
+ 'last_name': '',
+ 'grade': '',
+ 'school': '',
+ 'gender': ''
+ }
+
+ # Split text into lines for better parsing
+ lines = text.split('\n')
+
+ # First pass: Look for individual fields that appear on separate lines
+ for line in lines:
+ line = line.strip()
+ if not line:
+ continue
+
+ # Look for grade (numbers, including K-12)
+ grade_match = re.match(r'^(?:Grade\s*)?(\d{1,2}|K|k|\d+(?:st|nd|rd|th))$', line)
+ if grade_match:
+ grade = grade_match.group(1).upper()
+ if grade in ['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']:
+ extracted['grade'] = grade
+
+ # Look for gender (case insensitive, handle common variations)
+ if re.match(r'^(Male|Female|M|F|Boy|Girl)$', line, re.IGNORECASE):
+ gender = line.capitalize()
+ if gender in ['Male', 'Female']:
+ extracted['gender'] = gender
+
+ # Look for school names (more flexible pattern)
+ school_match = re.match(r'^(.+?)(?:\s+(?:High|School|Academy|Middle|Elementary|Christian|Catholic|Public|Private))$', line, re.IGNORECASE)
+ if school_match and len(school_match.group(1)) > 3: # Avoid false positives
+ extracted['school'] = line
+
+ # Second pass: Try various concatenation patterns
+ patterns = [
+ # Pattern 1: Gender + School + LastName, Grade + FirstName
+ {
+ 'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
+ 'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$',
+ 'map': map_pattern_1
+ },
+ # Pattern 2: FirstName, Gender + School + LastName, Grade
+ {
+ 'regex1': r'^([A-Z][a-z]+)$',
+ 'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
+ 'regex3': r'^(\d{1,2}|K)$',
+ 'map': map_pattern_2
+ },
+ # Pattern 3: Gender + FirstName, School + LastName, Grade
+ {
+ 'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
+ 'regex2': r'^(.+?)([A-Z][a-z]+)$',
+ 'regex3': r'^(\d{1,2}|K)$',
+ 'map': map_pattern_3
+ },
+ # Pattern 4: School + Grade, Gender + FirstName, LastName
+ {
+ 'regex1': r'^(.+?)(\d{1,2}|K)$',
+ 'regex2': r'^(Female|Male)([A-Z][a-z]+)$',
+ 'regex3': r'^([A-Z][a-z]+)$',
+ 'map': map_pattern_4
+ },
+ # Pattern 5: FirstName, Gender + Grade, School + LastName
+ {
+ 'regex1': r'^([A-Z][a-z]+)$',
+ 'regex2': r'^(Female|Male)(\d{1,2}|K)$',
+ 'regex3': r'^(.+?)([A-Z][a-z]+)$',
+ 'map': map_pattern_5
+ },
+ # Pattern 6: FirstName, School + Grade + LastName, Gender
+ {
+ 'regex1': r'^([A-Z][a-z]+)$',
+ 'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
+ 'regex3': r'^(Female|Male)$',
+ 'map': map_pattern_6
+ },
+ # Pattern 7: Gender + FirstName, School + Grade + LastName
+ {
+ 'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
+ 'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
+ 'map': map_pattern_7
+ },
+ # Pattern 8: School + FirstName, LastName, Gender + Grade
+ {
+ 'regex1': r'^(.+?)([A-Z][a-z]+)$',
+ 'regex2': r'^([A-Z][a-z]+)$',
+ 'regex3': r'^(Female|Male)(\d{1,2}|K)$',
+ 'map': map_pattern_8
+ },
+ # Pattern 9: FirstLast (concatenated), Grade, School, Gender
+ {
+ 'regex1': r'^([A-Z][a-z]+)([A-Z][a-z]+)$',
+ 'regex2': r'^(\d{1,2}|K)$',
+ 'regex3': r'^(.+)$',
+ 'regex4': r'^(Female|Male)$',
+ 'map': map_pattern_9
+ },
+ # Pattern 10: Gender + School + LastName, Grade + FirstName
+ {
+ 'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
+ 'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)',
+ 'map': map_pattern_10
+ }
+ ]
+
+ # Try each pattern
+ for pattern in patterns:
+ if all(extracted.values()): # Already found all fields
+ break
+
+ for i, line in enumerate(lines):
+ line = line.strip()
+ if not line:
+ continue
+
+ match1 = re.match(pattern['regex1'], line)
+ if not match1:
+ continue
+
+ mapped = None
+
+ if 'regex4' in pattern and i + 3 < len(lines):
+ # Try 4-line pattern
+ match2 = re.match(pattern['regex2'], lines[i + 1].strip())
+ match3 = re.match(pattern['regex3'], lines[i + 2].strip())
+ match4 = re.match(pattern['regex4'], lines[i + 3].strip())
+ if match2 and match3 and match4:
+ mapped = pattern['map'](match1, match2, match3, match4)
+ elif 'regex3' in pattern and i + 2 < len(lines):
+ # Try 3-line pattern
+ match2 = re.match(pattern['regex2'], lines[i + 1].strip())
+ match3 = re.match(pattern['regex3'], lines[i + 2].strip())
+ if match2 and match3:
+ mapped = pattern['map'](match1, match2, match3)
+ elif 'regex2' in pattern and i + 1 < len(lines):
+ # Try 2-line pattern
+ match2 = re.match(pattern['regex2'], lines[i + 1].strip())
+ if match2:
+ if 'regex3' in pattern or 'regex4' in pattern:
+ # This pattern expects more matches but only has 2 lines available
+ continue
+ mapped = pattern['map'](match1, match2)
+
+ if mapped:
+ extracted.update(mapped)
+
+ # Third pass: Fallback logic for concatenated names and missing fields
+ if not extracted['first_name'] or not extracted['last_name']:
+ # Find all capitalized words that could be names
+ all_text = ' '.join(lines)
+ name_candidates = re.findall(r'\b([A-Z][a-z]+)\b', all_text)
+
+ # Filter out common non-name words
+ common_words = {'Student', 'First', 'Last', 'Name', 'Grade', 'School', 'Gender', 'Date', 'Id', 'Envelope', 'Docusign', 'Male', 'Female', 'Details', 'Registration', 'Episode', 'Page', 'Signature', 'Participant', 'Parent', 'Guardian', 'High', 'Academy', 'Christian', 'Liberty', 'Middle', 'Elementary', 'Catholic', 'Public', 'Private', 'Other', 'Not', 'Listed'}
+ name_words = [w for w in name_candidates if w not in common_words and len(w) > 1]
+
+ # Try to assign names based on context
+ if len(name_words) >= 2:
+ if not extracted['first_name']:
+ extracted['first_name'] = name_words[0]
+ if not extracted['last_name']:
+ extracted['last_name'] = name_words[-1] # Usually last name is last
+
+ # Final validation and cleanup
+ required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
+ if all(extracted.get(field) for field in required_fields):
+ # Clean up school names (remove extra spaces, standardize)
+ extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
+ return extracted
+
+ return None
+
def process_job(job_id, files):
- job_queue[job_id]['status'] = 'processing'
- job_queue[job_id]['progress'] = 'Starting job processing...'
- processed_files = []
- temp_extract_dirs = []
-
- # Create base Episode directory for this job
- episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'Episode_{job_id}')
- os.makedirs(episode_dir, exist_ok=True)
- job_queue[job_id]['progress'] = 'Created output directory structure'
-
- # Analyze ZIP contents before processing
- zip_contents = {}
- for file_info in files:
- if file_info['name'].lower().endswith('.zip'):
- try:
- with zipfile.ZipFile(file_info['path'], 'r') as zip_ref:
- pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')]
- zip_contents[file_info['name']] = pdf_files
- except Exception as e:
- zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"]
-
- job_queue[job_id]['zip_contents'] = zip_contents
-
- total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \
- sum(len(zip_contents.get(file_info['name'], []))
- for file_info in files if file_info['name'].lower().endswith('.zip'))
-
- processed_count = 0
-
- processed_count = 0
-
- for file_info in files:
- filepath = file_info['path']
- filename = file_info['name']
+ try:
+ job_queue[job_id]['status'] = 'processing'
+ job_queue[job_id]['progress'] = 'Starting job processing...'
+ processed_files = []
+ temp_extract_dirs = []
- if filename.lower().endswith('.zip'):
- job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
- # Extract ZIP
- extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
- os.makedirs(extract_dir, exist_ok=True)
- temp_extract_dirs.append(extract_dir)
- with zipfile.ZipFile(filepath, 'r') as zip_ref:
- zip_ref.extractall(extract_dir)
- job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}'
+ # Create base Episode directory for this job
+ episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'Episode')
+ os.makedirs(episode_dir, exist_ok=True)
+ job_queue[job_id]['progress'] = 'Created output directory structure'
+
+ # Analyze ZIP contents before processing
+ zip_contents = {}
+ for file_info in files:
+ if file_info['name'].lower().endswith('.zip'):
+ try:
+ with zipfile.ZipFile(file_info['path'], 'r') as zip_ref:
+ pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')]
+ zip_contents[file_info['name']] = pdf_files
+ except Exception as e:
+ zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"]
+
+ job_queue[job_id]['zip_contents'] = zip_contents
+
+ total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \
+ sum(len(zip_contents.get(file_info['name'], []))
+ for file_info in files if file_info['name'].lower().endswith('.zip'))
+
+ processed_count = 0
+
+ for file_info in files:
+ filepath = file_info['path']
+ filename = file_info['name']
- # Process PDFs in extract_dir
- for root, dirs, files_in_zip in os.walk(extract_dir):
- for f in files_in_zip:
- if f.lower().endswith('.pdf'):
- pdf_path = os.path.join(root, f)
- job_queue[job_id]['progress'] = f'Processing PDF: {f}'
- pdf_info = extract_names_from_pdf(pdf_path)
- if 'error' in pdf_info:
- processed_files.append(('skipped', f, pdf_info['error']))
- job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
- else:
- job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
- # Create folder structure: Episode/School/Grade/
- school_dir = os.path.join(episode_dir, pdf_info['school'])
- grade_dir = os.path.join(school_dir, pdf_info['grade'])
- os.makedirs(grade_dir, exist_ok=True)
-
- # Move file to appropriate folder with new name
- new_path = os.path.join(grade_dir, pdf_info['filename'])
- counter = 1
- while os.path.exists(new_path):
- base, ext = os.path.splitext(pdf_info['filename'])
- new_path = os.path.join(grade_dir, f"{base}_{counter}{ext}")
- counter += 1
- os.rename(pdf_path, new_path)
- processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
- processed_count += 1
- job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
- else:
- # Single PDF
- job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
- pdf_info = extract_names_from_pdf(filepath)
- if 'error' in pdf_info:
- processed_files.append(('skipped', filename, pdf_info['error']))
- job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
- else:
- job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
- # Create folder structure: Episode/School/Grade/
- school_dir = os.path.join(episode_dir, pdf_info['school'])
- grade_dir = os.path.join(school_dir, pdf_info['grade'])
- os.makedirs(grade_dir, exist_ok=True)
+ if filename.lower().endswith('.zip'):
+ job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
+ # Extract ZIP
+ extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
+ os.makedirs(extract_dir, exist_ok=True)
+ temp_extract_dirs.append(extract_dir)
+ with zipfile.ZipFile(filepath, 'r') as zip_ref:
+ zip_ref.extractall(extract_dir)
+ job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}'
- # Move file to appropriate folder with new name
- new_filepath = os.path.join(grade_dir, pdf_info['filename'])
- counter = 1
- while os.path.exists(new_filepath):
- base, ext = os.path.splitext(pdf_info['filename'])
- new_filepath = os.path.join(grade_dir, f"{base}_{counter}{ext}")
- counter += 1
- os.rename(filepath, new_filepath)
- processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
- processed_count += 1
- job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
-
- job_queue[job_id]['progress'] = 'Creating ZIP archive...'
- # Create ZIP of the entire Episode directory structure
- successful_files = [p for p in processed_files if p[0] == 'success']
- if successful_files:
- try:
- zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip')
- with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
- for root, dirs, files_in_dir in os.walk(episode_dir):
- for file in files_in_dir:
- file_path = os.path.join(root, file)
- arcname = os.path.relpath(file_path, episode_dir)
- zip_out.write(file_path, arcname)
- job_queue[job_id]['zip_path'] = zip_output
- job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
- job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
- except Exception as e:
- job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
- job_queue[job_id]['status'] = 'error'
- else:
- job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
-
- job_queue[job_id]['status'] = 'completed'
- job_queue[job_id]['processed_files'] = processed_files
- job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
+ # Process PDFs in extract_dir
+ for root, dirs, files_in_zip in os.walk(extract_dir):
+ for f in files_in_zip:
+ if f.lower().endswith('.pdf'):
+ pdf_path = os.path.join(root, f)
+ job_queue[job_id]['progress'] = f'Processing PDF: {f}'
+ pdf_info = extract_names_from_pdf(pdf_path)
+ if 'error' in pdf_info:
+ processed_files.append(('skipped', f, pdf_info['error']))
+ job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
+ else:
+ job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
+ # Create folder structure: Episode/School/Grade/Gender/
+ school_dir = os.path.join(episode_dir, pdf_info['school'])
+ grade_dir = os.path.join(school_dir, pdf_info['grade'])
+ gender_dir = os.path.join(grade_dir, pdf_info['gender'])
+ os.makedirs(gender_dir, exist_ok=True)
+
+ # Move file to appropriate folder with new name
+ new_path = os.path.join(gender_dir, pdf_info['filename'])
+ counter = 1
+ while os.path.exists(new_path):
+ base, ext = os.path.splitext(pdf_info['filename'])
+ new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
+ counter += 1
+ os.rename(pdf_path, new_path)
+ processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
+ processed_count += 1
+ job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+ else:
+ # Single PDF
+ job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
+ pdf_info = extract_names_from_pdf(filepath)
+ if 'error' in pdf_info:
+ processed_files.append(('skipped', filename, pdf_info['error']))
+ job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
+ else:
+ job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
+ # Create folder structure: Episode/School/Grade/Gender/
+ school_dir = os.path.join(episode_dir, pdf_info['school'])
+ grade_dir = os.path.join(school_dir, pdf_info['grade'])
+ gender_dir = os.path.join(grade_dir, pdf_info['gender'])
+ os.makedirs(gender_dir, exist_ok=True)
+
+ # Move file to appropriate folder with new name
+ new_filepath = os.path.join(gender_dir, pdf_info['filename'])
+ counter = 1
+ while os.path.exists(new_filepath):
+ base, ext = os.path.splitext(pdf_info['filename'])
+ new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
+ counter += 1
+ os.rename(filepath, new_filepath)
+ processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
+ processed_count += 1
+ job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
+
+ job_queue[job_id]['progress'] = 'Creating ZIP archive...'
+ # Create ZIP of the entire Episode directory structure
+ successful_files = [p for p in processed_files if p[0] == 'success']
+ if successful_files:
+ try:
+ zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip')
+ with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
+ for root, dirs, files_in_dir in os.walk(episode_dir):
+ for file in files_in_dir:
+ file_path = os.path.join(root, file)
+ arcname = os.path.relpath(file_path, episode_dir)
+ zip_out.write(file_path, arcname)
+ job_queue[job_id]['zip_path'] = zip_output
+ job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
+ job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
+ except Exception as e:
+ job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
+ job_queue[job_id]['status'] = 'error'
+ else:
+ job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
+
+ job_queue[job_id]['status'] = 'completed'
+ job_queue[job_id]['processed_files'] = processed_files
+ job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
+
+ except Exception as e:
+ job_queue[job_id]['status'] = 'error'
+ job_queue[job_id]['progress'] = f'Job failed with error: {str(e)}'
+ print(f"Job {job_id} failed: {str(e)}") # Debug logging
@app.route('/login')
def login():
@@ -304,6 +656,7 @@ def upload_files():
if request.method == 'POST':
files = request.files.getlist('files')
if not files or all(not f.filename for f in files):
+ flash('No files selected.', 'error')
return redirect(url_for('upload_files'))
job_id = str(uuid.uuid4())
@@ -323,14 +676,22 @@ def upload_files():
filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
file.save(filepath)
file_list.append({'name': filename, 'path': filepath})
+ print(f"Uploaded file: {filename} -> {filepath}") # Debug logging
+
+ if not file_list:
+ flash('No valid files uploaded. Only PDF and ZIP files are allowed.', 'error')
+ job_queue.pop(job_id, None) # Remove empty job
+ return redirect(url_for('upload_files'))
job_queue[job_id]['files'] = file_list
+ print(f"Created job {job_id} with {len(file_list)} files") # Debug logging
# Start processing in background
thread = threading.Thread(target=process_job, args=(job_id, file_list))
thread.daemon = True
thread.start()
+ flash(f'Job created successfully. Processing {len(file_list)} files...', 'success')
return redirect(url_for('jobs'))
return render_template_string('''
@@ -388,6 +749,11 @@ def jobs():
.progress-text { color: #17a2b8; font-style: italic; }
.download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; }
.download-btn:hover { background-color: #0056b3; }
+ .delete-btn { background-color: #dc3545; color: white; border: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; cursor: pointer; }
+ .delete-btn:hover { background-color: #c82333; }
+ .flash { padding: 10px; margin: 10px 0; border-radius: 5px; text-align: center; }
+ .flash.success { background-color: #d4edda; color: #155724; border: 1px solid #c3e6cb; }
+ .flash.error { background-color: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
.logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
.logout:hover { background-color: #c82333; }
.zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
@@ -399,6 +765,13 @@ def jobs():
Logout
No files were successfully processed to create a ZIP archive.
+No files were successfully processed to create a ZIP archive. +
{% endif %} {% if job.skipped %}Skipped Files:
@@ -440,10 +819,90 @@ def jobs(): {% if not jobs %}No jobs yet. Upload some files to get started.
{% endif %} +