diff --git a/README.md b/README.md index a0e2d77..ad3f719 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A Python web application built with Flask that automatically renames PDF files b - **Batch Processing**: Handle multiple PDFs or ZIP archives containing PDFs - **Form Field Extraction**: Uses PyPDF2 to read PDF form fields - **Intelligent Renaming**: Renames files to "LastName_FirstName.pdf" format -- **Structured Organization**: Creates Episode/School/Grade folder hierarchy +- **Structured Organization**: Creates Episode/School/Grade/Gender folder hierarchy - **Duplicate Handling**: Automatically appends counters for duplicate names - **ZIP Support**: Upload ZIP files and download processed results as ZIP with folder structure - **Job Management**: Background processing with job status tracking and history @@ -122,12 +122,14 @@ Users can choose their preferred login method on the login page. ## Important Notes -- PDFs must contain fillable form fields named exactly: "Student First Name", "Student Last Name", "Student Grade", and "Student School" -- All four fields must be present and filled for the PDF to be processed -- Files without these fields or with extraction errors will be skipped -- All processing happens server-side with temporary file storage -- Maximum upload size is limited to 500MB -- Output creates a structured folder hierarchy within the downloaded ZIP +- **PDF Compatibility**: The system works with both fillable PDFs (containing form fields) and flattened PDFs (filled forms that have been saved) +- **Required Information**: PDFs must contain student information for: "Student First Name", "Student Last Name", "Student Grade", "Student School", and "Gender" +- **Field Extraction**: For fillable PDFs, looks for form fields with exact names. For flattened PDFs, extracts information from the text content of the first page +- **Processing Requirements**: All five pieces of student information must be present for the PDF to be processed +- **Error Handling**: Files without the required information or with extraction errors will be skipped with detailed error messages +- **Security**: All processing happens server-side with temporary file storage +- **Limits**: Maximum upload size is limited to 500MB +- **Output**: Creates a structured folder hierarchy within the downloaded ZIP (Episode/School/Grade/Gender) ## Production Deployment diff --git a/pdf_renamer.py b/pdf_renamer.py index 5f3b448..c54db58 100644 --- a/pdf_renamer.py +++ b/pdf_renamer.py @@ -4,7 +4,7 @@ import zipfile import threading import uuid from functools import wraps -from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory +from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory, flash, get_flashed_messages, jsonify from werkzeug.utils import secure_filename from PyPDF2 import PdfReader from authlib.integrations.flask_client import OAuth @@ -32,6 +32,168 @@ microsoft = oauth.register( client_kwargs={'scope': 'openid profile email'} ) +def analyze_failed_extractions(log_path): + """ + Analyze failed extraction log to identify new patterns. + Returns suggestions for new regex patterns. + """ + import re + from collections import defaultdict + + if not os.path.exists(log_path): + return "No failed extraction log found." + + patterns_found = defaultdict(int) + + try: + with open(log_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Split into individual failed extractions + failed_entries = content.split('-' * 50) + + for entry in failed_entries: + if 'Text content:' in entry: + text_part = entry.split('Text content:')[1].strip() + lines = text_part.split('\n') + + # Analyze line patterns + for i, line in enumerate(lines): + line = line.strip() + if not line: + continue + + # Look for potential patterns + # Gender + something + if re.match(r'^(Female|Male)', line): + patterns_found['gender_start'] += 1 + + # School + something + if re.search(r'(?:High|School|Academy|Middle|Elementary)', line): + patterns_found['school_in_line'] += 1 + + # Grade + something + if re.match(r'^\d{1,2}', line): + patterns_found['grade_start'] += 1 + + # Names (capitalized words) + name_words = re.findall(r'\b([A-Z][a-z]+)\b', line) + if len(name_words) >= 2: + patterns_found['multiple_names'] += 1 + + # Check for concatenated patterns + if re.match(r'^[A-Z][a-z]+[A-Z][a-z]+', line): + patterns_found['concatenated_names'] += 1 + + except Exception as e: + return f"Error analyzing log: {str(e)}" + + # Generate suggestions + suggestions = [] + if patterns_found.get('gender_start', 0) > 0: + suggestions.append("Found gender at start of lines - may need new gender-first patterns") + if patterns_found.get('school_in_line', 0) > 0: + suggestions.append("Found school names in lines - may need school-based patterns") + if patterns_found.get('grade_start', 0) > 0: + suggestions.append("Found grades at start of lines - may need grade-first patterns") + if patterns_found.get('multiple_names', 0) > 0: + suggestions.append("Found multiple capitalized words - may need better name parsing") + if patterns_found.get('concatenated_names', 0) > 0: + suggestions.append("Found concatenated names - may need better concatenation handling") + + if not suggestions: + suggestions.append("No obvious patterns detected - may need manual inspection") + +def map_pattern_1(match1, match2): + return { + 'gender': match1.group(1), + 'school': match1.group(2).strip(), + 'last_name': match1.group(3), + 'grade': match2.group(1), + 'first_name': match2.group(2) + } + +def map_pattern_2(match1, match2, match3): + return { + 'first_name': match1.group(1), + 'gender': match2.group(1), + 'school': match2.group(2).strip(), + 'last_name': match2.group(3), + 'grade': match3.group(1) + } + +def map_pattern_3(match1, match2, match3): + return { + 'gender': match1.group(1), + 'first_name': match1.group(2), + 'school': match2.group(1).strip(), + 'last_name': match2.group(2), + 'grade': match3.group(1) + } + +def map_pattern_4(match1, match2, match3): + return { + 'school': match1.group(1).strip(), + 'grade': match1.group(2), + 'gender': match2.group(1), + 'first_name': match2.group(2), + 'last_name': match3.group(1) + } + +def map_pattern_5(match1, match2, match3): + return { + 'first_name': match1.group(1), + 'gender': match2.group(1), + 'grade': match2.group(2), + 'school': match3.group(1).strip(), + 'last_name': match3.group(2) + } + +def map_pattern_6(match1, match2, match3): + return { + 'first_name': match1.group(1), + 'school': match2.group(1).strip(), + 'grade': match2.group(2), + 'last_name': match2.group(3), + 'gender': match3.group(1) + } + +def map_pattern_7(match1, match2): + return { + 'gender': match1.group(1), + 'first_name': match1.group(2), + 'school': match2.group(1).strip(), + 'grade': match2.group(2), + 'last_name': match2.group(3) + } + +def map_pattern_8(match1, match2, match3): + return { + 'school': match1.group(1).strip(), + 'first_name': match1.group(2), + 'last_name': match2.group(1), + 'gender': match3.group(1), + 'grade': match3.group(2) + } + +def map_pattern_9(match1, match2, match3, match4): + return { + 'first_name': match1.group(2), + 'last_name': match1.group(1), + 'grade': match2.group(1), + 'school': match3.group(1).strip(), + 'gender': match4.group(1) + } + +def map_pattern_10(match1, match2): + return { + 'gender': match1.group(1), + 'school': match1.group(2).strip(), + 'last_name': match1.group(3), + 'grade': match2.group(1), + 'first_name': match2.group(2) + } + ALLOWED_EXTENSIONS = {'pdf', 'zip'} # Global jobs storage (in production, use database) @@ -51,162 +213,352 @@ def allowed_file(filename): def extract_names_from_pdf(pdf_path): try: reader = PdfReader(pdf_path) - fields = reader.get_fields() - - # Extract all required fields - last_name = fields.get('Last Name', {}).get('/V', '').strip() - first_name = fields.get('First Name', {}).get('/V', '').strip() - grade = fields.get('Student Grade', {}).get('/V', '').strip() - school = fields.get('Student School', {}).get('/V', '').strip() - - # Check for missing fields - missing_fields = [] - if not last_name: - missing_fields.append('Last Name') - if not first_name: - missing_fields.append('First Name') - if not grade: - missing_fields.append('Student Grade') - if not school: - missing_fields.append('Student School') - - if missing_fields: - return {'error': f'Missing or empty fields: {", ".join(missing_fields)}'} - - return { - 'filename': f"{last_name}_{first_name}.pdf", - 'last_name': last_name, - 'first_name': first_name, - 'grade': grade, - 'school': school - } + + # All PDFs are flattened, so only try text extraction from first page + if len(reader.pages) > 0: + first_page_text = reader.pages[0].extract_text() + extracted_data = extract_student_info_from_text(first_page_text) + + if extracted_data and all(extracted_data.values()): + return { + 'filename': f"{extracted_data['last_name']}{extracted_data['first_name']}_{extracted_data['grade']}.pdf", + 'last_name': extracted_data['last_name'], + 'first_name': extracted_data['first_name'], + 'grade': extracted_data['grade'], + 'school': extracted_data['school'], + 'gender': extracted_data['gender'] + } + + # If we get here, we couldn't extract the data + # Log the failed extraction for debugging + import os + failed_log_path = os.path.join(os.path.dirname(pdf_path), 'failed_extractions.log') + try: + with open(failed_log_path, 'a', encoding='utf-8') as f: + f.write(f"Failed to extract from: {os.path.basename(pdf_path)}\n") + if len(reader.pages) > 0: + text = reader.pages[0].extract_text() + f.write(f"Text content:\n{text}\n") + f.write("-" * 50 + "\n") + except Exception as log_error: + pass # Don't let logging errors break the main flow + + return {'error': 'Could not extract student information from PDF. Text extraction failed.'} + except Exception as e: return {'error': f'PDF read error: {str(e)}'} +def extract_student_info_from_text(text): + """ + Extract student information from flattened PDF text. + This function handles multiple concatenation patterns and edge cases. + """ + import re + + extracted = { + 'first_name': '', + 'last_name': '', + 'grade': '', + 'school': '', + 'gender': '' + } + + # Split text into lines for better parsing + lines = text.split('\n') + + # First pass: Look for individual fields that appear on separate lines + for line in lines: + line = line.strip() + if not line: + continue + + # Look for grade (numbers, including K-12) + grade_match = re.match(r'^(?:Grade\s*)?(\d{1,2}|K|k|\d+(?:st|nd|rd|th))$', line) + if grade_match: + grade = grade_match.group(1).upper() + if grade in ['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']: + extracted['grade'] = grade + + # Look for gender (case insensitive, handle common variations) + if re.match(r'^(Male|Female|M|F|Boy|Girl)$', line, re.IGNORECASE): + gender = line.capitalize() + if gender in ['Male', 'Female']: + extracted['gender'] = gender + + # Look for school names (more flexible pattern) + school_match = re.match(r'^(.+?)(?:\s+(?:High|School|Academy|Middle|Elementary|Christian|Catholic|Public|Private))$', line, re.IGNORECASE) + if school_match and len(school_match.group(1)) > 3: # Avoid false positives + extracted['school'] = line + + # Second pass: Try various concatenation patterns + patterns = [ + # Pattern 1: Gender + School + LastName, Grade + FirstName + { + 'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$', + 'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$', + 'map': map_pattern_1 + }, + # Pattern 2: FirstName, Gender + School + LastName, Grade + { + 'regex1': r'^([A-Z][a-z]+)$', + 'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$', + 'regex3': r'^(\d{1,2}|K)$', + 'map': map_pattern_2 + }, + # Pattern 3: Gender + FirstName, School + LastName, Grade + { + 'regex1': r'^(Female|Male)([A-Z][a-z]+)$', + 'regex2': r'^(.+?)([A-Z][a-z]+)$', + 'regex3': r'^(\d{1,2}|K)$', + 'map': map_pattern_3 + }, + # Pattern 4: School + Grade, Gender + FirstName, LastName + { + 'regex1': r'^(.+?)(\d{1,2}|K)$', + 'regex2': r'^(Female|Male)([A-Z][a-z]+)$', + 'regex3': r'^([A-Z][a-z]+)$', + 'map': map_pattern_4 + }, + # Pattern 5: FirstName, Gender + Grade, School + LastName + { + 'regex1': r'^([A-Z][a-z]+)$', + 'regex2': r'^(Female|Male)(\d{1,2}|K)$', + 'regex3': r'^(.+?)([A-Z][a-z]+)$', + 'map': map_pattern_5 + }, + # Pattern 6: FirstName, School + Grade + LastName, Gender + { + 'regex1': r'^([A-Z][a-z]+)$', + 'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$', + 'regex3': r'^(Female|Male)$', + 'map': map_pattern_6 + }, + # Pattern 7: Gender + FirstName, School + Grade + LastName + { + 'regex1': r'^(Female|Male)([A-Z][a-z]+)$', + 'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$', + 'map': map_pattern_7 + }, + # Pattern 8: School + FirstName, LastName, Gender + Grade + { + 'regex1': r'^(.+?)([A-Z][a-z]+)$', + 'regex2': r'^([A-Z][a-z]+)$', + 'regex3': r'^(Female|Male)(\d{1,2}|K)$', + 'map': map_pattern_8 + }, + # Pattern 9: FirstLast (concatenated), Grade, School, Gender + { + 'regex1': r'^([A-Z][a-z]+)([A-Z][a-z]+)$', + 'regex2': r'^(\d{1,2}|K)$', + 'regex3': r'^(.+)$', + 'regex4': r'^(Female|Male)$', + 'map': map_pattern_9 + }, + # Pattern 10: Gender + School + LastName, Grade + FirstName + { + 'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$', + 'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)', + 'map': map_pattern_10 + } + ] + + # Try each pattern + for pattern in patterns: + if all(extracted.values()): # Already found all fields + break + + for i, line in enumerate(lines): + line = line.strip() + if not line: + continue + + match1 = re.match(pattern['regex1'], line) + if not match1: + continue + + mapped = None + + if 'regex4' in pattern and i + 3 < len(lines): + # Try 4-line pattern + match2 = re.match(pattern['regex2'], lines[i + 1].strip()) + match3 = re.match(pattern['regex3'], lines[i + 2].strip()) + match4 = re.match(pattern['regex4'], lines[i + 3].strip()) + if match2 and match3 and match4: + mapped = pattern['map'](match1, match2, match3, match4) + elif 'regex3' in pattern and i + 2 < len(lines): + # Try 3-line pattern + match2 = re.match(pattern['regex2'], lines[i + 1].strip()) + match3 = re.match(pattern['regex3'], lines[i + 2].strip()) + if match2 and match3: + mapped = pattern['map'](match1, match2, match3) + elif 'regex2' in pattern and i + 1 < len(lines): + # Try 2-line pattern + match2 = re.match(pattern['regex2'], lines[i + 1].strip()) + if match2: + if 'regex3' in pattern or 'regex4' in pattern: + # This pattern expects more matches but only has 2 lines available + continue + mapped = pattern['map'](match1, match2) + + if mapped: + extracted.update(mapped) + + # Third pass: Fallback logic for concatenated names and missing fields + if not extracted['first_name'] or not extracted['last_name']: + # Find all capitalized words that could be names + all_text = ' '.join(lines) + name_candidates = re.findall(r'\b([A-Z][a-z]+)\b', all_text) + + # Filter out common non-name words + common_words = {'Student', 'First', 'Last', 'Name', 'Grade', 'School', 'Gender', 'Date', 'Id', 'Envelope', 'Docusign', 'Male', 'Female', 'Details', 'Registration', 'Episode', 'Page', 'Signature', 'Participant', 'Parent', 'Guardian', 'High', 'Academy', 'Christian', 'Liberty', 'Middle', 'Elementary', 'Catholic', 'Public', 'Private', 'Other', 'Not', 'Listed'} + name_words = [w for w in name_candidates if w not in common_words and len(w) > 1] + + # Try to assign names based on context + if len(name_words) >= 2: + if not extracted['first_name']: + extracted['first_name'] = name_words[0] + if not extracted['last_name']: + extracted['last_name'] = name_words[-1] # Usually last name is last + + # Final validation and cleanup + required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender'] + if all(extracted.get(field) for field in required_fields): + # Clean up school names (remove extra spaces, standardize) + extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip() + return extracted + + return None + def process_job(job_id, files): - job_queue[job_id]['status'] = 'processing' - job_queue[job_id]['progress'] = 'Starting job processing...' - processed_files = [] - temp_extract_dirs = [] - - # Create base Episode directory for this job - episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'Episode_{job_id}') - os.makedirs(episode_dir, exist_ok=True) - job_queue[job_id]['progress'] = 'Created output directory structure' - - # Analyze ZIP contents before processing - zip_contents = {} - for file_info in files: - if file_info['name'].lower().endswith('.zip'): - try: - with zipfile.ZipFile(file_info['path'], 'r') as zip_ref: - pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')] - zip_contents[file_info['name']] = pdf_files - except Exception as e: - zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"] - - job_queue[job_id]['zip_contents'] = zip_contents - - total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \ - sum(len(zip_contents.get(file_info['name'], [])) - for file_info in files if file_info['name'].lower().endswith('.zip')) - - processed_count = 0 - - processed_count = 0 - - for file_info in files: - filepath = file_info['path'] - filename = file_info['name'] + try: + job_queue[job_id]['status'] = 'processing' + job_queue[job_id]['progress'] = 'Starting job processing...' + processed_files = [] + temp_extract_dirs = [] - if filename.lower().endswith('.zip'): - job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}' - # Extract ZIP - extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}') - os.makedirs(extract_dir, exist_ok=True) - temp_extract_dirs.append(extract_dir) - with zipfile.ZipFile(filepath, 'r') as zip_ref: - zip_ref.extractall(extract_dir) - job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}' + # Create base Episode directory for this job + episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'Episode') + os.makedirs(episode_dir, exist_ok=True) + job_queue[job_id]['progress'] = 'Created output directory structure' + + # Analyze ZIP contents before processing + zip_contents = {} + for file_info in files: + if file_info['name'].lower().endswith('.zip'): + try: + with zipfile.ZipFile(file_info['path'], 'r') as zip_ref: + pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')] + zip_contents[file_info['name']] = pdf_files + except Exception as e: + zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"] + + job_queue[job_id]['zip_contents'] = zip_contents + + total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \ + sum(len(zip_contents.get(file_info['name'], [])) + for file_info in files if file_info['name'].lower().endswith('.zip')) + + processed_count = 0 + + for file_info in files: + filepath = file_info['path'] + filename = file_info['name'] - # Process PDFs in extract_dir - for root, dirs, files_in_zip in os.walk(extract_dir): - for f in files_in_zip: - if f.lower().endswith('.pdf'): - pdf_path = os.path.join(root, f) - job_queue[job_id]['progress'] = f'Processing PDF: {f}' - pdf_info = extract_names_from_pdf(pdf_path) - if 'error' in pdf_info: - processed_files.append(('skipped', f, pdf_info['error'])) - job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}' - else: - job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}' - # Create folder structure: Episode/School/Grade/ - school_dir = os.path.join(episode_dir, pdf_info['school']) - grade_dir = os.path.join(school_dir, pdf_info['grade']) - os.makedirs(grade_dir, exist_ok=True) - - # Move file to appropriate folder with new name - new_path = os.path.join(grade_dir, pdf_info['filename']) - counter = 1 - while os.path.exists(new_path): - base, ext = os.path.splitext(pdf_info['filename']) - new_path = os.path.join(grade_dir, f"{base}_{counter}{ext}") - counter += 1 - os.rename(pdf_path, new_path) - processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path)) - processed_count += 1 - job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files' - else: - # Single PDF - job_queue[job_id]['progress'] = f'Processing PDF: {filename}' - pdf_info = extract_names_from_pdf(filepath) - if 'error' in pdf_info: - processed_files.append(('skipped', filename, pdf_info['error'])) - job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}' - else: - job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}' - # Create folder structure: Episode/School/Grade/ - school_dir = os.path.join(episode_dir, pdf_info['school']) - grade_dir = os.path.join(school_dir, pdf_info['grade']) - os.makedirs(grade_dir, exist_ok=True) + if filename.lower().endswith('.zip'): + job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}' + # Extract ZIP + extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}') + os.makedirs(extract_dir, exist_ok=True) + temp_extract_dirs.append(extract_dir) + with zipfile.ZipFile(filepath, 'r') as zip_ref: + zip_ref.extractall(extract_dir) + job_queue[job_id]['progress'] = f'Extracted {len(zip_ref.namelist())} files from {filename}' - # Move file to appropriate folder with new name - new_filepath = os.path.join(grade_dir, pdf_info['filename']) - counter = 1 - while os.path.exists(new_filepath): - base, ext = os.path.splitext(pdf_info['filename']) - new_filepath = os.path.join(grade_dir, f"{base}_{counter}{ext}") - counter += 1 - os.rename(filepath, new_filepath) - processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath)) - processed_count += 1 - job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files' - - job_queue[job_id]['progress'] = 'Creating ZIP archive...' - # Create ZIP of the entire Episode directory structure - successful_files = [p for p in processed_files if p[0] == 'success'] - if successful_files: - try: - zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip') - with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out: - for root, dirs, files_in_dir in os.walk(episode_dir): - for file in files_in_dir: - file_path = os.path.join(root, file) - arcname = os.path.relpath(file_path, episode_dir) - zip_out.write(file_path, arcname) - job_queue[job_id]['zip_path'] = zip_output - job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip' - job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files' - except Exception as e: - job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}' - job_queue[job_id]['status'] = 'error' - else: - job_queue[job_id]['progress'] = 'No files to archive - all files were skipped' - - job_queue[job_id]['status'] = 'completed' - job_queue[job_id]['processed_files'] = processed_files - job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped'] + # Process PDFs in extract_dir + for root, dirs, files_in_zip in os.walk(extract_dir): + for f in files_in_zip: + if f.lower().endswith('.pdf'): + pdf_path = os.path.join(root, f) + job_queue[job_id]['progress'] = f'Processing PDF: {f}' + pdf_info = extract_names_from_pdf(pdf_path) + if 'error' in pdf_info: + processed_files.append(('skipped', f, pdf_info['error'])) + job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}' + else: + job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}' + # Create folder structure: Episode/School/Grade/Gender/ + school_dir = os.path.join(episode_dir, pdf_info['school']) + grade_dir = os.path.join(school_dir, pdf_info['grade']) + gender_dir = os.path.join(grade_dir, pdf_info['gender']) + os.makedirs(gender_dir, exist_ok=True) + + # Move file to appropriate folder with new name + new_path = os.path.join(gender_dir, pdf_info['filename']) + counter = 1 + while os.path.exists(new_path): + base, ext = os.path.splitext(pdf_info['filename']) + new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}") + counter += 1 + os.rename(pdf_path, new_path) + processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path)) + processed_count += 1 + job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files' + else: + # Single PDF + job_queue[job_id]['progress'] = f'Processing PDF: {filename}' + pdf_info = extract_names_from_pdf(filepath) + if 'error' in pdf_info: + processed_files.append(('skipped', filename, pdf_info['error'])) + job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}' + else: + job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}' + # Create folder structure: Episode/School/Grade/Gender/ + school_dir = os.path.join(episode_dir, pdf_info['school']) + grade_dir = os.path.join(school_dir, pdf_info['grade']) + gender_dir = os.path.join(grade_dir, pdf_info['gender']) + os.makedirs(gender_dir, exist_ok=True) + + # Move file to appropriate folder with new name + new_filepath = os.path.join(gender_dir, pdf_info['filename']) + counter = 1 + while os.path.exists(new_filepath): + base, ext = os.path.splitext(pdf_info['filename']) + new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}") + counter += 1 + os.rename(filepath, new_filepath) + processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath)) + processed_count += 1 + job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files' + + job_queue[job_id]['progress'] = 'Creating ZIP archive...' + # Create ZIP of the entire Episode directory structure + successful_files = [p for p in processed_files if p[0] == 'success'] + if successful_files: + try: + zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip') + with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out: + for root, dirs, files_in_dir in os.walk(episode_dir): + for file in files_in_dir: + file_path = os.path.join(root, file) + arcname = os.path.relpath(file_path, episode_dir) + zip_out.write(file_path, arcname) + job_queue[job_id]['zip_path'] = zip_output + job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip' + job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files' + except Exception as e: + job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}' + job_queue[job_id]['status'] = 'error' + else: + job_queue[job_id]['progress'] = 'No files to archive - all files were skipped' + + job_queue[job_id]['status'] = 'completed' + job_queue[job_id]['processed_files'] = processed_files + job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped'] + + except Exception as e: + job_queue[job_id]['status'] = 'error' + job_queue[job_id]['progress'] = f'Job failed with error: {str(e)}' + print(f"Job {job_id} failed: {str(e)}") # Debug logging @app.route('/login') def login(): @@ -304,6 +656,7 @@ def upload_files(): if request.method == 'POST': files = request.files.getlist('files') if not files or all(not f.filename for f in files): + flash('No files selected.', 'error') return redirect(url_for('upload_files')) job_id = str(uuid.uuid4()) @@ -323,14 +676,22 @@ def upload_files(): filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}") file.save(filepath) file_list.append({'name': filename, 'path': filepath}) + print(f"Uploaded file: {filename} -> {filepath}") # Debug logging + + if not file_list: + flash('No valid files uploaded. Only PDF and ZIP files are allowed.', 'error') + job_queue.pop(job_id, None) # Remove empty job + return redirect(url_for('upload_files')) job_queue[job_id]['files'] = file_list + print(f"Created job {job_id} with {len(file_list)} files") # Debug logging # Start processing in background thread = threading.Thread(target=process_job, args=(job_id, file_list)) thread.daemon = True thread.start() + flash(f'Job created successfully. Processing {len(file_list)} files...', 'success') return redirect(url_for('jobs')) return render_template_string(''' @@ -388,6 +749,11 @@ def jobs(): .progress-text { color: #17a2b8; font-style: italic; } .download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; } .download-btn:hover { background-color: #0056b3; } + .delete-btn { background-color: #dc3545; color: white; border: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; cursor: pointer; } + .delete-btn:hover { background-color: #c82333; } + .flash { padding: 10px; margin: 10px 0; border-radius: 5px; text-align: center; } + .flash.success { background-color: #d4edda; color: #155724; border: 1px solid #c3e6cb; } + .flash.error { background-color: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; } .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; } .logout:hover { background-color: #c82333; } .zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; } @@ -399,6 +765,13 @@ def jobs(): Logout Logo

Processing Jobs

+ {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + {% for category, message in messages %} +
{{ message }}
+ {% endfor %} + {% endif %} + {% endwith %} {% for job_id, job in jobs.items() %}

Job {{ job_id[:8] }}

@@ -422,9 +795,15 @@ def jobs(): {% endif %} {% if job.status == 'completed' %} {% if job.zip_name %} -

Download ZIP

+

Download ZIP +

+ +

{% else %} -

No files were successfully processed to create a ZIP archive.

+

No files were successfully processed to create a ZIP archive. +

+ +

{% endif %} {% if job.skipped %}

Skipped Files:

@@ -440,10 +819,90 @@ def jobs(): {% if not jobs %}

No jobs yet. Upload some files to get started.

{% endif %} + ''', jobs=job_queue) +@app.route('/api/jobs') +@login_required +def api_jobs(): + return jsonify(job_queue) + @app.route('/download/') @login_required def download(filename): @@ -455,5 +914,26 @@ def download(filename): return send_from_directory(app.config['UPLOAD_FOLDER'], filename, as_attachment=True) return "File not found or access denied", 404 +@app.route('/delete/', methods=['POST']) +@login_required +def delete_job(job_id): + if job_id in job_queue: + job = job_queue[job_id] + # Only allow deletion of completed jobs + if job.get('status') == 'completed': + # Clean up the ZIP file if it exists + if job.get('zip_name'): + zip_path = os.path.join(app.config['UPLOAD_FOLDER'], job['zip_name']) + if os.path.exists(zip_path): + os.remove(zip_path) + # Remove the job from the queue + del job_queue[job_id] + flash('Job deleted successfully.', 'success') + else: + flash('Only completed jobs can be deleted.', 'error') + else: + flash('Job not found.', 'error') + return redirect('/jobs') + if __name__ == '__main__': app.run(debug=True, port=3000) \ No newline at end of file