import os import tempfile import zipfile import threading import uuid from datetime import datetime from functools import wraps from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory, flash, get_flashed_messages, jsonify from werkzeug.utils import secure_filename from PyPDF2 import PdfReader from authlib.integrations.flask_client import OAuth app = Flask(__name__) app.config['UPLOAD_FOLDER'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'processed') os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True) app.config['MAX_CONTENT_LENGTH'] = 500 * 1024 * 1024 # 500MB limit app.secret_key = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') # Entra ID Configuration TENANT_ID = os.getenv('TENANT_ID', 'your-tenant-id') CLIENT_ID = os.getenv('CLIENT_ID', 'your-client-id') CLIENT_SECRET = os.getenv('CLIENT_SECRET', 'your-client-secret') # Admin Configuration ADMIN_USERNAME = os.getenv('ADMIN_USERNAME', 'trbcit') ADMIN_PASSWORD = os.getenv('ADMIN_PASSWORD', 'atdrmbicn') oauth = OAuth(app) microsoft = oauth.register( name='microsoft', client_id=CLIENT_ID, client_secret=CLIENT_SECRET, server_metadata_url=f'https://login.microsoftonline.com/{TENANT_ID}/v2.0/.well-known/openid_configuration', client_kwargs={'scope': 'openid profile email'} ) def analyze_failed_extractions(log_path): """ Analyze failed extraction log to identify new patterns. Returns suggestions for new regex patterns. """ import re from collections import defaultdict if not os.path.exists(log_path): return "No failed extraction log found." patterns_found = defaultdict(int) try: with open(log_path, 'r', encoding='utf-8') as f: content = f.read() # Split into individual failed extractions failed_entries = content.split('-' * 50) for entry in failed_entries: if 'Text content:' in entry: text_part = entry.split('Text content:')[1].strip() lines = text_part.split('\n') # Analyze line patterns for i, line in enumerate(lines): line = line.strip() if not line: continue # Look for potential patterns # Gender + something if re.match(r'^(Female|Male)', line): patterns_found['gender_start'] += 1 # School + something if re.search(r'(?:High|School|Academy|Middle|Elementary)', line): patterns_found['school_in_line'] += 1 # Grade + something if re.match(r'^\d{1,2}', line): patterns_found['grade_start'] += 1 # Names (capitalized words) name_words = re.findall(r'\b([A-Z][a-z]+)\b', line) if len(name_words) >= 2: patterns_found['multiple_names'] += 1 # Check for concatenated patterns if re.match(r'^[A-Z][a-z]+[A-Z][a-z]+', line): patterns_found['concatenated_names'] += 1 except Exception as e: return f"Error analyzing log: {str(e)}" # Generate suggestions suggestions = [] if patterns_found.get('gender_start', 0) > 0: suggestions.append("Found gender at start of lines - may need new gender-first patterns") if patterns_found.get('school_in_line', 0) > 0: suggestions.append("Found school names in lines - may need school-based patterns") if patterns_found.get('grade_start', 0) > 0: suggestions.append("Found grades at start of lines - may need grade-first patterns") if patterns_found.get('multiple_names', 0) > 0: suggestions.append("Found multiple capitalized words - may need better name parsing") if patterns_found.get('concatenated_names', 0) > 0: suggestions.append("Found concatenated names - may need better concatenation handling") if not suggestions: suggestions.append("No obvious patterns detected - may need manual inspection") def map_pattern_1(match1, match2): return { 'gender': match1.group(1), 'school': match1.group(2).strip(), 'last_name': match1.group(3), 'grade': match2.group(1), 'first_name': match2.group(2) } def map_pattern_2(match1, match2, match3): return { 'first_name': match1.group(1), 'gender': match2.group(1), 'school': match2.group(2).strip(), 'last_name': match2.group(3), 'grade': match3.group(1) } def map_pattern_3(match1, match2, match3): return { 'gender': match1.group(1), 'first_name': match1.group(2), 'school': match2.group(1).strip(), 'last_name': match2.group(2), 'grade': match3.group(1) } def map_pattern_4(match1, match2, match3): return { 'school': match1.group(1).strip(), 'grade': match1.group(2), 'gender': match2.group(1), 'first_name': match2.group(2), 'last_name': match3.group(1) } def map_pattern_5(match1, match2, match3): return { 'first_name': match1.group(1), 'gender': match2.group(1), 'grade': match2.group(2), 'school': match3.group(1).strip(), 'last_name': match3.group(2) } def map_pattern_6(match1, match2, match3): return { 'first_name': match1.group(1), 'school': match2.group(1).strip(), 'grade': match2.group(2), 'last_name': match2.group(3), 'gender': match3.group(1) } def map_pattern_7(match1, match2): return { 'gender': match1.group(1), 'first_name': match1.group(2), 'school': match2.group(1).strip(), 'grade': match2.group(2), 'last_name': match2.group(3) } def map_pattern_8(match1, match2, match3): return { 'school': match1.group(1).strip(), 'first_name': match1.group(2), 'last_name': match2.group(1), 'gender': match3.group(1), 'grade': match3.group(2) } def map_pattern_9(match1, match2, match3, match4): return { 'first_name': match1.group(2), 'last_name': match1.group(1), 'grade': match2.group(1), 'school': match3.group(1).strip(), 'gender': match4.group(1) } def map_pattern_10(match1, match2): return { 'gender': match1.group(1), 'school': match1.group(2).strip(), 'last_name': match1.group(3), 'grade': match2.group(1), 'first_name': match2.group(2) } ALLOWED_EXTENSIONS = {'pdf', 'zip'} # Global jobs storage (in production, use database) job_queue = {} def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): if 'user' not in session and 'admin' not in session: return redirect(url_for('login')) return f(*args, **kwargs) return decorated_function def allowed_file(filename): return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS def extract_names_from_pdf(pdf_path): try: reader = PdfReader(pdf_path) # All PDFs are flattened, so only try text extraction from first page if len(reader.pages) > 0: first_page_text = reader.pages[0].extract_text() extracted_data = extract_student_info_from_text(first_page_text) if extracted_data and all(extracted_data.values()): return { 'filename': f"{extracted_data['last_name']}{extracted_data['first_name']}_{extracted_data['grade']}.pdf", 'last_name': extracted_data['last_name'], 'first_name': extracted_data['first_name'], 'grade': extracted_data['grade'], 'school': extracted_data['school'], 'gender': extracted_data['gender'] } # If we get here, we couldn't extract the data # Log the failed extraction for debugging import os failed_log_path = os.path.join(os.path.dirname(pdf_path), 'failed_extractions.log') try: with open(failed_log_path, 'a', encoding='utf-8') as f: f.write(f"Failed to extract from: {os.path.basename(pdf_path)}\n") if len(reader.pages) > 0: text = reader.pages[0].extract_text() f.write(f"Text content:\n{text}\n") f.write("-" * 50 + "\n") except Exception as log_error: pass # Don't let logging errors break the main flow return {'error': 'Could not extract student information from PDF. Text extraction failed.'} except Exception as e: return {'error': f'PDF read error: {str(e)}'} def extract_student_info_from_text(text): """ Extract student information from flattened PDF text. This function handles multiple concatenation patterns and edge cases. """ import re extracted = { 'first_name': '', 'last_name': '', 'grade': '', 'school': '', 'gender': '' } # Split text into lines for better parsing lines = text.split('\n') # First pass: Look for individual fields that appear on separate lines for line in lines: line = line.strip() if not line: continue # Look for grade (numbers, including K-12) grade_match = re.match(r'^(?:Grade\s*)?(\d{1,2}|K|k|\d+(?:st|nd|rd|th))$', line) if grade_match: grade = grade_match.group(1).upper() if grade in ['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']: extracted['grade'] = grade # Look for gender (case insensitive, handle common variations) if re.match(r'^(Male|Female|M|F|Boy|Girl)$', line, re.IGNORECASE): gender = line.capitalize() if gender in ['Male', 'Female']: extracted['gender'] = gender # Look for school names (more flexible pattern) school_match = re.match(r'^(.+?)(?:\s+(?:High|School|Academy|Middle|Elementary|Christian|Catholic|Public|Private))$', line, re.IGNORECASE) if school_match and len(school_match.group(1)) > 3: # Avoid false positives extracted['school'] = line # Second pass: Try various concatenation patterns patterns = [ # Pattern 1: Gender + School + LastName, Grade + FirstName { 'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$', 'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$', 'map': map_pattern_1 }, # Pattern 2: FirstName, Gender + School + LastName, Grade { 'regex1': r'^([A-Z][a-z]+)$', 'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$', 'regex3': r'^(\d{1,2}|K)$', 'map': map_pattern_2 }, # Pattern 3: Gender + FirstName, School + LastName, Grade { 'regex1': r'^(Female|Male)([A-Z][a-z]+)$', 'regex2': r'^(.+?)([A-Z][a-z]+)$', 'regex3': r'^(\d{1,2}|K)$', 'map': map_pattern_3 }, # Pattern 4: School + Grade, Gender + FirstName, LastName { 'regex1': r'^(.+?)(\d{1,2}|K)$', 'regex2': r'^(Female|Male)([A-Z][a-z]+)$', 'regex3': r'^([A-Z][a-z]+)$', 'map': map_pattern_4 }, # Pattern 5: FirstName, Gender + Grade, School + LastName { 'regex1': r'^([A-Z][a-z]+)$', 'regex2': r'^(Female|Male)(\d{1,2}|K)$', 'regex3': r'^(.+?)([A-Z][a-z]+)$', 'map': map_pattern_5 }, # Pattern 6: FirstName, School + Grade + LastName, Gender { 'regex1': r'^([A-Z][a-z]+)$', 'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$', 'regex3': r'^(Female|Male)$', 'map': map_pattern_6 }, # Pattern 7: Gender + FirstName, School + Grade + LastName { 'regex1': r'^(Female|Male)([A-Z][a-z]+)$', 'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$', 'map': map_pattern_7 }, # Pattern 8: School + FirstName, LastName, Gender + Grade { 'regex1': r'^(.+?)([A-Z][a-z]+)$', 'regex2': r'^([A-Z][a-z]+)$', 'regex3': r'^(Female|Male)(\d{1,2}|K)$', 'map': map_pattern_8 }, # Pattern 9: FirstLast (concatenated), Grade, School, Gender { 'regex1': r'^([A-Z][a-z]+)([A-Z][a-z]+)$', 'regex2': r'^(\d{1,2}|K)$', 'regex3': r'^(.+)$', 'regex4': r'^(Female|Male)$', 'map': map_pattern_9 }, # Pattern 10: Gender + School + LastName, Grade + FirstName { 'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$', 'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)', 'map': map_pattern_10 } ] # Try each pattern for pattern in patterns: if all(extracted.values()): # Already found all fields break for i, line in enumerate(lines): line = line.strip() if not line: continue match1 = re.match(pattern['regex1'], line) if not match1: continue mapped = None if 'regex4' in pattern and i + 3 < len(lines): # Try 4-line pattern match2 = re.match(pattern['regex2'], lines[i + 1].strip()) match3 = re.match(pattern['regex3'], lines[i + 2].strip()) match4 = re.match(pattern['regex4'], lines[i + 3].strip()) if match2 and match3 and match4: mapped = pattern['map'](match1, match2, match3, match4) elif 'regex3' in pattern and i + 2 < len(lines): # Try 3-line pattern match2 = re.match(pattern['regex2'], lines[i + 1].strip()) match3 = re.match(pattern['regex3'], lines[i + 2].strip()) if match2 and match3: mapped = pattern['map'](match1, match2, match3) elif 'regex2' in pattern and i + 1 < len(lines): # Try 2-line pattern match2 = re.match(pattern['regex2'], lines[i + 1].strip()) if match2: if 'regex3' in pattern or 'regex4' in pattern: # This pattern expects more matches but only has 2 lines available continue mapped = pattern['map'](match1, match2) if mapped: extracted.update(mapped) # Third pass: Fallback logic for concatenated names and missing fields if not extracted['first_name'] or not extracted['last_name']: # Find all capitalized words that could be names all_text = ' '.join(lines) name_candidates = re.findall(r'\b([A-Z][a-z]+)\b', all_text) # Filter out common non-name words common_words = {'Student', 'First', 'Last', 'Name', 'Grade', 'School', 'Gender', 'Date', 'Id', 'Envelope', 'Docusign', 'Male', 'Female', 'Details', 'Registration', 'Episode', 'Page', 'Signature', 'Participant', 'Parent', 'Guardian', 'High', 'Academy', 'Christian', 'Liberty', 'Middle', 'Elementary', 'Catholic', 'Public', 'Private', 'Other', 'Not', 'Listed'} name_words = [w for w in name_candidates if w not in common_words and len(w) > 1] # Try to assign names based on context if len(name_words) >= 2: if not extracted['first_name']: extracted['first_name'] = name_words[0] if not extracted['last_name']: extracted['last_name'] = name_words[-1] # Usually last name is last # Final validation and cleanup required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender'] if all(extracted.get(field) for field in required_fields): # Clean up school names (remove extra spaces, standardize) extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip() return extracted return None def process_job(job_id, files): try: print("=== DEBUG: process_job STARTED ===") job_queue[job_id]['status'] = 'processing' job_queue[job_id]['progress'] = 'Starting job processing...' processed_files = [] temp_extract_dirs = [] # Create base Episode directory for this job episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'Episode') os.makedirs(episode_dir, exist_ok=True) job_queue[job_id]['progress'] = 'Created output directory structure' # Analyze ZIP contents before processing zip_contents = {} job_queue[job_id]['progress'] = 'Analyzing uploaded files...' for file_info in files: if file_info['name'].lower().endswith('.zip'): try: with zipfile.ZipFile(file_info['path'], 'r') as zip_ref: pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')] zip_contents[file_info['name']] = pdf_files except Exception as e: zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"] job_queue[job_id]['zip_contents'] = zip_contents total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \ sum(len(zip_contents.get(file_info['name'], [])) for file_info in files if file_info['name'].lower().endswith('.zip')) job_queue[job_id]['progress'] = f'Found {total_files} files to process' processed_count = 0 for file_info in files: filepath = file_info['path'] filename = file_info['name'] if filename.lower().endswith('.zip'): try: job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}' print(f"DEBUG: Starting ZIP extraction for {filename}") # Extract ZIP extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}') os.makedirs(extract_dir, exist_ok=True) temp_extract_dirs.append(extract_dir) print(f"DEBUG: Created extract directory: {extract_dir}") with zipfile.ZipFile(filepath, 'r') as zip_ref: file_count = len(zip_ref.namelist()) print(f"DEBUG: ZIP contains {file_count} files: {zip_ref.namelist()}") zip_ref.extractall(extract_dir) job_queue[job_id]['progress'] = f'Extracted {file_count} files from {filename}' print(f"DEBUG: ZIP extraction completed, progress updated: {job_queue[job_id]['progress']}") # Process PDFs in extract_dir pdf_files_found = [] for root, dirs, files_in_zip in os.walk(extract_dir): for f in files_in_zip: if f.lower().endswith('.pdf'): pdf_files_found.append(f) pdf_path = os.path.join(root, f) job_queue[job_id]['progress'] = f'Processing PDF: {f}' print(f"DEBUG: Processing PDF: {f} at {pdf_path}") print(f"DEBUG: File exists: {os.path.exists(pdf_path)}") pdf_info = extract_names_from_pdf(pdf_path) print(f"DEBUG: PDF extraction result: {pdf_info}") if 'error' in pdf_info: processed_files.append(('skipped', f, pdf_info['error'])) job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}' print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}") else: job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}' print(f"DEBUG: Successfully extracted data for {pdf_info['first_name']} {pdf_info['last_name']}") # Create folder structure: Episode/School/Grade/Gender/ school_dir = os.path.join(episode_dir, pdf_info['school']) grade_dir = os.path.join(school_dir, pdf_info['grade']) gender_dir = os.path.join(grade_dir, pdf_info['gender']) print(f"DEBUG: Creating directories: {school_dir} -> {grade_dir} -> {gender_dir}") os.makedirs(gender_dir, exist_ok=True) print(f"DEBUG: Directories created successfully") # Move file to appropriate folder with new name new_path = os.path.join(gender_dir, pdf_info['filename']) print(f"DEBUG: Attempting to move {pdf_path} to {new_path}") counter = 1 while os.path.exists(new_path): base, ext = os.path.splitext(pdf_info['filename']) new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}") counter += 1 print(f"DEBUG: File exists, trying {new_path}") print(f"DEBUG: Moving file to {new_path}") os.rename(pdf_path, new_path) processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path)) processed_count += 1 job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files' print(f"DEBUG: Organized file {processed_count}/{total_files}") print(f"DEBUG: Found {len(pdf_files_found)} PDF files in ZIP: {pdf_files_found}") except Exception as e: processed_files.append(('skipped', filename, f'ZIP processing error: {str(e)}')) job_queue[job_id]['progress'] = f'Failed to process ZIP {filename}: {str(e)}' print(f"ZIP processing error for {filename}: {str(e)}") continue else: # Single PDF try: job_queue[job_id]['progress'] = f'Processing PDF: {filename}' pdf_info = extract_names_from_pdf(filepath) if 'error' in pdf_info: processed_files.append(('skipped', filename, pdf_info['error'])) job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}' else: job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}' # Create folder structure: Episode/School/Grade/Gender/ school_dir = os.path.join(episode_dir, pdf_info['school']) grade_dir = os.path.join(school_dir, pdf_info['grade']) gender_dir = os.path.join(grade_dir, pdf_info['gender']) os.makedirs(gender_dir, exist_ok=True) # Move file to appropriate folder with new name new_filepath = os.path.join(gender_dir, pdf_info['filename']) counter = 1 while os.path.exists(new_filepath): base, ext = os.path.splitext(pdf_info['filename']) new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}") counter += 1 os.rename(filepath, new_filepath) processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath)) processed_count += 1 job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files' except Exception as e: processed_files.append(('skipped', filename, f'PDF processing error: {str(e)}')) job_queue[job_id]['progress'] = f'Failed to process PDF {filename}: {str(e)}' print(f"PDF processing error for {filename}: {str(e)}") continue job_queue[job_id]['progress'] = 'Creating ZIP archive...' # Create ZIP of the entire Episode directory structure successful_files = [p for p in processed_files if p[0] == 'success'] print(f"DEBUG: Starting ZIP creation, successful_files: {len(successful_files)}") if successful_files: try: zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip') print(f"DEBUG: Creating ZIP at {zip_output}") with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out: for root, dirs, files_in_dir in os.walk(episode_dir): for file in files_in_dir: file_path = os.path.join(root, file) arcname = os.path.relpath(file_path, episode_dir) print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}") zip_out.write(file_path, arcname) job_queue[job_id]['zip_path'] = zip_output job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip' job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files' print(f"DEBUG: ZIP creation completed successfully") except Exception as e: job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}' job_queue[job_id]['status'] = 'error' print(f"DEBUG: ZIP creation failed: {str(e)}") else: job_queue[job_id]['progress'] = 'No files to archive - all files were skipped' print(f"DEBUG: No successful files to archive") job_queue[job_id]['status'] = 'completed' job_queue[job_id]['processed_files'] = processed_files job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped'] print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}") except Exception as e: job_queue[job_id]['status'] = 'error' job_queue[job_id]['progress'] = f'Job failed with error: {str(e)}' print(f"Job {job_id} failed: {str(e)}") # Debug logging @app.route('/login') def login(): return render_template_string('''
Status: {{ job.status.title() }}
{% if job.status == 'processing' and job.progress %}Progress: {{ job.progress }}
{% endif %}Files: {{ job.files|length }}
{% if job.zip_contents %}ZIP Contents:
{% for zip_name, pdf_files in job.zip_contents.items() %}No files were successfully processed to create a ZIP archive.
{% endif %} {% if job.skipped %}No jobs yet. Upload some files to get started.
{% endif %} ''', jobs=job_queue) @app.route('/api/jobs') @login_required def api_jobs(): return jsonify(job_queue) @app.route('/download/
Original ZIP Contents:
{% for zip_name, pdf_files in job.zip_contents.items() %}Download: 📁 Download Organized ZIP ({{ job.zip_name }})
{% endif %} {% if job.skipped %}No completed jobs yet. Upload some files to get started.
{% endif %} ''', jobs=completed_jobs) if __name__ == '__main__': app.run(debug=True, port=3000)