3299 lines
147 KiB
Python
3299 lines
147 KiB
Python
# -*- coding: utf-8 -*-
|
||
import os
|
||
import tempfile
|
||
import zipfile
|
||
import threading
|
||
import uuid
|
||
import json
|
||
from datetime import datetime
|
||
from functools import wraps
|
||
from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory, flash, get_flashed_messages, jsonify
|
||
from werkzeug.utils import secure_filename
|
||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||
from PyPDF2 import PdfReader
|
||
from authlib.integrations.flask_client import OAuth
|
||
|
||
app = Flask(__name__)
|
||
|
||
# Trust proxy headers (since we're behind Nginx)
|
||
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
|
||
|
||
app.config['UPLOAD_FOLDER'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'processed')
|
||
app.config['ARCHIVE_FOLDER'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'archives')
|
||
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
|
||
os.makedirs(app.config['ARCHIVE_FOLDER'], exist_ok=True)
|
||
app.config['MAX_CONTENT_LENGTH'] = 500 * 1024 * 1024 # 500MB limit
|
||
app.secret_key = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production')
|
||
|
||
# Secure session cookie settings
|
||
app.config['SESSION_COOKIE_SECURE'] = True # Only send cookie over HTTPS
|
||
app.config['SESSION_COOKIE_HTTPONLY'] = True # Prevent JavaScript access
|
||
app.config['SESSION_COOKIE_SAMESITE'] = 'Lax' # CSRF protection
|
||
app.config['PERMANENT_SESSION_LIFETIME'] = 3600 # 1 hour session timeout
|
||
|
||
# Entra ID Configuration
|
||
TENANT_ID = os.getenv('TENANT_ID', 'your-tenant-id')
|
||
CLIENT_ID = os.getenv('CLIENT_ID', 'your-client-id')
|
||
CLIENT_SECRET = os.getenv('CLIENT_SECRET', 'your-client-secret')
|
||
|
||
print(f"DEBUG: TENANT_ID = {TENANT_ID}")
|
||
print(f"DEBUG: CLIENT_ID = {CLIENT_ID}")
|
||
print(f"DEBUG: CLIENT_SECRET = {'*' * len(CLIENT_SECRET) if CLIENT_SECRET else 'None'}")
|
||
|
||
# Admin Configuration
|
||
ADMIN_USERNAME = os.getenv('ADMIN_USERNAME', 'trbcit')
|
||
ADMIN_PASSWORD = os.getenv('ADMIN_PASSWORD', 'atdrmbicn')
|
||
|
||
# Initialize OAuth
|
||
oauth = OAuth(app)
|
||
|
||
# Register Microsoft OAuth provider with proper configuration
|
||
try:
|
||
microsoft = oauth.register(
|
||
name='microsoft',
|
||
client_id=CLIENT_ID,
|
||
client_secret=CLIENT_SECRET,
|
||
server_metadata_url=f'https://login.microsoftonline.com/{TENANT_ID}/v2.0/.well-known/openid-configuration',
|
||
client_kwargs={
|
||
'scope': 'openid email profile',
|
||
'token_endpoint_auth_method': 'client_secret_post'
|
||
}
|
||
)
|
||
print("DEBUG: Microsoft OAuth provider registered successfully")
|
||
except Exception as e:
|
||
print(f"ERROR: Failed to register Microsoft OAuth provider: {e}")
|
||
microsoft = None
|
||
|
||
# Job persistence
|
||
JOBS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'jobs.json')
|
||
job_queue = {}
|
||
job_lock = threading.Lock()
|
||
|
||
def _save_jobs_unlocked():
|
||
"""Internal: Save jobs without acquiring lock (assumes lock is already held)"""
|
||
try:
|
||
# Convert datetime objects to strings for JSON serialization
|
||
jobs_to_save = {}
|
||
for job_id, job in job_queue.items():
|
||
jobs_to_save[job_id] = job.copy()
|
||
# Convert any datetime objects if needed
|
||
if 'created_at' in jobs_to_save[job_id]:
|
||
jobs_to_save[job_id]['created_at'] = str(jobs_to_save[job_id]['created_at'])
|
||
|
||
with open(JOBS_FILE, 'w') as f:
|
||
json.dump(jobs_to_save, f, indent=2)
|
||
except Exception as e:
|
||
print(f"ERROR: Failed to save jobs: {e}")
|
||
|
||
def save_jobs():
|
||
"""Save jobs to persistent storage"""
|
||
with job_lock:
|
||
_save_jobs_unlocked()
|
||
|
||
def load_jobs():
|
||
"""Load jobs from persistent storage"""
|
||
global job_queue
|
||
try:
|
||
if os.path.exists(JOBS_FILE):
|
||
with open(JOBS_FILE, 'r') as f:
|
||
loaded_jobs = json.load(f)
|
||
|
||
# Only load jobs where archive files still exist
|
||
valid_jobs = {}
|
||
skipped_count = 0
|
||
|
||
for job_id, job_data in loaded_jobs.items():
|
||
# Only keep completed jobs with existing archive files
|
||
if job_data.get('status') == 'completed':
|
||
zip_path = job_data.get('zip_path', '')
|
||
if zip_path and os.path.exists(zip_path):
|
||
valid_jobs[job_id] = job_data
|
||
else:
|
||
skipped_count += 1
|
||
print(f"DEBUG: Skipping job {job_id} - archive file missing: {zip_path}")
|
||
# Skip jobs that were in progress (they failed during restart)
|
||
elif job_data.get('status') in ['pending', 'processing']:
|
||
skipped_count += 1
|
||
print(f"DEBUG: Skipping incomplete job {job_id} - status: {job_data.get('status')}")
|
||
# Keep error state jobs for visibility
|
||
elif job_data.get('status') == 'error':
|
||
valid_jobs[job_id] = job_data
|
||
|
||
job_queue = valid_jobs
|
||
print(f"DEBUG: Loaded {len(job_queue)} jobs from persistent storage ({skipped_count} skipped)")
|
||
|
||
# Save the cleaned up job list back to disk
|
||
if skipped_count > 0:
|
||
save_jobs()
|
||
else:
|
||
print("DEBUG: No existing jobs file found, starting fresh")
|
||
except Exception as e:
|
||
print(f"ERROR: Failed to load jobs: {e}")
|
||
job_queue = {}
|
||
|
||
# Load jobs on startup
|
||
load_jobs()
|
||
|
||
# Job cleanup settings
|
||
JOB_RETENTION_DAYS = 7
|
||
SAVE_BATCH_SIZE = 5 # Only save after this many updates
|
||
save_counter = 0
|
||
|
||
def update_job_status(job_id, status, **kwargs):
|
||
"""Update job status and persist changes"""
|
||
with job_lock:
|
||
if job_id in job_queue:
|
||
job_queue[job_id]['status'] = status
|
||
job_queue[job_id].update(kwargs)
|
||
job_queue[job_id]['updated_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
|
||
# Only save for important status changes
|
||
if status in ['completed', 'error', 'processing']:
|
||
_save_jobs_unlocked() # Use unlocked version since we already hold the lock
|
||
|
||
def cleanup_old_jobs():
|
||
"""Remove jobs older than JOB_RETENTION_DAYS"""
|
||
from datetime import timedelta
|
||
cutoff = datetime.now() - timedelta(days=JOB_RETENTION_DAYS)
|
||
|
||
with job_lock:
|
||
jobs_to_remove = []
|
||
for job_id, job in list(job_queue.items()):
|
||
if job.get('status') == 'completed':
|
||
try:
|
||
created_at = datetime.strptime(job.get('created_at', ''), '%Y-%m-%d %H:%M:%S')
|
||
if created_at < cutoff:
|
||
# Clean up associated files
|
||
if job.get('zip_path') and os.path.exists(job['zip_path']):
|
||
try:
|
||
os.remove(job['zip_path'])
|
||
except Exception as e:
|
||
print(f"Error removing ZIP file: {e}")
|
||
|
||
# Clean up uploaded files
|
||
for file_info in job.get('files', []):
|
||
if 'path' in file_info and os.path.exists(file_info['path']):
|
||
try:
|
||
os.remove(file_info['path'])
|
||
except Exception as e:
|
||
print(f"Error removing file: {e}")
|
||
|
||
jobs_to_remove.append(job_id)
|
||
except (ValueError, TypeError) as e:
|
||
print(f"Error parsing date for job {job_id}: {e}")
|
||
|
||
for job_id in jobs_to_remove:
|
||
del job_queue[job_id]
|
||
print(f"Cleaned up old job: {job_id}")
|
||
|
||
if jobs_to_remove:
|
||
_save_jobs_unlocked() # Use unlocked version since we already hold the lock
|
||
print(f"Removed {len(jobs_to_remove)} old jobs")
|
||
|
||
def save_jobs_batched():
|
||
"""Save jobs to disk with batching to reduce I/O"""
|
||
global save_counter
|
||
save_counter += 1
|
||
|
||
# Save immediately for critical operations, otherwise batch
|
||
if save_counter >= SAVE_BATCH_SIZE:
|
||
save_jobs()
|
||
save_counter = 0
|
||
|
||
def analyze_failed_extractions(log_path):
|
||
"""
|
||
Analyze failed extraction log to identify new patterns.
|
||
Returns suggestions for new regex patterns.
|
||
"""
|
||
import re
|
||
from collections import defaultdict
|
||
|
||
if not os.path.exists(log_path):
|
||
return "No failed extraction log found."
|
||
|
||
patterns_found = defaultdict(int)
|
||
|
||
try:
|
||
with open(log_path, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
|
||
# Split into individual failed extractions
|
||
failed_entries = content.split('-' * 50)
|
||
|
||
for entry in failed_entries:
|
||
if 'Text content:' in entry:
|
||
text_part = entry.split('Text content:')[1].strip()
|
||
lines = text_part.split('\n')
|
||
|
||
# Analyze line patterns
|
||
for i, line in enumerate(lines):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
|
||
# Look for potential patterns
|
||
# Gender + something
|
||
if re.match(r'^(Female|Male)', line):
|
||
patterns_found['gender_start'] += 1
|
||
|
||
# School + something
|
||
if re.search(r'(?:High|School|Academy|Middle|Elementary)', line):
|
||
patterns_found['school_in_line'] += 1
|
||
|
||
# Grade + something
|
||
if re.match(r'^\d{1,2}', line):
|
||
patterns_found['grade_start'] += 1
|
||
|
||
# Names (capitalized words)
|
||
name_words = re.findall(r'\b([A-Z][a-z]+)\b', line)
|
||
if len(name_words) >= 2:
|
||
patterns_found['multiple_names'] += 1
|
||
|
||
# Check for concatenated patterns
|
||
if re.match(r'^[A-Z][a-z]+[A-Z][a-z]+', line):
|
||
patterns_found['concatenated_names'] += 1
|
||
|
||
except Exception as e:
|
||
return f"Error analyzing log: {str(e)}"
|
||
|
||
# Generate suggestions
|
||
suggestions = []
|
||
if patterns_found.get('gender_start', 0) > 0:
|
||
suggestions.append("Found gender at start of lines - may need new gender-first patterns")
|
||
if patterns_found.get('school_in_line', 0) > 0:
|
||
suggestions.append("Found school names in lines - may need school-based patterns")
|
||
if patterns_found.get('grade_start', 0) > 0:
|
||
suggestions.append("Found grades at start of lines - may need grade-first patterns")
|
||
if patterns_found.get('multiple_names', 0) > 0:
|
||
suggestions.append("Found multiple capitalized words - may need better name parsing")
|
||
if patterns_found.get('concatenated_names', 0) > 0:
|
||
suggestions.append("Found concatenated names - may need better concatenation handling")
|
||
|
||
if not suggestions:
|
||
suggestions.append("No obvious patterns detected - may need manual inspection")
|
||
|
||
def map_pattern_1(match1, match2):
|
||
return {
|
||
'gender': match1.group(1),
|
||
'school': match1.group(2).strip(),
|
||
'last_name': match1.group(3),
|
||
'grade': match2.group(1),
|
||
'first_name': match2.group(2)
|
||
}
|
||
|
||
def map_pattern_2(match1, match2, match3):
|
||
return {
|
||
'first_name': match1.group(1),
|
||
'gender': match2.group(1),
|
||
'school': match2.group(2).strip(),
|
||
'last_name': match2.group(3),
|
||
'grade': match3.group(1)
|
||
}
|
||
|
||
def map_pattern_3(match1, match2, match3):
|
||
return {
|
||
'gender': match1.group(1),
|
||
'first_name': match1.group(2),
|
||
'school': match2.group(1).strip(),
|
||
'last_name': match2.group(2),
|
||
'grade': match3.group(1)
|
||
}
|
||
|
||
def map_pattern_4(match1, match2, match3):
|
||
return {
|
||
'school': match1.group(1).strip(),
|
||
'grade': match1.group(2),
|
||
'gender': match2.group(1),
|
||
'first_name': match2.group(2),
|
||
'last_name': match3.group(1)
|
||
}
|
||
|
||
def map_pattern_5(match1, match2, match3):
|
||
return {
|
||
'first_name': match1.group(1),
|
||
'gender': match2.group(1),
|
||
'grade': match2.group(2),
|
||
'school': match3.group(1).strip(),
|
||
'last_name': match3.group(2)
|
||
}
|
||
|
||
def map_pattern_6(match1, match2, match3):
|
||
return {
|
||
'first_name': match1.group(1),
|
||
'school': match2.group(1).strip(),
|
||
'grade': match2.group(2),
|
||
'last_name': match2.group(3),
|
||
'gender': match3.group(1)
|
||
}
|
||
|
||
def map_pattern_7(match1, match2):
|
||
return {
|
||
'gender': match1.group(1),
|
||
'first_name': match1.group(2),
|
||
'school': match2.group(1).strip(),
|
||
'grade': match2.group(2),
|
||
'last_name': match2.group(3)
|
||
}
|
||
|
||
def map_pattern_8(match1, match2, match3):
|
||
return {
|
||
'school': match1.group(1).strip(),
|
||
'first_name': match1.group(2),
|
||
'last_name': match2.group(1),
|
||
'gender': match3.group(1),
|
||
'grade': match3.group(2)
|
||
}
|
||
|
||
def map_pattern_9(match1, match2, match3, match4):
|
||
return {
|
||
'first_name': match1.group(2),
|
||
'last_name': match1.group(1),
|
||
'grade': match2.group(1),
|
||
'school': match3.group(1).strip(),
|
||
'gender': match4.group(1)
|
||
}
|
||
|
||
def map_pattern_10(match1, match2):
|
||
return {
|
||
'gender': match1.group(1),
|
||
'school': match1.group(2).strip(),
|
||
'last_name': match1.group(3),
|
||
'grade': match2.group(1),
|
||
'first_name': match2.group(2)
|
||
}
|
||
|
||
ALLOWED_EXTENSIONS = {'pdf', 'zip'}
|
||
ALLOWED_MIME_TYPES = {
|
||
'pdf': 'application/pdf',
|
||
'zip': 'application/zip'
|
||
}
|
||
|
||
def login_required(f):
|
||
@wraps(f)
|
||
def decorated_function(*args, **kwargs):
|
||
print(f"DEBUG login_required: session = {dict(session)}")
|
||
if 'user' not in session and 'admin' not in session:
|
||
print("DEBUG: No user or admin in session, redirecting to login")
|
||
return redirect(url_for('login'))
|
||
print(f"DEBUG: User authenticated: {session.get('user', session.get('admin'))}")
|
||
return f(*args, **kwargs)
|
||
return decorated_function
|
||
|
||
def allowed_file(filename, file_stream=None):
|
||
"""
|
||
Validate file based on extension and optionally MIME type.
|
||
|
||
Args:
|
||
filename: The name of the file
|
||
file_stream: Optional file stream to check MIME type
|
||
|
||
Returns:
|
||
bool: True if file is allowed, False otherwise
|
||
"""
|
||
# Check extension
|
||
if '.' not in filename:
|
||
return False
|
||
|
||
ext = filename.rsplit('.', 1)[1].lower()
|
||
if ext not in ALLOWED_EXTENSIONS:
|
||
return False
|
||
|
||
# Additional MIME type validation if file stream provided
|
||
if file_stream:
|
||
try:
|
||
import magic
|
||
file_stream.seek(0)
|
||
mime_type = magic.from_buffer(file_stream.read(2048), mime=True)
|
||
file_stream.seek(0)
|
||
|
||
expected_mime = ALLOWED_MIME_TYPES.get(ext)
|
||
if expected_mime and mime_type != expected_mime:
|
||
print(f"MIME type mismatch: expected {expected_mime}, got {mime_type}")
|
||
return False
|
||
except ImportError:
|
||
# python-magic not installed, skip MIME check
|
||
pass
|
||
except Exception as e:
|
||
print(f"Error checking MIME type: {e}")
|
||
return False
|
||
|
||
return True
|
||
|
||
def sanitize_segment(value):
|
||
"""Sanitize a path segment to avoid invalid/unsafe filesystem characters."""
|
||
import re
|
||
if value is None:
|
||
return ''
|
||
cleaned = str(value)
|
||
# Remove path separators and other unsafe characters
|
||
cleaned = cleaned.replace(os.sep, ' ')
|
||
if os.altsep:
|
||
cleaned = cleaned.replace(os.altsep, ' ')
|
||
cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]', ' ', cleaned)
|
||
cleaned = re.sub(r'\s+', ' ', cleaned).strip()
|
||
return cleaned
|
||
|
||
def extract_names_from_pdf(pdf_path):
|
||
reader = None
|
||
try:
|
||
reader = PdfReader(pdf_path)
|
||
|
||
# All PDFs are flattened, so only try text extraction from first page
|
||
|
||
if len(reader.pages) > 0:
|
||
first_page_text = reader.pages[0].extract_text()
|
||
extracted_data = extract_student_info_from_text(first_page_text)
|
||
|
||
if extracted_data and all(extracted_data.values()):
|
||
return {
|
||
'filename': f"{extracted_data['last_name']}_{extracted_data['first_name']}_{extracted_data['grade']}.pdf",
|
||
'last_name': extracted_data['last_name'],
|
||
'first_name': extracted_data['first_name'],
|
||
'grade': extracted_data['grade'],
|
||
'school': extracted_data['school'],
|
||
'gender': extracted_data['gender']
|
||
}
|
||
# If extraction failed, include raw text for logging
|
||
return {
|
||
'error': 'Could not extract student information from PDF. Text extraction failed.',
|
||
'raw_text': first_page_text or ''
|
||
}
|
||
# If we get here, we couldn't extract the data or text
|
||
return {'error': 'Could not extract student information from PDF. No text found.'}
|
||
|
||
except Exception as e:
|
||
return {'error': f'PDF read error: {str(e)}'}
|
||
finally:
|
||
# Properly close PDF to prevent resource leaks
|
||
if reader is not None:
|
||
try:
|
||
reader.stream.close()
|
||
except:
|
||
pass
|
||
|
||
def extract_student_info_from_text(text):
|
||
import re
|
||
extracted = {
|
||
'first_name': '',
|
||
'last_name': '',
|
||
'grade': '',
|
||
'school': '',
|
||
'gender': ''
|
||
}
|
||
required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
|
||
lines = [l.strip() for l in text.split('\n')]
|
||
name_re = r"[A-Z][a-zA-Z]+(?:[-'][A-Za-zA-Z]+)*"
|
||
def norm_name(value):
|
||
return value.strip().replace('"', '').title()
|
||
|
||
# --- Custom patterns for failed extractions ---
|
||
# Pattern: GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['school'] = m1.group(2).strip()
|
||
extracted['last_name'] = m1.group(3)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['first_name'] = m2.group(2)
|
||
break
|
||
|
||
# Pattern: ConcatenatedNames, Grade, School, Gender (e.g., MaynardJenny, 11, Liberty Christian Academy, Female)
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
|
||
m3 = lines[i+2].strip()
|
||
m4 = re.match(r'^(Female|Male)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['last_name'] = m1.group(1)
|
||
extracted['first_name'] = m1.group(2)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['school'] = m3
|
||
extracted['gender'] = m4.group(1)
|
||
break
|
||
|
||
# Pattern: SchoolFirstName, LastName, GenderGrade (e.g., Liberty Christian AcademySally, Sue, Female9)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^([A-Z][a-z]+)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['first_name'] = m1.group(2)
|
||
extracted['last_name'] = m2.group(1)
|
||
extracted['gender'] = m3.group(1)
|
||
extracted['grade'] = m3.group(2)
|
||
break
|
||
|
||
# Pattern: FirstName, SchoolGradeLastName, Gender (e.g., Michael, Home School7Shuck, Male)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['grade'] = m2.group(2)
|
||
extracted['last_name'] = m2.group(3)
|
||
extracted['gender'] = m3.group(1)
|
||
break
|
||
|
||
# Pattern: GradeFirstName, SchoolLastName, Gender (e.g., 9Joshua, Liberty Christian AcademyMcCain, Male)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['first_name'] = m1.group(2)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['last_name'] = m2.group(2)
|
||
extracted['gender'] = m3.group(1)
|
||
break
|
||
|
||
# Pattern: School, GenderFirstName, GradeLastName (e.g., Other (Not Listed), FemaleSophia, 11Porter)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(.+)$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)([A-Z][a-z]+)$', lines[i+1])
|
||
m3 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['first_name'] = m2.group(2)
|
||
extracted['grade'] = m3.group(1)
|
||
extracted['last_name'] = m3.group(2)
|
||
return extracted
|
||
|
||
# Pattern: GenderGradeLastNameFirstName, School (e.g., Female11SpearAva, Home School)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(.+)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['last_name'] = m1.group(3)
|
||
extracted['first_name'] = m1.group(4)
|
||
extracted['school'] = m2.group(1).strip()
|
||
break
|
||
|
||
# Pattern: GenderGradeLastName, SchoolFirstName (e.g., Male11Seckman, New CovenantMicharl)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['last_name'] = m1.group(3)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['first_name'] = m2.group(2)
|
||
break
|
||
|
||
# Pattern: GenderLastName, GradeFirstName, School (e.g., MaleKirk, 10Jaihden "JJ", Home School)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(Female|Male)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)', lines[i+1])
|
||
m3 = re.match(r'^(.+)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['first_name'] = m2.group(2)
|
||
extracted['school'] = m3.group(1).strip()
|
||
break
|
||
|
||
# Pattern: FirstName, Grade, SchoolLastName, Gender (e.g., Tracey, 12, Other (Not Listed)Lindsay, Female)
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
|
||
m3 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i+2])
|
||
m4 = re.match(r'^(Female|Male)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['school'] = m3.group(1).strip()
|
||
extracted['last_name'] = m3.group(2)
|
||
extracted['gender'] = m4.group(1)
|
||
break
|
||
|
||
# Pattern: GenderGrade, SchoolLastNameFirstName (e.g., Female7, Brookville MiddleBrownMicah)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i])
|
||
m2 = re.match(r'^(.+?)([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['last_name'] = m2.group(2)
|
||
extracted['first_name'] = m2.group(3)
|
||
break
|
||
|
||
# Pattern: GenderSchoolGradeFirstName, LastName (e.g., FemaleRustburg High9Lexi, Wilson)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(.+?)(\d{1,2}|K)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['school'] = m1.group(2).strip()
|
||
extracted['grade'] = m1.group(3)
|
||
extracted['first_name'] = m1.group(4)
|
||
extracted['last_name'] = m2.group(1)
|
||
break
|
||
|
||
# Pattern: GenderSchoolGradeLastNameFirstName (e.g., MaleLiberty Christian Academy11SnakenbergBryson)
|
||
for i in range(len(lines) - 0):
|
||
m1 = re.match(r'^(Female|Male)(.+?)(\d{1,2}|K)([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i])
|
||
if m1:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['school'] = m1.group(2).strip()
|
||
extracted['grade'] = m1.group(3)
|
||
extracted['last_name'] = m1.group(4)
|
||
extracted['first_name'] = m1.group(5)
|
||
break
|
||
|
||
# Pattern: SchoolLastNameFirstName, Grade, Gender (e.g., Monelison MiddleCarmodyLandon, 7, Male)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(.+?)([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['first_name'] = m1.group(3)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['gender'] = m3.group(1)
|
||
break
|
||
|
||
# Pattern: FirstName, GradeLastName, School, Gender (e.g., Jesse, 6McGinniss, Timberlake Christian, Male)
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(.+)$', lines[i+2])
|
||
m4 = re.match(r'^(Female|Male)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['last_name'] = m2.group(2)
|
||
extracted['school'] = m3.group(1).strip()
|
||
extracted['gender'] = m4.group(1)
|
||
break
|
||
|
||
# Pattern: GenderFirstName, LastName, Grade, School (e.g., MaleBenjamin, Byrne-Cox, 12, Home School)
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^(Female|Male)([A-Z][a-z]+)$', lines[i])
|
||
m2 = re.match(r'^(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(\d{1,2}|K)$', lines[i+2])
|
||
m4 = re.match(r'^(.+)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['first_name'] = m1.group(2)
|
||
extracted['last_name'] = m2.group(1)
|
||
extracted['grade'] = m3.group(1)
|
||
extracted['school'] = m4.group(1).strip()
|
||
break
|
||
|
||
# Pattern: FirstName, LastName, Grade, GenderSchool (e.g., Andrew, Servin-Hernandez, 8, MaleSandusky Middle)
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(\d{1,2}|K)$', lines[i+2])
|
||
m4 = re.match(r'^(Female|Male)(.+)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['last_name'] = m2.group(1)
|
||
extracted['grade'] = m3.group(1)
|
||
extracted['gender'] = m4.group(1)
|
||
extracted['school'] = m4.group(2).strip()
|
||
break
|
||
|
||
# Pattern: SchoolGradeFirstName, GenderLastName (e.g., Temple Christian6Morgan, MaleChristian)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(.+?)(\d{1,2}|K)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['first_name'] = m1.group(3)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['last_name'] = m2.group(2)
|
||
break
|
||
|
||
# Pattern: GenderGradeLastName, SchoolFirstName (e.g., Male9Bieri, Liberty Christian AcademySJ)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+?)(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['last_name'] = m1.group(3)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['first_name'] = m2.group(2)
|
||
break
|
||
|
||
# Pattern: SchoolLastNameFirstName, GenderGrade (e.g., Home SchoolBethelAbigail, Female7)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(.+?)(' + name_re + r')(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['first_name'] = m1.group(3)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['grade'] = m2.group(2)
|
||
break
|
||
|
||
# Pattern: SchoolLastName, GradeFirstName, Gender (e.g., Other (Not Listed)Caballero, 7Mason, Male)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(.+?)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['first_name'] = m2.group(2)
|
||
extracted['gender'] = m3.group(1)
|
||
break
|
||
|
||
# Pattern: GradeLastName, School, GenderFirstName (e.g., 6Petty, Other (Not Listed), FemaleAnia)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(\d{1,2}|K)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)(' + name_re + r')$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['gender'] = m3.group(1)
|
||
extracted['first_name'] = m3.group(2)
|
||
break
|
||
|
||
# Pattern: Grade, SchoolLastNameFirstName, Gender (e.g., 9, Home SchoolSpearCharlotte, Female)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(\d{1,2}|K)$', lines[i])
|
||
m2 = re.match(r'^(.+?)(' + name_re + r')(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['last_name'] = m2.group(2)
|
||
extracted['first_name'] = m2.group(3)
|
||
extracted['gender'] = m3.group(1)
|
||
break
|
||
|
||
# Pattern: Grade, GenderSchoolLastNameFirstName (e.g., 9, FemaleHome SchoolHeagyPaige)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(\d{1,2}|K)$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(.+?)(' + name_re + r')(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['school'] = m2.group(2).strip()
|
||
extracted['last_name'] = m2.group(3)
|
||
extracted['first_name'] = m2.group(4)
|
||
break
|
||
|
||
# Pattern: GenderGradeLastName FirstName, School (e.g., Male7Trochesset Oliver, Liberty Christian Academy)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')\s+(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['last_name'] = m1.group(3)
|
||
extracted['first_name'] = m1.group(4)
|
||
extracted['school'] = m2.group(1).strip()
|
||
break
|
||
|
||
# Pattern: FirstName, SchoolLastName, Grade, Gender (e.g., Neale, Other (Not Listed)Hans, 9, Male)
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+?)(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(\d{1,2}|K)$', lines[i+2])
|
||
m4 = re.match(r'^(Female|Male)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['last_name'] = m2.group(2)
|
||
extracted['grade'] = m3.group(1)
|
||
extracted['gender'] = m4.group(1)
|
||
break
|
||
|
||
# Pattern: GradeLastNameFirstName, GenderSchool (e.g., 11DorsettJaxon, MaleLiberty Christian Academy)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(\d{1,2}|K)(' + name_re + r')(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(.+)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['first_name'] = m1.group(3)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['school'] = m2.group(2).strip()
|
||
break
|
||
|
||
# Pattern: FirstName, Grade, GenderSchoolLastName (e.g., Neale, 6, MaleOther (Not Listed)Sebastian)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)(.+?)(' + name_re + r')$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['gender'] = m3.group(1)
|
||
extracted['school'] = m3.group(2).strip()
|
||
extracted['last_name'] = m3.group(3)
|
||
break
|
||
|
||
# Pattern: Name, Grade, GenderSchool (missing first or last name)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^([A-Za-z]+)$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)(.+)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['last_name'] = norm_name(m1.group(1))
|
||
extracted['first_name'] = 'Unknown'
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['gender'] = m3.group(1)
|
||
extracted['school'] = m3.group(2).strip()
|
||
return extracted
|
||
|
||
# Pattern: GradeLastNameFirstName, School, Gender (e.g., 12ClarkAva, Jefferson Forest High, Female)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(\d{1,2}|K)(' + name_re + r')(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['first_name'] = m1.group(3)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['gender'] = m3.group(1)
|
||
break
|
||
|
||
# Pattern: LastName, Grade, GenderFirstName, School (e.g., Johnke, 6, FemaleKyla, Other (Not Listed))
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)(' + name_re + r')$', lines[i+2])
|
||
m4 = re.match(r'^(.+)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['last_name'] = m1.group(1)
|
||
extracted['grade'] = m2.group(1)
|
||
extracted['gender'] = m3.group(1)
|
||
extracted['first_name'] = m3.group(2)
|
||
extracted['school'] = m4.group(1).strip()
|
||
break
|
||
|
||
# Pattern: SchoolFirstName (case-insensitive), GenderGradeLastName (case-insensitive)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(.+?)([A-Za-z]+)$', lines[i], re.IGNORECASE)
|
||
m2 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Za-z]+)$', lines[i+1], re.IGNORECASE)
|
||
if m1 and m2:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['first_name'] = norm_name(m1.group(2))
|
||
extracted['gender'] = m2.group(1).capitalize()
|
||
extracted['grade'] = m2.group(2)
|
||
extracted['last_name'] = norm_name(m2.group(3))
|
||
break
|
||
|
||
# Pattern: LastName, GenderSchoolGradeFirstName (e.g., Porter, MaleLiberty Christian Academy8Samuel)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(.+?)(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['last_name'] = m1.group(1)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['school'] = m2.group(2).strip()
|
||
extracted['grade'] = m2.group(3)
|
||
extracted['first_name'] = m2.group(4)
|
||
break
|
||
|
||
# Pattern: LastName, GenderFirstName, Grade, School (e.g., Brainard, FemaleEmily, 10, Other (Not Listed))
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(\d{1,2}|K)$', lines[i+2])
|
||
m4 = re.match(r'^(.+)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['last_name'] = m1.group(1)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['first_name'] = m2.group(2)
|
||
extracted['grade'] = m3.group(1)
|
||
extracted['school'] = m4.group(1).strip()
|
||
break
|
||
|
||
# Pattern: LastName, GenderSchoolGradeFirstName (e.g., Mays, FemaleAmherst High9Kara)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(.+?)(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['last_name'] = m1.group(1)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['school'] = m2.group(2).strip()
|
||
extracted['grade'] = m2.group(3)
|
||
extracted['first_name'] = m2.group(4)
|
||
break
|
||
|
||
# Pattern: GradeLastName, GenderSchoolFirstName (e.g., 10Davis, FemaleE.C. Glass HighTa'Mya)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(\d{1,2}|K)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(.+?)(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['school'] = m2.group(2).strip()
|
||
extracted['first_name'] = m2.group(3)
|
||
break
|
||
|
||
# Pattern: School, GenderLastName, GradeFirstName (e.g., Liberty Christian Academy, MaleHagner, 9Jarad Jr.)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(.+)$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(\d{1,2}|K)(' + name_re + r')', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['last_name'] = m2.group(2)
|
||
extracted['grade'] = m3.group(1)
|
||
extracted['first_name'] = m3.group(2)
|
||
break
|
||
|
||
# Pattern: GenderLastName, SchoolGradeFirstName (e.g., MaleMartin, Monelison Middle7Levi)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(Female|Male)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+?)(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['gender'] = m1.group(1)
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['grade'] = m2.group(2)
|
||
extracted['first_name'] = m2.group(3)
|
||
break
|
||
|
||
# Pattern: FirstName, School, GenderGradeLastName (e.g., Lillian, Other (Not Listed), Female6Buchanan)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['gender'] = m3.group(1)
|
||
extracted['grade'] = m3.group(2)
|
||
extracted['last_name'] = m3.group(3)
|
||
break
|
||
|
||
# Pattern: FirstName, School, GenderLastName, Grade (e.g., Luke, Other (Not Listed), MaleHottle, 11)
|
||
for i in range(len(lines) - 3):
|
||
m1 = re.match(r'^(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(.+)$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)(' + name_re + r')$', lines[i+2])
|
||
m4 = re.match(r'^(\d{1,2}|K)$', lines[i+3])
|
||
if m1 and m2 and m3 and m4:
|
||
extracted['first_name'] = m1.group(1)
|
||
extracted['school'] = m2.group(1).strip()
|
||
extracted['gender'] = m3.group(1)
|
||
extracted['last_name'] = m3.group(2)
|
||
extracted['grade'] = m4.group(1)
|
||
break
|
||
|
||
# Pattern: SchoolGradeFirstName, LastName, Gender (e.g., Monelison Middle8Laila, Mays, Female)
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(r'^(.+?)(\d{1,2}|K)(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(' + name_re + r')$', lines[i+1])
|
||
m3 = re.match(r'^(Female|Male)$', lines[i+2])
|
||
if m1 and m2 and m3:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['first_name'] = m1.group(3)
|
||
extracted['last_name'] = m2.group(1)
|
||
extracted['gender'] = m3.group(1)
|
||
break
|
||
|
||
# Pattern: GradeLastNameFirstName (multi-word first), GenderSchool (e.g., 6HoskinsMary Emma, FemaleNew Covenant)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(\d{1,2}|K)(' + name_re + r')(.+)$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)(.+)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['grade'] = m1.group(1)
|
||
extracted['last_name'] = m1.group(2)
|
||
extracted['first_name'] = m1.group(3).strip()
|
||
extracted['gender'] = m2.group(1)
|
||
extracted['school'] = m2.group(2).strip()
|
||
break
|
||
|
||
# Pattern: SchoolGradeLastNameFirstName, Gender (e.g., Brookville Middle6DraganEmily, Female)
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(r'^(.+?)(\d{1,2}|K)(' + name_re + r')(' + name_re + r')$', lines[i])
|
||
m2 = re.match(r'^(Female|Male)$', lines[i+1])
|
||
if m1 and m2:
|
||
extracted['school'] = m1.group(1).strip()
|
||
extracted['grade'] = m1.group(2)
|
||
extracted['last_name'] = m1.group(3)
|
||
extracted['first_name'] = m1.group(4)
|
||
extracted['gender'] = m2.group(1)
|
||
break
|
||
|
||
if all(extracted.get(field) for field in required_fields):
|
||
return extracted
|
||
|
||
# Enhanced preprocessing: aggressively split lines with multiple fields stuck together
|
||
processed_lines = []
|
||
skip_headers = [
|
||
'student first name:', 'student last name:', 'student grade:',
|
||
'student school:', 'details', 'gender:', 'docusign envelope id:',
|
||
'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian'
|
||
]
|
||
valid_schools = [
|
||
'Amherst', 'Appomattox', 'Brookville', 'Dunbar', 'E.C. Glass', 'Forest', 'Heritage', 'Home School',
|
||
'Jefferson Forest', 'Liberty Christian Academy', 'Linkhorne', 'Monelison', 'Rustburg', 'Sandusky',
|
||
'Temple Christian', 'Timberlake Christian'
|
||
]
|
||
school_suffixes = [' High', ' Middle', ' Academy', ' Elementary', ' School', ' Christian']
|
||
all_school_names = sorted([base + suffix for base in valid_schools for suffix in [''] + school_suffixes], key=len, reverse=True)
|
||
|
||
for line in lines:
|
||
orig_line = line.strip()
|
||
# Ignore header/title/label lines
|
||
if not orig_line or any(h in orig_line.lower() for h in skip_headers):
|
||
continue
|
||
# Aggressive split: if line starts with a valid school name (with suffix) and is immediately followed by a capitalized name or digit, split
|
||
split_done = False
|
||
for school_name in all_school_names:
|
||
if orig_line.startswith(school_name):
|
||
rest = orig_line[len(school_name):]
|
||
if rest and (re.match(r'^[A-Z]', rest) or re.match(r'^\d', rest)):
|
||
processed_lines.append(school_name)
|
||
processed_lines.append(rest)
|
||
split_done = True
|
||
break
|
||
if split_done:
|
||
continue
|
||
# --- Always split off leading gender ---
|
||
m_gender = re.match(r'^(Female|Male)(.+)', orig_line)
|
||
if m_gender:
|
||
processed_lines.append(m_gender.group(1))
|
||
orig_line = m_gender.group(2).strip()
|
||
# --- Split at last school suffix if followed by capitalized name or number ---
|
||
last_pos = -1
|
||
last_suffix = ''
|
||
for suffix in school_suffixes:
|
||
idx = orig_line.rfind(suffix)
|
||
if idx != -1 and idx + len(suffix) > last_pos:
|
||
last_pos = idx + len(suffix)
|
||
last_suffix = suffix
|
||
if last_pos != -1 and last_pos < len(orig_line):
|
||
# Always split at the last suffix: everything after is not part of the school name
|
||
processed_lines.append(orig_line[:last_pos].strip())
|
||
trailing = orig_line[last_pos:].strip()
|
||
if trailing:
|
||
# Split all words (including lowercase, apostrophes, spaces, etc.) as separate lines
|
||
for part in re.findall(r"[\w'\-]+", trailing):
|
||
processed_lines.append(part)
|
||
continue
|
||
# Pattern: Gender + School + LastName (e.g., 'FemaleBrookville HighMouse')
|
||
m = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)$', orig_line)
|
||
if m:
|
||
processed_lines.append(m.group(1))
|
||
processed_lines.append(m.group(2).strip())
|
||
processed_lines.append(m.group(3))
|
||
continue
|
||
# Pattern: Grade + FirstName (e.g., '11Mickey')
|
||
m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', orig_line)
|
||
if m2:
|
||
processed_lines.append(m2.group(1))
|
||
processed_lines.append(m2.group(2))
|
||
continue
|
||
# Pattern: Two concatenated names (e.g., 'MaynardJenny')
|
||
m3 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
|
||
if m3 and orig_line not in processed_lines:
|
||
processed_lines.append(m3.group(1))
|
||
processed_lines.append(m3.group(2))
|
||
continue
|
||
# Pattern: Gender + FirstName (e.g., 'MaleJoshua')
|
||
m7 = re.match(r'^(Female|Male)([A-Z][a-z]+)$', orig_line)
|
||
if m7:
|
||
processed_lines.append(m7.group(1))
|
||
processed_lines.append(m7.group(2))
|
||
continue
|
||
# Pattern: Gender + School + FirstName + LastName + Grade (all concatenated)
|
||
m4 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)(\d{1,2}|K)$', orig_line)
|
||
if m4:
|
||
processed_lines.append(m4.group(1))
|
||
processed_lines.append(m4.group(2).strip())
|
||
processed_lines.append(m4.group(3))
|
||
processed_lines.append(m4.group(4))
|
||
processed_lines.append(m4.group(5))
|
||
continue
|
||
# Pattern: Gender + School + FirstName + LastName (no grade)
|
||
m5 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
|
||
if m5:
|
||
processed_lines.append(m5.group(1))
|
||
processed_lines.append(m5.group(2).strip())
|
||
processed_lines.append(m5.group(3))
|
||
processed_lines.append(m5.group(4))
|
||
continue
|
||
# Pattern: Grade + Gender + School + FirstName + LastName (e.g., '10FemaleHomeSchoolAbigailDober')
|
||
m6 = re.match(r'^(\d{1,2}|K)(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
|
||
if m6:
|
||
processed_lines.append(m6.group(1))
|
||
processed_lines.append(m6.group(2))
|
||
processed_lines.append(m6.group(3).strip())
|
||
processed_lines.append(m6.group(4))
|
||
processed_lines.append(m6.group(5))
|
||
continue
|
||
# Otherwise, keep line as is
|
||
processed_lines.append(orig_line)
|
||
# Remove empty lines
|
||
lines = [l for l in processed_lines if l.strip()]
|
||
|
||
# First pass: Look for individual fields that appear on separate lines
|
||
for line in lines:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
|
||
# Look for grade (numbers, including K-12)
|
||
grade_match = re.match(r'^(?:Grade\s*)?(\d{1,2}|K|k|\d+(?:st|nd|rd|th))$', line)
|
||
if grade_match:
|
||
grade = grade_match.group(1).upper()
|
||
if grade in ['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']:
|
||
extracted['grade'] = grade
|
||
|
||
# Look for gender (case insensitive, handle common variations)
|
||
if re.match(r'^(Male|Female|M|F|Boy|Girl)$', line, re.IGNORECASE):
|
||
gender = line.capitalize()
|
||
if gender in ['Male', 'Female']:
|
||
extracted['gender'] = gender
|
||
|
||
# Look for school names (more flexible pattern)
|
||
school_match = re.match(r'^(.+?)(?:\s+(?:High|School|Academy|Middle|Elementary|Christian|Catholic|Public|Private))$', line, re.IGNORECASE)
|
||
if school_match and len(school_match.group(1)) > 3: # Avoid false positives
|
||
extracted['school'] = line
|
||
|
||
# If line is all uppercase and not a known field, treat as possible school
|
||
if line.isupper() and len(line) > 5 and not any(x in line for x in ['STUDENT', 'NAME', 'GRADE', 'GENDER', 'DETAILS', 'DOCUSIGN', 'ENVELOPE', 'ID']):
|
||
extracted['school'] = line.title()
|
||
|
||
# Second pass: Try various concatenation patterns
|
||
patterns = [
|
||
# Pattern 1: Gender + School + LastName, Grade + FirstName
|
||
{
|
||
'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
|
||
'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$',
|
||
'map': lambda m1, m2: {
|
||
'gender': m1.group(1),
|
||
'school': m1.group(2).strip(),
|
||
'last_name': m1.group(3),
|
||
'grade': m2.group(1),
|
||
'first_name': m2.group(2)
|
||
}
|
||
},
|
||
# Pattern 2: FirstName, Gender + School + LastName, Grade
|
||
{
|
||
'regex1': r'^([A-Z][a-z]+)$',
|
||
'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
|
||
'regex3': r'^(\d{1,2}|K)$',
|
||
'map': lambda m1, m2, m3: {
|
||
'first_name': m1.group(1),
|
||
'gender': m2.group(1),
|
||
'school': m2.group(2).strip(),
|
||
'last_name': m2.group(3),
|
||
'grade': m3.group(1)
|
||
}
|
||
},
|
||
# Pattern 3: Gender + FirstName, School + LastName, Grade
|
||
{
|
||
'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
|
||
'regex2': r'^(.+?)([A-Z][a-z]+)$',
|
||
'regex3': r'^(\d{1,2}|K)$',
|
||
'map': lambda m1, m2, m3: {
|
||
'gender': m1.group(1),
|
||
'first_name': m1.group(2),
|
||
'school': m2.group(1).strip(),
|
||
'last_name': m2.group(2),
|
||
'grade': m3.group(1)
|
||
}
|
||
},
|
||
# Pattern 4: School + Grade, Gender + FirstName, LastName
|
||
{
|
||
'regex1': r'^(.+?)(\d{1,2}|K)$',
|
||
'regex2': r'^(Female|Male)([A-Z][a-z]+)$',
|
||
'regex3': r'^([A-Z][a-z]+)$',
|
||
'map': lambda m1, m2, m3: {
|
||
'school': m1.group(1).strip(),
|
||
'grade': m1.group(2),
|
||
'gender': m2.group(1),
|
||
'first_name': m2.group(2),
|
||
'last_name': m3.group(1)
|
||
}
|
||
},
|
||
# Pattern 5: FirstName, Gender + Grade, School + LastName
|
||
{
|
||
'regex1': r'^([A-Z][a-z]+)$',
|
||
'regex2': r'^(Female|Male)(\d{1,2}|K)$',
|
||
'regex3': r'^(.+?)([A-Z][a-z]+)$',
|
||
'map': lambda m1, m2, m3: {
|
||
'first_name': m1.group(1),
|
||
'gender': m2.group(1),
|
||
'grade': m2.group(2),
|
||
'school': m3.group(1).strip(),
|
||
'last_name': m3.group(2)
|
||
}
|
||
},
|
||
# Pattern 6: FirstName, School + Grade + LastName, Gender
|
||
{
|
||
'regex1': r'^([A-Z][a-z]+)$',
|
||
'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
|
||
'regex3': r'^(Female|Male)$',
|
||
'map': lambda m1, m2, m3: {
|
||
'first_name': m1.group(1),
|
||
'school': m2.group(1).strip(),
|
||
'grade': m2.group(2),
|
||
'last_name': m2.group(3),
|
||
'gender': m3.group(1)
|
||
}
|
||
},
|
||
# Pattern 7: Gender + FirstName, School + Grade + LastName
|
||
{
|
||
'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
|
||
'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
|
||
'map': lambda m1, m2: {
|
||
'gender': m1.group(1),
|
||
'first_name': m1.group(2),
|
||
'school': m2.group(1).strip(),
|
||
'grade': m2.group(2),
|
||
'last_name': m2.group(3)
|
||
}
|
||
},
|
||
]
|
||
|
||
for pattern in patterns:
|
||
if 'regex3' in pattern:
|
||
for i in range(len(lines) - 2):
|
||
m1 = re.match(pattern['regex1'], lines[i])
|
||
m2 = re.match(pattern['regex2'], lines[i+1])
|
||
m3 = re.match(pattern['regex3'], lines[i+2])
|
||
if m1 and m2 and m3:
|
||
mapped = pattern['map'](m1, m2, m3)
|
||
extracted.update(mapped)
|
||
break
|
||
else:
|
||
for i in range(len(lines) - 1):
|
||
m1 = re.match(pattern['regex1'], lines[i])
|
||
m2 = re.match(pattern['regex2'], lines[i+1])
|
||
if m1 and m2:
|
||
mapped = pattern['map'](m1, m2)
|
||
extracted.update(mapped)
|
||
break
|
||
if all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
|
||
break
|
||
|
||
# Post-processing and validation
|
||
required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
|
||
if all(extracted.get(field) for field in required_fields):
|
||
# Clean up school name: remove trailing student name or stray numbers, but keep valid suffixes
|
||
school_clean = extracted['school'].strip()
|
||
suffix_keep = {'School', 'Academy', 'Middle', 'High', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'}
|
||
parts = school_clean.split()
|
||
if parts:
|
||
last_part = parts[-1]
|
||
first_name = extracted['first_name'] or ''
|
||
last_name = extracted['last_name'] or ''
|
||
if last_part.isdigit():
|
||
parts = parts[:-1]
|
||
elif last_part in {first_name, last_name}:
|
||
parts = parts[:-1]
|
||
elif last_part not in suffix_keep and last_part in {first_name.title(), last_name.title()}:
|
||
parts = parts[:-1]
|
||
school_clean = ' '.join(parts).strip()
|
||
# Remove any names from school if they match first/last name
|
||
for n in [extracted['first_name'], extracted['last_name']]:
|
||
if n:
|
||
school_clean = re.sub(rf'\b{re.escape(n)}\b', '', school_clean).strip()
|
||
extracted['school'] = school_clean
|
||
# Final check: discard invalid schools (pure grade or same as student name)
|
||
school_clean = extracted['school'].strip()
|
||
if re.fullmatch(r'(\d{1,2}|K)', school_clean):
|
||
extracted['school'] = ''
|
||
else:
|
||
first_lower = extracted['first_name'].lower() if extracted['first_name'] else ''
|
||
last_lower = extracted['last_name'].lower() if extracted['last_name'] else ''
|
||
if school_clean.lower() in {first_lower, last_lower}:
|
||
extracted['school'] = ''
|
||
else:
|
||
extracted['school'] = school_clean
|
||
return extracted
|
||
|
||
# Fallback: Try to extract fields from any order in the lines (for highly concatenated or out-of-order data)
|
||
forbidden_names = set(['School', 'Home', 'High', 'Middle', 'Academy', 'Christian', 'Temple', 'Rustburg', 'Liberty', 'Forest', 'Heritage', 'Amherst', 'Appomattox', 'Dunbar', 'Glass', 'Jefferson', 'Linkhorne', 'Monelison', 'Sandusky', 'Timberlake'])
|
||
gender_words = set(['Male', 'Female'])
|
||
grade_words = set([str(i) for i in range(1,13)] + ['K'])
|
||
school_name = extracted['school'].lower()
|
||
names = []
|
||
for line in lines:
|
||
for word in line.split():
|
||
w_lower = word.lower()
|
||
# Exclude if word matches or contains any part of the school name
|
||
if (re.match(r'^[A-Z][a-z]+$', word)
|
||
and word not in forbidden_names
|
||
and word not in gender_words
|
||
and word not in grade_words
|
||
and school_name not in w_lower
|
||
and w_lower not in school_name):
|
||
names.append(word)
|
||
# Require two distinct names
|
||
if len(names) >= 2 and names[0] != names[1]:
|
||
if not extracted['first_name']:
|
||
extracted['first_name'] = names[0]
|
||
if not extracted['last_name']:
|
||
extracted['last_name'] = names[1]
|
||
else:
|
||
return None
|
||
# Look for grade
|
||
for line in lines:
|
||
m = re.search(r'(\d{1,2}|K)', line)
|
||
if m and not extracted['grade']:
|
||
extracted['grade'] = m.group(1)
|
||
# Look for gender
|
||
for line in lines:
|
||
if 'female' in line.lower() and not extracted['gender']:
|
||
extracted['gender'] = 'Female'
|
||
elif 'male' in line.lower() and not extracted['gender']:
|
||
extracted['gender'] = 'Male'
|
||
# Look for school (any line with 'school', 'academy', 'high', etc.)
|
||
for line in lines:
|
||
if any(x in line.lower() for x in ['school', 'academy', 'high', 'middle', 'elementary', 'christian', 'catholic', 'public', 'private']):
|
||
extracted['school'] = line.strip()
|
||
break
|
||
if all(extracted.get(field) for field in required_fields):
|
||
extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
|
||
return extracted
|
||
|
||
return None
|
||
|
||
def process_job(job_id, files):
|
||
temp_extract_dirs = []
|
||
job_failed = False
|
||
try:
|
||
print("=== DEBUG: process_job STARTED ===")
|
||
with job_lock:
|
||
job_queue[job_id]['status'] = 'processing'
|
||
job_queue[job_id]['progress'] = 'Starting job processing...'
|
||
save_jobs() # Persist initial progress
|
||
# Create base Episode directory for this job
|
||
episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'Episode_{job_id}')
|
||
os.makedirs(episode_dir, exist_ok=True)
|
||
processed_files = []
|
||
skipped_dir = os.path.join(episode_dir, 'Skipped')
|
||
os.makedirs(skipped_dir, exist_ok=True)
|
||
skipped_log_path = os.path.join(skipped_dir, 'skipped_files.log')
|
||
job_queue[job_id]['progress'] = 'Created output directory structure'
|
||
|
||
# Analyze ZIP contents before processing
|
||
zip_contents = {}
|
||
job_queue[job_id]['progress'] = 'Analyzing uploaded files...'
|
||
for file_info in files:
|
||
if file_info['name'].lower().endswith('.zip'):
|
||
try:
|
||
with zipfile.ZipFile(file_info['path'], 'r') as zip_ref:
|
||
pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')]
|
||
zip_contents[file_info['name']] = pdf_files
|
||
except Exception as e:
|
||
zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"]
|
||
|
||
job_queue[job_id]['zip_contents'] = zip_contents
|
||
|
||
total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \
|
||
sum(len(zip_contents.get(file_info['name'], []))
|
||
for file_info in files if file_info['name'].lower().endswith('.zip'))
|
||
|
||
job_queue[job_id]['progress'] = f'Found {total_files} files to process'
|
||
save_jobs() # Persist file count
|
||
|
||
processed_count = 0
|
||
|
||
for file_info in files:
|
||
filepath = file_info['path']
|
||
filename = file_info['name']
|
||
|
||
if filename.lower().endswith('.zip'):
|
||
try:
|
||
job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
|
||
print(f"DEBUG: Starting ZIP extraction for {filename}")
|
||
# Extract ZIP
|
||
extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
|
||
os.makedirs(extract_dir, exist_ok=True)
|
||
temp_extract_dirs.append(extract_dir)
|
||
print(f"DEBUG: Created extract directory: {extract_dir}")
|
||
with zipfile.ZipFile(filepath, 'r') as zip_ref:
|
||
file_count = len(zip_ref.namelist())
|
||
print(f"DEBUG: ZIP contains {file_count} files: {zip_ref.namelist()}")
|
||
zip_ref.extractall(extract_dir)
|
||
job_queue[job_id]['progress'] = f'Extracted {file_count} files from {filename}'
|
||
save_jobs() # Persist ZIP extraction progress
|
||
print(f"DEBUG: ZIP extraction completed, progress updated: {job_queue[job_id]['progress']}")
|
||
|
||
# Process PDFs in extract_dir
|
||
pdf_files_found = []
|
||
for root, dirs, files_in_zip in os.walk(extract_dir):
|
||
for f in files_in_zip:
|
||
if f.lower().endswith('.pdf'):
|
||
pdf_files_found.append(f)
|
||
pdf_path = os.path.join(root, f)
|
||
job_queue[job_id]['progress'] = f'Processing PDF: {f}'
|
||
save_jobs() # Persist current file progress
|
||
print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
|
||
print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
|
||
# Skip MacOS resource fork files (._*)
|
||
if f.startswith('._'):
|
||
print(f"DEBUG: Skipping MacOS resource fork file: {f}")
|
||
continue
|
||
pdf_info = extract_names_from_pdf(pdf_path)
|
||
print(f"DEBUG: PDF extraction result: {pdf_info}")
|
||
if 'error' in pdf_info:
|
||
# Move skipped file to Skipped/
|
||
skipped_pdf_path = os.path.join(skipped_dir, f)
|
||
try:
|
||
os.rename(pdf_path, skipped_pdf_path)
|
||
except Exception as move_err:
|
||
print(f"WARNING: Could not move skipped PDF {f}: {move_err}")
|
||
# Log skip reason and raw text if available
|
||
with open(skipped_log_path, 'a', encoding='utf-8') as slog:
|
||
slog.write(f"File: {f}\nReason: {pdf_info['error']}\n")
|
||
if 'raw_text' in pdf_info:
|
||
slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
|
||
slog.write("-"*50 + "\n")
|
||
# Store reason and raw_text for web GUI
|
||
processed_files.append(('skipped', f, pdf_info['error'], pdf_info.get('raw_text', '')))
|
||
job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
|
||
print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
|
||
else:
|
||
job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
|
||
print(f"DEBUG: Successfully extracted data for {pdf_info['first_name']} {pdf_info['last_name']}")
|
||
# Create folder structure: Episode/School/Grade/Gender/
|
||
school_dir = os.path.join(episode_dir, sanitize_segment(pdf_info['school']) or 'Unknown_School')
|
||
grade_dir = os.path.join(school_dir, sanitize_segment(pdf_info['grade']) or 'Unknown_Grade')
|
||
gender_dir = os.path.join(grade_dir, sanitize_segment(pdf_info['gender']) or 'Unknown_Gender')
|
||
print(f"DEBUG: Creating directories: {school_dir} -> {grade_dir} -> {gender_dir}")
|
||
os.makedirs(gender_dir, exist_ok=True)
|
||
print(f"DEBUG: Directories created successfully")
|
||
|
||
# Move file to appropriate folder with new name
|
||
safe_filename = secure_filename(pdf_info['filename']) or pdf_info['filename']
|
||
new_path = os.path.join(gender_dir, safe_filename)
|
||
print(f"DEBUG: Attempting to move {pdf_path} to {new_path}")
|
||
counter = 1
|
||
while os.path.exists(new_path):
|
||
base, ext = os.path.splitext(pdf_info['filename'])
|
||
new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
|
||
counter += 1
|
||
print(f"DEBUG: File exists, trying {new_path}")
|
||
print(f"DEBUG: Moving file to {new_path}")
|
||
os.rename(pdf_path, new_path)
|
||
processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
|
||
processed_count += 1
|
||
job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
|
||
# Batch saves - only every 5 files
|
||
if processed_count % 5 == 0:
|
||
save_jobs()
|
||
print(f"DEBUG: Organized file {processed_count}/{total_files}")
|
||
|
||
print(f"DEBUG: Found {len(pdf_files_found)} PDF files in ZIP: {pdf_files_found}")
|
||
|
||
except Exception as e:
|
||
processed_files.append(('skipped', filename, f'ZIP processing error: {str(e)}'))
|
||
job_queue[job_id]['progress'] = f'Failed to process ZIP {filename}: {str(e)}'
|
||
print(f"ZIP processing error for {filename}: {str(e)}")
|
||
continue
|
||
else:
|
||
# Single PDF
|
||
try:
|
||
job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
|
||
save_jobs() # Persist current file progress
|
||
# Skip MacOS resource fork files (._*)
|
||
if filename.startswith('._'):
|
||
print(f"DEBUG: Skipping MacOS resource fork file: {filename}")
|
||
continue
|
||
pdf_info = extract_names_from_pdf(filepath)
|
||
if 'error' in pdf_info:
|
||
# Move skipped file to Skipped/
|
||
skipped_pdf_path = os.path.join(skipped_dir, filename)
|
||
try:
|
||
os.rename(filepath, skipped_pdf_path)
|
||
except Exception as move_err:
|
||
print(f"WARNING: Could not move skipped PDF {filename}: {move_err}")
|
||
# Log skip reason and raw text if available
|
||
with open(skipped_log_path, 'a', encoding='utf-8') as slog:
|
||
slog.write(f"File: {filename}\nReason: {pdf_info['error']}\n")
|
||
if 'raw_text' in pdf_info:
|
||
slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
|
||
slog.write("-"*50 + "\n")
|
||
processed_files.append(('skipped', filename, pdf_info['error'], pdf_info.get('raw_text', '')))
|
||
job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
|
||
else:
|
||
job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
|
||
# Create folder structure: Episode/School/Grade/Gender/
|
||
school_dir = os.path.join(episode_dir, sanitize_segment(pdf_info['school']) or 'Unknown_School')
|
||
grade_dir = os.path.join(school_dir, sanitize_segment(pdf_info['grade']) or 'Unknown_Grade')
|
||
gender_dir = os.path.join(grade_dir, sanitize_segment(pdf_info['gender']) or 'Unknown_Gender')
|
||
os.makedirs(gender_dir, exist_ok=True)
|
||
|
||
# Move file to appropriate folder with new name
|
||
safe_filename = secure_filename(pdf_info['filename']) or pdf_info['filename']
|
||
new_filepath = os.path.join(gender_dir, safe_filename)
|
||
counter = 1
|
||
while os.path.exists(new_filepath):
|
||
base, ext = os.path.splitext(pdf_info['filename'])
|
||
new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
|
||
counter += 1
|
||
os.rename(filepath, new_filepath)
|
||
processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
|
||
processed_count += 1
|
||
job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
|
||
# Batch saves - only every 5 files
|
||
if processed_count % 5 == 0:
|
||
save_jobs()
|
||
except Exception as e:
|
||
processed_files.append(('skipped', filename, f'PDF processing error: {str(e)}'))
|
||
job_queue[job_id]['progress'] = f'Failed to process PDF {filename}: {str(e)}'
|
||
print(f"PDF processing error for {filename}: {str(e)}")
|
||
continue
|
||
|
||
job_queue[job_id]['progress'] = 'Creating ZIP archive...'
|
||
# Create ZIP of the entire Episode directory structure
|
||
# Always create ZIP, including Skipped/ folder if present
|
||
try:
|
||
zip_output = os.path.join(app.config['ARCHIVE_FOLDER'], f'renamed_{job_id}.zip')
|
||
print(f"DEBUG: Creating ZIP at {zip_output}")
|
||
with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
|
||
for root, dirs, files_in_dir in os.walk(episode_dir):
|
||
for file in files_in_dir:
|
||
if file.startswith('.') or file.startswith('._'):
|
||
continue # Ignore dotfiles and MacOS resource forks
|
||
file_path = os.path.join(root, file)
|
||
arcname = os.path.relpath(file_path, episode_dir)
|
||
print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
|
||
zip_out.write(file_path, arcname)
|
||
job_queue[job_id]['zip_path'] = zip_output
|
||
job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
|
||
job_queue[job_id]['progress'] = f'Created ZIP archive with {len([p for p in processed_files if p[0] == 'success'])} organized files and {len([p for p in processed_files if p[0] == 'skipped'])} skipped files'
|
||
save_jobs() # Persist completion progress
|
||
print(f"DEBUG: ZIP creation completed successfully")
|
||
# Clean up Episode directory after successful ZIP creation
|
||
try:
|
||
import shutil
|
||
shutil.rmtree(episode_dir)
|
||
print(f"DEBUG: Cleaned up Episode directory: {episode_dir}")
|
||
except Exception as cleanup_error:
|
||
print(f"WARNING: Failed to clean up Episode directory: {cleanup_error}")
|
||
# Clean up original uploaded files
|
||
for file_info in files:
|
||
try:
|
||
if os.path.exists(file_info['path']):
|
||
os.remove(file_info['path'])
|
||
print(f"DEBUG: Removed uploaded file: {file_info['path']}")
|
||
except Exception as file_cleanup_error:
|
||
print(f"WARNING: Failed to remove uploaded file: {file_cleanup_error}")
|
||
except Exception as e:
|
||
job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
|
||
job_queue[job_id]['status'] = 'error'
|
||
job_failed = True
|
||
save_jobs() # Persist error status
|
||
print(f"DEBUG: ZIP creation failed: {str(e)}")
|
||
|
||
with job_lock:
|
||
if not job_failed:
|
||
job_queue[job_id]['status'] = 'completed'
|
||
job_queue[job_id]['processed_files'] = processed_files
|
||
# Store skipped as (filename, reason, raw_text)
|
||
job_queue[job_id]['skipped'] = [(p[1], p[2], p[3]) for p in processed_files if p[0] == 'skipped']
|
||
save_jobs() # Persist completed job
|
||
print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")
|
||
|
||
except Exception as e:
|
||
with job_lock:
|
||
job_queue[job_id]['status'] = 'error'
|
||
job_queue[job_id]['progress'] = f'Job failed with error: {str(e)}'
|
||
save_jobs() # Persist error status
|
||
print(f"Job {job_id} failed: {str(e)}") # Debug logging
|
||
finally:
|
||
# Clean up temporary directories
|
||
for temp_dir in temp_extract_dirs:
|
||
if os.path.exists(temp_dir):
|
||
try:
|
||
import shutil
|
||
shutil.rmtree(temp_dir)
|
||
print(f"DEBUG: Cleaned up temporary directory: {temp_dir}")
|
||
except Exception as cleanup_error:
|
||
print(f"ERROR: Failed to clean up {temp_dir}: {cleanup_error}")
|
||
|
||
@app.route('/login')
|
||
def login():
|
||
return render_template_string('''
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>NextGenPDF - Login</title>
|
||
<style>
|
||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||
body {
|
||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
min-height: 100vh;
|
||
display: flex;
|
||
justify-content: center;
|
||
align-items: center;
|
||
padding: 20px;
|
||
}
|
||
.login-wrapper {
|
||
background: white;
|
||
border-radius: 15px;
|
||
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
|
||
overflow: hidden;
|
||
max-width: 900px;
|
||
width: 100%;
|
||
display: grid;
|
||
grid-template-columns: 1fr 1fr;
|
||
}
|
||
.login-left {
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
padding: 60px 40px;
|
||
display: flex;
|
||
flex-direction: column;
|
||
justify-content: center;
|
||
align-items: center;
|
||
color: white;
|
||
text-align: center;
|
||
}
|
||
.login-left img { width: 180px; margin-bottom: 30px; filter: brightness(0) invert(1); }
|
||
.login-left h1 { font-size: 32px; margin-bottom: 15px; font-weight: 600; }
|
||
.login-left p { font-size: 16px; line-height: 1.6; opacity: 0.9; }
|
||
.login-right {
|
||
padding: 60px 40px;
|
||
display: flex;
|
||
flex-direction: column;
|
||
justify-content: center;
|
||
}
|
||
.login-right h2 {
|
||
font-size: 24px;
|
||
color: #333;
|
||
margin-bottom: 40px;
|
||
text-align: center;
|
||
font-weight: 600;
|
||
}
|
||
.login-options { display: flex; flex-direction: column; gap: 20px; }
|
||
.login-option { }
|
||
.login-option h3 {
|
||
font-size: 14px;
|
||
color: #666;
|
||
text-transform: uppercase;
|
||
letter-spacing: 1px;
|
||
margin-bottom: 15px;
|
||
font-weight: 600;
|
||
}
|
||
.login-btn {
|
||
width: 100%;
|
||
padding: 14px 24px;
|
||
font-size: 16px;
|
||
border: none;
|
||
border-radius: 8px;
|
||
cursor: pointer;
|
||
font-weight: 600;
|
||
transition: all 0.3s ease;
|
||
text-decoration: none;
|
||
display: block;
|
||
text-align: center;
|
||
}
|
||
.btn-microsoft {
|
||
background: #0078d4;
|
||
color: white;
|
||
}
|
||
.btn-microsoft:hover {
|
||
background: #106ebe;
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 10px 25px rgba(0, 120, 212, 0.3);
|
||
}
|
||
form { width: 100%; }
|
||
.form-group { margin-bottom: 15px; }
|
||
.form-group input {
|
||
width: 100%;
|
||
padding: 12px 16px;
|
||
border: 1px solid #ddd;
|
||
border-radius: 8px;
|
||
font-size: 15px;
|
||
transition: all 0.3s ease;
|
||
font-family: inherit;
|
||
}
|
||
.form-group input:focus {
|
||
outline: none;
|
||
border-color: #667eea;
|
||
box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
|
||
}
|
||
.btn-admin {
|
||
width: 100%;
|
||
padding: 12px 24px;
|
||
background: #28a745;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: 600;
|
||
cursor: pointer;
|
||
transition: all 0.3s ease;
|
||
}
|
||
.btn-admin:hover {
|
||
background: #218838;
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 10px 25px rgba(40, 167, 69, 0.3);
|
||
}
|
||
.divider {
|
||
text-align: center;
|
||
margin: 25px 0;
|
||
position: relative;
|
||
color: #999;
|
||
}
|
||
.divider::before {
|
||
content: '';
|
||
position: absolute;
|
||
top: 50%;
|
||
left: 0;
|
||
right: 0;
|
||
height: 1px;
|
||
background: #ddd;
|
||
}
|
||
.divider span {
|
||
background: white;
|
||
padding: 0 10px;
|
||
position: relative;
|
||
font-size: 13px;
|
||
}
|
||
@media (max-width: 768px) {
|
||
.login-wrapper { grid-template-columns: 1fr; }
|
||
.login-left { padding: 40px 30px; }
|
||
.login-right { padding: 40px 30px; }
|
||
.login-left h1 { font-size: 24px; }
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="login-wrapper">
|
||
<div class="login-left">
|
||
<img src="/static/trbc-logo-full.png" alt="Logo">
|
||
<h1>NextGenPDF</h1>
|
||
<p>Automated PDF Organization & Renaming System</p>
|
||
</div>
|
||
<div class="login-right">
|
||
<h2>Welcome Back</h2>
|
||
<div class="login-options">
|
||
<div class="login-option">
|
||
<h3>Microsoft Entra ID</h3>
|
||
<a href="/entra-login" class="login-btn btn-microsoft">Sign in with Microsoft</a>
|
||
</div>
|
||
<div class="divider"><span>or</span></div>
|
||
<div class="login-option">
|
||
<h3>Administrator Login</h3>
|
||
<form method="post" action="/admin-login">
|
||
<div class="form-group">
|
||
<input type="text" name="username" placeholder="Username" required>
|
||
</div>
|
||
<div class="form-group">
|
||
<input type="password" name="password" placeholder="Password" required>
|
||
</div>
|
||
<button type="submit" class="btn-admin">Sign In</button>
|
||
</form>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
''')
|
||
|
||
@app.route('/entra-login')
|
||
def entra_login():
|
||
# Check if Microsoft OAuth provider is available
|
||
if microsoft is None:
|
||
flash('Microsoft Entra ID authentication is not configured.', 'error')
|
||
return redirect(url_for('login'))
|
||
|
||
# Check if Microsoft Entra ID is configured
|
||
if TENANT_ID in ['your-tenant-id', 'YOUR_MICROSOFT_ENTRA_TENANT_ID'] or \
|
||
CLIENT_ID in ['your-client-id', 'YOUR_MICROSOFT_ENTRA_CLIENT_ID'] or \
|
||
CLIENT_SECRET in ['your-client-secret', 'YOUR_MICROSOFT_ENTRA_CLIENT_SECRET']:
|
||
flash('Microsoft Entra ID is not properly configured. Please check your environment variables.', 'error')
|
||
return redirect(url_for('login'))
|
||
|
||
try:
|
||
redirect_uri = url_for('auth', _external=True)
|
||
print(f"DEBUG: Initiating OAuth with redirect_uri: {redirect_uri}")
|
||
return microsoft.authorize_redirect(redirect_uri)
|
||
except Exception as e:
|
||
print(f"ERROR: Failed to initiate OAuth: {e}")
|
||
import traceback
|
||
print(f"Traceback: {traceback.format_exc()}")
|
||
flash('Failed to initiate Microsoft authentication. Please try again or use admin login.', 'error')
|
||
return redirect(url_for('login'))
|
||
|
||
@app.route('/admin-login', methods=['POST'])
|
||
def admin_login():
|
||
username = request.form.get('username')
|
||
password = request.form.get('password')
|
||
if username == ADMIN_USERNAME and password == ADMIN_PASSWORD:
|
||
session['admin'] = True
|
||
return redirect(url_for('upload_files'))
|
||
else:
|
||
return render_template_string('''
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>Login Failed</title>
|
||
<style>
|
||
body { font-family: Arial, sans-serif; background-color: #f4f4f4; text-align: center; padding: 50px; }
|
||
.error { color: #dc3545; background-color: #f8d7da; padding: 20px; border-radius: 5px; border: 1px solid #f5c6cb; }
|
||
a { color: #007bff; text-decoration: none; }
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="error">
|
||
<h1>Login Failed</h1>
|
||
<p>Invalid username or password.</p>
|
||
<a href="/login">Try Again</a>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
''')
|
||
|
||
@app.route('/auth')
|
||
def auth():
|
||
try:
|
||
print("DEBUG: Starting OAuth callback processing")
|
||
token = microsoft.authorize_access_token()
|
||
print(f"DEBUG: Got token: {token}")
|
||
|
||
# Get user info from the token's userinfo (already parsed by Authlib)
|
||
user_info = token.get('userinfo')
|
||
|
||
if not user_info:
|
||
print("ERROR: No userinfo in token")
|
||
flash('Failed to get user information from Microsoft.', 'error')
|
||
return redirect(url_for('login'))
|
||
|
||
print(f"DEBUG: Parsed user info: {user_info}")
|
||
|
||
# Store user information in session
|
||
session['user'] = {
|
||
'name': user_info.get('name', 'Unknown'),
|
||
'email': user_info.get('email') or user_info.get('preferred_username', 'unknown@example.com'),
|
||
'oid': user_info.get('oid'),
|
||
}
|
||
session.permanent = True
|
||
|
||
print(f"DEBUG: User logged in successfully: {session['user']}")
|
||
return redirect(url_for('upload_files'))
|
||
|
||
except Exception as e:
|
||
print(f"OAuth error: {e}") # Log the error for debugging
|
||
import traceback
|
||
print(f"Full traceback: {traceback.format_exc()}")
|
||
flash('Authentication failed. Please try again.', 'error')
|
||
return redirect(url_for('login'))
|
||
|
||
@app.route('/logout')
|
||
@login_required
|
||
def logout():
|
||
session.pop('user', None)
|
||
session.pop('admin', None)
|
||
return redirect(url_for('login'))
|
||
|
||
@app.route('/', methods=['GET', 'POST'])
|
||
@login_required
|
||
def upload_files():
|
||
if request.method == 'POST':
|
||
files = request.files.getlist('files')
|
||
if not files or all(not f.filename for f in files):
|
||
flash('No files selected.', 'error')
|
||
return redirect(url_for('upload_files'))
|
||
|
||
# Create job ID using timestamp format: YYYY-MM-DD_HH-MM-SS
|
||
job_id = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
|
||
job_queue[job_id] = {
|
||
'status': 'queued',
|
||
'files': [],
|
||
'zip_path': None,
|
||
'zip_name': None,
|
||
'processed_files': [],
|
||
'skipped': [],
|
||
'created_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||
'job_id': job_id # Store the job ID for reference
|
||
}
|
||
|
||
file_list = []
|
||
for file in files:
|
||
if file and not file.filename.startswith('.') and allowed_file(file.filename):
|
||
filename = secure_filename(file.filename)
|
||
filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
|
||
file.save(filepath)
|
||
file_list.append({'name': filename, 'path': filepath})
|
||
print(f"Uploaded file: {filename} -> {filepath}") # Debug logging
|
||
|
||
if not file_list:
|
||
flash('No valid files uploaded. Only PDF and ZIP files are allowed.', 'error')
|
||
job_queue.pop(job_id, None) # Remove empty job
|
||
return redirect(url_for('upload_files'))
|
||
|
||
job_queue[job_id]['files'] = file_list
|
||
print(f"Created job {job_id} with {len(file_list)} files") # Debug logging
|
||
|
||
# Save job to persistent storage
|
||
save_jobs()
|
||
|
||
# Start processing in background
|
||
def process_with_context():
|
||
with app.app_context():
|
||
process_job(job_id, file_list)
|
||
|
||
thread = threading.Thread(target=process_with_context)
|
||
thread.daemon = True
|
||
thread.start()
|
||
|
||
flash(f'Job created successfully. Processing {len(file_list)} files...', 'success')
|
||
return redirect(url_for('jobs'))
|
||
|
||
return render_template_string('''
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>NextGenPDF - Upload Files</title>
|
||
<style>
|
||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||
body {
|
||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||
background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%);
|
||
min-height: 100vh;
|
||
padding: 30px 20px;
|
||
}
|
||
.container { max-width: 1200px; margin: 0 auto; }
|
||
.header {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
margin-bottom: 40px;
|
||
}
|
||
.header-info { display: flex; align-items: center; gap: 20px; }
|
||
.logo img { height: 50px; }
|
||
.header-text h1 {
|
||
font-size: 28px;
|
||
color: #333;
|
||
margin: 0;
|
||
font-weight: 600;
|
||
}
|
||
.header-text p {
|
||
font-size: 14px;
|
||
color: #666;
|
||
margin: 5px 0 0 0;
|
||
}
|
||
.header-actions { display: flex; gap: 10px; }
|
||
.nav-link {
|
||
display: inline-flex;
|
||
align-items: center;
|
||
gap: 8px;
|
||
padding: 12px 20px;
|
||
background: white;
|
||
color: #667eea;
|
||
text-decoration: none;
|
||
border-radius: 8px;
|
||
font-weight: 600;
|
||
transition: all 0.3s ease;
|
||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||
}
|
||
.nav-link:hover {
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 5px 15px rgba(0, 0, 0, 0.15);
|
||
}
|
||
.logout-btn {
|
||
background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
|
||
color: white !important;
|
||
}
|
||
.main-content { display: grid; grid-template-columns: 1fr 1fr; gap: 30px; }
|
||
.upload-section {
|
||
background: white;
|
||
padding: 40px;
|
||
border-radius: 15px;
|
||
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.1);
|
||
}
|
||
.upload-section h2 {
|
||
font-size: 22px;
|
||
color: #333;
|
||
margin-bottom: 30px;
|
||
font-weight: 600;
|
||
}
|
||
.upload-area {
|
||
border: 2px dashed #667eea;
|
||
border-radius: 12px;
|
||
padding: 60px 20px;
|
||
text-align: center;
|
||
background: #f8f9ff;
|
||
cursor: pointer;
|
||
transition: all 0.3s ease;
|
||
position: relative;
|
||
}
|
||
.upload-area:hover {
|
||
border-color: #764ba2;
|
||
background: #f3f4ff;
|
||
}
|
||
.upload-area.active {
|
||
border-color: #28a745;
|
||
background: #f0f9f4;
|
||
}
|
||
.upload-icon {
|
||
font-size: 48px;
|
||
margin-bottom: 15px;
|
||
color: #667eea;
|
||
}
|
||
.upload-area.active .upload-icon { color: #28a745; }
|
||
.upload-text {
|
||
color: #333;
|
||
font-size: 16px;
|
||
font-weight: 500;
|
||
}
|
||
.upload-text p {
|
||
margin: 5px 0;
|
||
color: #666;
|
||
font-size: 14px;
|
||
}
|
||
.upload-input {
|
||
position: absolute;
|
||
top: 0;
|
||
left: 0;
|
||
width: 100%;
|
||
height: 100%;
|
||
opacity: 0;
|
||
cursor: pointer;
|
||
}
|
||
.submit-btn {
|
||
width: 100%;
|
||
padding: 14px 24px;
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: 600;
|
||
cursor: pointer;
|
||
margin-top: 20px;
|
||
transition: all 0.3s ease;
|
||
}
|
||
.submit-btn:hover {
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 10px 25px rgba(102, 126, 234, 0.3);
|
||
}
|
||
.submit-btn:disabled {
|
||
opacity: 0.5;
|
||
cursor: not-allowed;
|
||
transform: none;
|
||
}
|
||
.file-list {
|
||
margin-top: 20px;
|
||
max-height: 300px;
|
||
overflow-y: auto;
|
||
}
|
||
.file-item {
|
||
background: #f8f9fa;
|
||
padding: 10px 15px;
|
||
border-radius: 6px;
|
||
margin: 8px 0;
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
font-size: 14px;
|
||
}
|
||
.file-item .remove-btn {
|
||
background: #dc3545;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 4px;
|
||
padding: 4px 8px;
|
||
cursor: pointer;
|
||
font-size: 12px;
|
||
}
|
||
.info-section {
|
||
background: white;
|
||
padding: 40px;
|
||
border-radius: 15px;
|
||
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.1);
|
||
}
|
||
.info-section h2 {
|
||
font-size: 22px;
|
||
color: #333;
|
||
margin-bottom: 20px;
|
||
font-weight: 600;
|
||
}
|
||
.info-box {
|
||
background: linear-gradient(135deg, #667eea15 0%, #764ba215 100%);
|
||
border-left: 4px solid #667eea;
|
||
padding: 20px;
|
||
border-radius: 8px;
|
||
margin-bottom: 20px;
|
||
}
|
||
.info-box h3 {
|
||
color: #667eea;
|
||
font-size: 16px;
|
||
margin-bottom: 12px;
|
||
font-weight: 600;
|
||
}
|
||
.info-box p, .info-box ul {
|
||
color: #555;
|
||
font-size: 14px;
|
||
line-height: 1.6;
|
||
margin: 0;
|
||
}
|
||
.info-box ul {
|
||
padding-left: 20px;
|
||
margin: 10px 0;
|
||
}
|
||
.info-box li {
|
||
margin: 8px 0;
|
||
color: #555;
|
||
}
|
||
.feature-item {
|
||
display: flex;
|
||
align-items: flex-start;
|
||
gap: 12px;
|
||
margin-bottom: 12px;
|
||
}
|
||
.feature-icon {
|
||
color: #28a745;
|
||
font-weight: bold;
|
||
font-size: 18px;
|
||
flex-shrink: 0;
|
||
}
|
||
.feature-text {
|
||
color: #555;
|
||
font-size: 14px;
|
||
}
|
||
.quick-links {
|
||
display: flex;
|
||
gap: 10px;
|
||
margin-top: 20px;
|
||
}
|
||
.quick-link {
|
||
flex: 1;
|
||
padding: 15px;
|
||
background: #f8f9fa;
|
||
border-radius: 8px;
|
||
text-align: center;
|
||
text-decoration: none;
|
||
color: #667eea;
|
||
font-weight: 600;
|
||
transition: all 0.3s ease;
|
||
border: 1px solid #e9ecef;
|
||
}
|
||
.quick-link:hover {
|
||
background: #667eea;
|
||
color: white;
|
||
}
|
||
.alert {
|
||
padding: 15px 20px;
|
||
border-radius: 8px;
|
||
margin-bottom: 20px;
|
||
font-size: 14px;
|
||
}
|
||
.alert.success {
|
||
background: #d4edda;
|
||
color: #155724;
|
||
border: 1px solid #c3e6cb;
|
||
}
|
||
.alert.error {
|
||
background: #f8d7da;
|
||
color: #721c24;
|
||
border: 1px solid #f5c6cb;
|
||
}
|
||
@media (max-width: 768px) {
|
||
.main-content { grid-template-columns: 1fr; }
|
||
.header { flex-direction: column; text-align: center; gap: 20px; }
|
||
.header-actions { justify-content: center; flex-wrap: wrap; }
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="container">
|
||
<div class="header">
|
||
<div class="header-info">
|
||
<div class="logo"><img src="/static/trbc-logo-full.png" alt="Logo"></div>
|
||
<div class="header-text">
|
||
<h1>NextGenPDF</h1>
|
||
<p>Automated PDF Organization & Renaming</p>
|
||
</div>
|
||
</div>
|
||
<div class="header-actions">
|
||
<a href="/jobs" class="nav-link">📊 View Jobs</a>
|
||
<a href="/logout" class="nav-link logout-btn">🚪 Logout</a>
|
||
</div>
|
||
</div>
|
||
|
||
{% with messages = get_flashed_messages(with_categories=true) %}
|
||
{% if messages %}
|
||
{% for category, message in messages %}
|
||
<div class="alert {{ category }}">{{ message }}</div>
|
||
{% endfor %}
|
||
{% endif %}
|
||
{% endwith %}
|
||
|
||
<div class="main-content">
|
||
<div class="upload-section">
|
||
<h2>📁 Upload Files</h2>
|
||
<form method="post" enctype="multipart/form-data" id="upload-form">
|
||
<div class="upload-area" id="upload-area">
|
||
<div class="upload-icon">⬆️</div>
|
||
<div class="upload-text">
|
||
<p style="font-weight: 600;">Drag & drop your files here</p>
|
||
<p>or click to select PDF and ZIP files</p>
|
||
</div>
|
||
<input type="file" id="file-input" name="files" multiple accept=".pdf,.zip" class="upload-input">
|
||
</div>
|
||
<div class="file-list" id="file-list"></div>
|
||
<button type="submit" class="submit-btn" id="submit-btn">Upload & Process Files</button>
|
||
</form>
|
||
</div>
|
||
|
||
<div class="info-section">
|
||
<h2>ℹ️ How It Works</h2>
|
||
<div class="info-box">
|
||
<h3>What is NextGenPDF?</h3>
|
||
<p>NextGenPDF automatically extracts student information from PDF registration forms and organizes them into a structured folder system.</p>
|
||
</div>
|
||
<div class="info-box">
|
||
<h3>📋 Processing Steps</h3>
|
||
<div class="feature-item">
|
||
<div class="feature-icon">1</div>
|
||
<div class="feature-text">Upload individual PDFs or ZIP files</div>
|
||
</div>
|
||
<div class="feature-item">
|
||
<div class="feature-icon">2</div>
|
||
<div class="feature-text">System extracts name, school, grade, and gender</div>
|
||
</div>
|
||
<div class="feature-item">
|
||
<div class="feature-icon">3</div>
|
||
<div class="feature-text">Files renamed to: <strong>LastName_FirstName_Grade.pdf</strong></div>
|
||
</div>
|
||
<div class="feature-item">
|
||
<div class="feature-icon">4</div>
|
||
<div class="feature-text">Organized into: <strong>Episode/School/Grade/Gender/</strong></div>
|
||
</div>
|
||
<div class="feature-item">
|
||
<div class="feature-icon">5</div>
|
||
<div class="feature-text">Download organized ZIP archive</div>
|
||
</div>
|
||
</div>
|
||
<div class="quick-links">
|
||
<a href="/jobs" class="quick-link">View Jobs →</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
const uploadArea = document.getElementById('upload-area');
|
||
const fileInput = document.getElementById('file-input');
|
||
const fileList = document.getElementById('file-list');
|
||
const uploadForm = document.getElementById('upload-form');
|
||
let selectedFiles = [];
|
||
|
||
// Drag and drop handlers
|
||
uploadArea.addEventListener('dragover', (e) => {
|
||
e.preventDefault();
|
||
uploadArea.classList.add('active');
|
||
});
|
||
|
||
uploadArea.addEventListener('dragleave', () => {
|
||
uploadArea.classList.remove('active');
|
||
});
|
||
|
||
uploadArea.addEventListener('drop', (e) => {
|
||
e.preventDefault();
|
||
uploadArea.classList.remove('active');
|
||
const files = e.dataTransfer.files;
|
||
fileInput.files = files;
|
||
updateFileList(files);
|
||
});
|
||
|
||
uploadArea.addEventListener('click', () => fileInput.click());
|
||
|
||
fileInput.addEventListener('change', (e) => {
|
||
updateFileList(e.target.files);
|
||
});
|
||
|
||
function updateFileList(files) {
|
||
selectedFiles = Array.from(files);
|
||
fileList.innerHTML = '';
|
||
|
||
if (selectedFiles.length === 0) {
|
||
document.getElementById('submit-btn').disabled = true;
|
||
return;
|
||
}
|
||
|
||
selectedFiles.forEach((file, index) => {
|
||
const item = document.createElement('div');
|
||
item.className = 'file-item';
|
||
const size = (file.size / 1024 / 1024).toFixed(2);
|
||
item.innerHTML = `
|
||
<span>📄 ${file.name} (${size} MB)</span>
|
||
<button type="button" class="remove-btn" onclick="removeFile(${index})">Remove</button>
|
||
`;
|
||
fileList.appendChild(item);
|
||
});
|
||
|
||
document.getElementById('submit-btn').disabled = false;
|
||
}
|
||
|
||
function removeFile(index) {
|
||
selectedFiles.splice(index, 1);
|
||
const dt = new DataTransfer();
|
||
selectedFiles.forEach(file => dt.items.add(file));
|
||
fileInput.files = dt.files;
|
||
updateFileList(dt.files);
|
||
}
|
||
|
||
uploadForm.addEventListener('submit', (e) => {
|
||
if (selectedFiles.length === 0) {
|
||
e.preventDefault();
|
||
alert('Please select files to upload');
|
||
}
|
||
});
|
||
</script>
|
||
</body>
|
||
</html>
|
||
''')
|
||
|
||
@app.route('/jobs')
|
||
@login_required
|
||
def jobs():
|
||
return render_template_string('''
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>NextGenPDF - Jobs Dashboard</title>
|
||
<style>
|
||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||
body {
|
||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||
background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%);
|
||
min-height: 100vh;
|
||
padding: 30px 20px;
|
||
}
|
||
.container { max-width: 1200px; margin: 0 auto; }
|
||
.header {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
margin-bottom: 40px;
|
||
}
|
||
.header-info { display: flex; align-items: center; gap: 20px; }
|
||
.logo img { height: 50px; }
|
||
.header-text h1 {
|
||
font-size: 28px;
|
||
color: #333;
|
||
margin: 0;
|
||
font-weight: 600;
|
||
}
|
||
.header-text p {
|
||
font-size: 14px;
|
||
color: #666;
|
||
margin: 5px 0 0 0;
|
||
}
|
||
.header-actions { display: flex; gap: 10px; }
|
||
.nav-link {
|
||
display: inline-flex;
|
||
align-items: center;
|
||
gap: 8px;
|
||
padding: 12px 20px;
|
||
background: white;
|
||
color: #667eea;
|
||
text-decoration: none;
|
||
border-radius: 8px;
|
||
font-weight: 600;
|
||
transition: all 0.3s ease;
|
||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||
}
|
||
.nav-link:hover {
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 5px 15px rgba(0, 0, 0, 0.15);
|
||
}
|
||
.logout-btn {
|
||
background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
|
||
color: white !important;
|
||
}
|
||
.jobs-container { }
|
||
.job-card {
|
||
background: white;
|
||
border-radius: 12px;
|
||
box-shadow: 0 5px 20px rgba(0, 0, 0, 0.1);
|
||
margin-bottom: 25px;
|
||
overflow: hidden;
|
||
transition: all 0.3s ease;
|
||
}
|
||
.job-card:hover {
|
||
box-shadow: 0 10px 35px rgba(0, 0, 0, 0.15);
|
||
transform: translateY(-2px);
|
||
}
|
||
.job-header {
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
padding: 20px;
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
}
|
||
.job-title { font-size: 18px; font-weight: 600; }
|
||
.job-meta { font-size: 12px; opacity: 0.9; }
|
||
.status-badge {
|
||
padding: 8px 16px;
|
||
border-radius: 20px;
|
||
font-size: 12px;
|
||
font-weight: 600;
|
||
text-transform: uppercase;
|
||
letter-spacing: 0.5px;
|
||
}
|
||
.status-queued { background: rgba(255, 193, 7, 0.2); color: #f57f17; }
|
||
.status-processing { background: rgba(0, 150, 136, 0.2); color: #00897b; }
|
||
.status-completed { background: rgba(76, 175, 80, 0.2); color: #558b2f; }
|
||
.status-error { background: rgba(244, 67, 54, 0.2); color: #c62828; }
|
||
.job-body { padding: 25px; }
|
||
.job-info { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin-bottom: 20px; }
|
||
.info-item { }
|
||
.info-label {
|
||
font-size: 12px;
|
||
text-transform: uppercase;
|
||
letter-spacing: 0.5px;
|
||
color: #999;
|
||
font-weight: 600;
|
||
margin-bottom: 5px;
|
||
}
|
||
.info-value {
|
||
font-size: 16px;
|
||
color: #333;
|
||
font-weight: 500;
|
||
}
|
||
.progress-bar {
|
||
width: 100%;
|
||
height: 8px;
|
||
background: #f0f0f0;
|
||
border-radius: 10px;
|
||
overflow: hidden;
|
||
margin-top: 10px;
|
||
}
|
||
.progress-fill {
|
||
height: 100%;
|
||
background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);
|
||
width: 0%;
|
||
transition: width 0.5s ease;
|
||
}
|
||
.progress-text {
|
||
font-size: 12px;
|
||
color: #667eea;
|
||
font-style: italic;
|
||
margin-top: 8px;
|
||
}
|
||
.zip-contents {
|
||
background: #f8f9fa;
|
||
border-radius: 8px;
|
||
padding: 15px;
|
||
margin: 15px 0;
|
||
}
|
||
.zip-toggle {
|
||
cursor: pointer;
|
||
font-weight: 600;
|
||
color: #667eea;
|
||
display: flex;
|
||
align-items: center;
|
||
justify-content: space-between;
|
||
padding: 10px 0;
|
||
user-select: none;
|
||
}
|
||
.zip-toggle:hover { color: #764ba2; }
|
||
.zip-details {
|
||
display: none;
|
||
margin-top: 15px;
|
||
max-height: 400px;
|
||
overflow-y: auto;
|
||
}
|
||
.file-list { list-style: none; }
|
||
.file-list li {
|
||
padding: 8px 12px;
|
||
background: white;
|
||
margin: 5px 0;
|
||
border-radius: 4px;
|
||
font-size: 13px;
|
||
color: #555;
|
||
border-left: 3px solid #667eea;
|
||
}
|
||
.skipped-section {
|
||
background: #fff3cd;
|
||
border-left: 4px solid #ffc107;
|
||
border-radius: 8px;
|
||
padding: 15px;
|
||
margin: 15px 0;
|
||
}
|
||
.skipped-toggle {
|
||
cursor: pointer;
|
||
font-weight: 600;
|
||
color: #856404;
|
||
display: flex;
|
||
align-items: center;
|
||
justify-content: space-between;
|
||
user-select: none;
|
||
}
|
||
.skipped-toggle:hover { color: #533f03; }
|
||
.skipped-details { display: none; margin-top: 15px; }
|
||
.action-buttons {
|
||
display: flex;
|
||
gap: 10px;
|
||
flex-wrap: wrap;
|
||
margin-top: 20px;
|
||
}
|
||
.btn {
|
||
padding: 10px 20px;
|
||
border: none;
|
||
border-radius: 6px;
|
||
font-size: 14px;
|
||
font-weight: 600;
|
||
cursor: pointer;
|
||
transition: all 0.3s ease;
|
||
text-decoration: none;
|
||
display: inline-flex;
|
||
align-items: center;
|
||
gap: 8px;
|
||
}
|
||
.btn-download {
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
}
|
||
.btn-download:hover {
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 5px 15px rgba(102, 126, 234, 0.3);
|
||
}
|
||
.btn-delete {
|
||
background: #dc3545;
|
||
color: white;
|
||
}
|
||
.btn-delete:hover {
|
||
background: #c82333;
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 5px 15px rgba(220, 53, 69, 0.3);
|
||
}
|
||
.empty-state {
|
||
text-align: center;
|
||
padding: 60px 20px;
|
||
background: white;
|
||
border-radius: 12px;
|
||
box-shadow: 0 5px 20px rgba(0, 0, 0, 0.1);
|
||
}
|
||
.empty-state-icon { font-size: 64px; margin-bottom: 20px; }
|
||
.empty-state-text {
|
||
font-size: 18px;
|
||
color: #666;
|
||
margin-bottom: 20px;
|
||
}
|
||
.empty-state-link {
|
||
display: inline-block;
|
||
padding: 12px 24px;
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
text-decoration: none;
|
||
border-radius: 6px;
|
||
font-weight: 600;
|
||
transition: all 0.3s ease;
|
||
}
|
||
.empty-state-link:hover {
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 5px 15px rgba(102, 126, 234, 0.3);
|
||
}
|
||
.alert {
|
||
padding: 15px 20px;
|
||
border-radius: 8px;
|
||
margin-bottom: 20px;
|
||
font-size: 14px;
|
||
}
|
||
.alert.success {
|
||
background: #d4edda;
|
||
color: #155724;
|
||
border: 1px solid #c3e6cb;
|
||
}
|
||
.alert.error {
|
||
background: #f8d7da;
|
||
color: #721c24;
|
||
border: 1px solid #f5c6cb;
|
||
}
|
||
@media (max-width: 768px) {
|
||
.header { flex-direction: column; gap: 20px; }
|
||
.header-actions { justify-content: center; flex-wrap: wrap; }
|
||
.job-header { flex-direction: column; align-items: flex-start; gap: 10px; }
|
||
.job-info { grid-template-columns: 1fr; }
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="container">
|
||
<div class="header">
|
||
<div class="header-info">
|
||
<div class="logo"><img src="/static/trbc-logo-full.png" alt="Logo"></div>
|
||
<div class="header-text">
|
||
<h1>Jobs Dashboard</h1>
|
||
<p>Monitor your PDF processing jobs</p>
|
||
</div>
|
||
</div>
|
||
<div class="header-actions">
|
||
<a href="/" class="nav-link">📤 Upload Files</a>
|
||
<a href="/logout" class="nav-link logout-btn">🚪 Logout</a>
|
||
</div>
|
||
</div>
|
||
|
||
{% with messages = get_flashed_messages(with_categories=true) %}
|
||
{% if messages %}
|
||
{% for category, message in messages %}
|
||
<div class="alert {{ category }}">{{ message }}</div>
|
||
{% endfor %}
|
||
{% endif %}
|
||
{% endwith %}
|
||
|
||
<div class="jobs-container">
|
||
{% if jobs %}
|
||
{% for job_id, job in jobs.items() %}
|
||
<div class="job-card">
|
||
<div class="job-header">
|
||
<div>
|
||
<div class="job-title">📋 Job {{ job_id[:8] }}</div>
|
||
<div class="job-meta">Created: {{ job.created_at }}</div>
|
||
</div>
|
||
<div class="status-badge status-{{ job.status }}">{{ job.status }}</div>
|
||
</div>
|
||
<div class="job-body">
|
||
<div class="job-info">
|
||
<div class="info-item">
|
||
<div class="info-label">Total Files</div>
|
||
<div class="info-value">{{ job.files|length }}</div>
|
||
</div>
|
||
<div class="info-item">
|
||
<div class="info-label">Status</div>
|
||
<div class="info-value" style="color: {% if job.status == 'completed' %}#558b2f{% elif job.status == 'processing' %}#00897b{% elif job.status == 'error' %}#c62828{% else %}#f57f17{% endif %};">
|
||
{% if job.status == 'completed' %}✓ Completed{% elif job.status == 'processing' %}⏳ Processing{% elif job.status == 'error' %}✗ Error{% else %}⌛ Queued{% endif %}
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
{% if job.status == 'processing' and job.progress %}
|
||
<div>
|
||
<div class="info-label">Progress</div>
|
||
<div class="progress-bar">
|
||
<div class="progress-fill" id="progress-{{ job_id }}"></div>
|
||
</div>
|
||
<div class="progress-text" id="progress-text-{{ job_id }}">{{ job.progress }}</div>
|
||
</div>
|
||
{% endif %}
|
||
|
||
{% if job.zip_contents %}
|
||
<div class="zip-contents">
|
||
{% for zip_name, pdf_files in job.zip_contents.items() %}
|
||
<div class="zip-toggle" onclick="toggleDetails(this)">
|
||
<span>📦 {{ zip_name }} ({{ pdf_files|length }} files)</span>
|
||
<span>▶</span>
|
||
</div>
|
||
<div class="zip-details">
|
||
<ul class="file-list">
|
||
{% for pdf_file in pdf_files %}
|
||
<li>{{ pdf_file }}</li>
|
||
{% endfor %}
|
||
</ul>
|
||
</div>
|
||
{% endfor %}
|
||
</div>
|
||
{% endif %}
|
||
|
||
{% if job.status == 'completed' %}
|
||
{% if job.skipped and job.skipped|length > 0 %}
|
||
<div class="skipped-section">
|
||
<div class="skipped-toggle" onclick="toggleSkipped(this)">
|
||
<span>⚠️ Skipped Files ({{ job.skipped|length }})</span>
|
||
<span>▶</span>
|
||
</div>
|
||
<div class="skipped-details">
|
||
<ul class="file-list">
|
||
{% for skip_info in job.skipped %}
|
||
<li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}</li>
|
||
{% endfor %}
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
{% endif %}
|
||
|
||
<div class="action-buttons">
|
||
{% if job.zip_name %}
|
||
<a href="/download/{{ job.zip_name }}" class="btn btn-download">⬇️ Download ZIP</a>
|
||
{% else %}
|
||
<span style="color: #999; font-size: 14px;">No files were successfully processed</span>
|
||
{% endif %}
|
||
<form method="post" action="/delete/{{ job_id }}" style="display: contents;">
|
||
<button type="submit" class="btn btn-delete" onclick="return confirm('Are you sure? This cannot be undone.')">🗑️ Delete Job</button>
|
||
</form>
|
||
</div>
|
||
{% endif %}
|
||
</div>
|
||
</div>
|
||
{% endfor %}
|
||
{% else %}
|
||
<div class="empty-state">
|
||
<div class="empty-state-icon">📭</div>
|
||
<div class="empty-state-text">No jobs yet</div>
|
||
<a href="/" class="empty-state-link">Upload Files to Get Started</a>
|
||
</div>
|
||
{% endif %}
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
let previousJobStatuses = {};
|
||
|
||
function toggleDetails(element) {
|
||
const details = element.nextElementSibling;
|
||
const arrow = element.querySelector('span:last-child');
|
||
if (details.style.display === 'block') {
|
||
details.style.display = 'none';
|
||
arrow.textContent = '▶';
|
||
} else {
|
||
details.style.display = 'block';
|
||
arrow.textContent = '▼';
|
||
}
|
||
}
|
||
|
||
function toggleSkipped(element) {
|
||
const details = element.nextElementSibling;
|
||
const arrow = element.querySelector('span:last-child');
|
||
if (details.style.display === 'block') {
|
||
details.style.display = 'none';
|
||
arrow.textContent = '▶';
|
||
} else {
|
||
details.style.display = 'block';
|
||
arrow.textContent = '▼';
|
||
}
|
||
}
|
||
|
||
function updateJobs() {
|
||
fetch('/api/jobs', { credentials: 'same-origin' })
|
||
.then(response => {
|
||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||
return response.json();
|
||
})
|
||
.then(data => {
|
||
const jobs = Object.entries(data);
|
||
|
||
// Check for job completions
|
||
let hasNewCompletion = false;
|
||
if (jobs.length > 0) {
|
||
jobs.forEach(([jobId, job]) => {
|
||
if (job.status === 'completed' && previousJobStatuses[jobId] === 'processing') {
|
||
hasNewCompletion = true;
|
||
}
|
||
previousJobStatuses[jobId] = job.status;
|
||
});
|
||
}
|
||
|
||
if (hasNewCompletion) {
|
||
setTimeout(() => { window.location.href = '/completed-jobs'; }, 500);
|
||
return;
|
||
}
|
||
|
||
// Update progress bars for processing jobs
|
||
jobs.forEach(([jobId, job]) => {
|
||
if (job.status === 'processing' && job.progress) {
|
||
const progressBar = document.getElementById(`progress-${jobId}`);
|
||
const progressText = document.getElementById(`progress-text-${jobId}`);
|
||
if (progressBar) {
|
||
const match = job.progress.match(/(\d+)\/(\d+)/);
|
||
if (match) {
|
||
const current = parseInt(match[1]);
|
||
const total = parseInt(match[2]);
|
||
const percentage = (current / total) * 100;
|
||
progressBar.style.width = percentage + '%';
|
||
}
|
||
}
|
||
if (progressText) {
|
||
progressText.textContent = job.progress;
|
||
}
|
||
}
|
||
});
|
||
})
|
||
.catch(err => console.error('Error updating jobs:', err));
|
||
}
|
||
|
||
// Update every 2 seconds
|
||
setInterval(updateJobs, 2000);
|
||
updateJobs(); // Initial update
|
||
</script>
|
||
</body>
|
||
</html>
|
||
''')
|
||
<script>
|
||
// Track previous job statuses to detect completion
|
||
let previousJobStatuses = {};
|
||
|
||
// Collapsible section functions
|
||
function toggleZipDetails(element) {
|
||
const details = element.nextElementSibling;
|
||
const isVisible = details.style.display === 'block';
|
||
details.style.display = isVisible ? 'none' : 'block';
|
||
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
||
}
|
||
|
||
function toggleSkippedDetails(element) {
|
||
const details = element.nextElementSibling;
|
||
const isVisible = details.style.display === 'block';
|
||
details.style.display = isVisible ? 'none' : 'block';
|
||
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
||
}
|
||
|
||
function updateJobs() {
|
||
console.log('[updateJobs] Starting update...');
|
||
fetch('/api/jobs', { credentials: 'same-origin' })
|
||
.then(response => {
|
||
console.log('[updateJobs] Response received:', response.status);
|
||
if (!response.ok) {
|
||
throw new Error(`HTTP ${response.status}`);
|
||
}
|
||
return response.json();
|
||
})
|
||
.then(data => {
|
||
console.log('[updateJobs] Job data received:', data);
|
||
const jobs = Object.entries(data);
|
||
console.log('[updateJobs] Number of jobs:', jobs.length);
|
||
|
||
// Check for job completions
|
||
let hasNewCompletion = false;
|
||
if (jobs.length > 0) {
|
||
jobs.forEach(([jobId, job]) => {
|
||
console.log(`[updateJobs] Job ${jobId}: status=${job.status}, progress=${job.progress || 'none'}`);
|
||
|
||
// Check if job just completed
|
||
if (job.status === 'completed' && previousJobStatuses[jobId] === 'processing') {
|
||
hasNewCompletion = true;
|
||
console.log(`[updateJobs] Job ${jobId} just completed!`);
|
||
}
|
||
|
||
// Update previous status
|
||
previousJobStatuses[jobId] = job.status;
|
||
});
|
||
}
|
||
|
||
// Redirect to completed jobs if a job just finished
|
||
if (hasNewCompletion) {
|
||
console.log('[updateJobs] Redirecting to completed jobs page...');
|
||
setTimeout(() => {
|
||
window.location.href = '/completed-jobs';
|
||
}, 500); // Small delay to show completion briefly
|
||
return; // Stop processing to avoid UI updates during redirect
|
||
}
|
||
|
||
console.log('[updateJobs] Starting DOM manipulation...');
|
||
// Find the container where jobs are rendered
|
||
const body = document.body;
|
||
const existingJobs = body.querySelectorAll('.job');
|
||
console.log(`[updateJobs] Found ${existingJobs.length} existing job elements`);
|
||
|
||
// Remove existing job elements
|
||
existingJobs.forEach(job => {
|
||
console.log('[updateJobs] Removing job element');
|
||
job.remove();
|
||
});
|
||
|
||
// Remove existing "No jobs yet" messages
|
||
const noJobsMsgs = body.querySelectorAll('p');
|
||
noJobsMsgs.forEach(p => {
|
||
if (p.textContent.includes('No jobs yet')) {
|
||
console.log('[updateJobs] Removing "no jobs" message');
|
||
p.remove();
|
||
}
|
||
});
|
||
|
||
// Find where to insert new jobs (after nav-links div)
|
||
const navLinks = body.querySelector('.nav-links');
|
||
const scriptTag = body.querySelector('script');
|
||
console.log('[updateJobs] navLinks found:', !!navLinks, 'scriptTag found:', !!scriptTag);
|
||
|
||
// Add updated job elements
|
||
const jobEntries = Object.entries(data);
|
||
console.log(`[updateJobs] Rendering ${jobEntries.length} jobs`);
|
||
|
||
if (jobEntries.length > 0) {
|
||
jobEntries.forEach(([jobId, job]) => {
|
||
console.log(`[updateJobs] Creating element for job ${jobId}`);
|
||
try {
|
||
const jobDiv = document.createElement('div');
|
||
jobDiv.className = 'job';
|
||
|
||
// Build the HTML step by step to avoid syntax errors
|
||
let jobHTML = `
|
||
<h3>Job ${jobId.substring(0, 8)}</h3>
|
||
<p><strong>Status:</strong> <span class="status ${job.status}">${job.status.charAt(0).toUpperCase() + job.status.slice(1)}</span></p>`;
|
||
|
||
// Add progress if processing
|
||
if (job.status === 'processing' && job.progress) {
|
||
jobHTML += `<p><strong>Progress:</strong> <span class="progress-text">${job.progress}</span></p>`;
|
||
}
|
||
|
||
jobHTML += `<p><strong>Files:</strong> ${job.files ? job.files.length : 0}</p>`;
|
||
|
||
// Add ZIP contents if available
|
||
if (job.zip_contents) {
|
||
jobHTML += `<p><strong>ZIP Contents:</strong></p>`;
|
||
Object.entries(job.zip_contents).forEach(([zipName, pdfFiles]) => {
|
||
jobHTML += `
|
||
<div class="zip-content">
|
||
<div class="zip-toggle" onclick="toggleZipDetails(this)">
|
||
<strong>${zipName}</strong> (${pdfFiles.length} files) ▶
|
||
</div>
|
||
<div class="zip-details">
|
||
<ul>
|
||
${pdfFiles.map(pdfFile => `<li>${pdfFile}</li>`).join('')}
|
||
</ul>
|
||
</div>
|
||
</div>`;
|
||
});
|
||
}
|
||
|
||
// Add download/delete buttons if completed
|
||
if (job.status === 'completed') {
|
||
if (job.zip_name) {
|
||
jobHTML += `
|
||
<p><a href="/download/${job.zip_name}" class="download-btn">Download ZIP</a>
|
||
<form method="post" action="/delete/${jobId}" style="display: inline;">
|
||
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
|
||
</form></p>`;
|
||
} else {
|
||
jobHTML += `
|
||
<p><em>No files were successfully processed to create a ZIP archive.</em>
|
||
<form method="post" action="/delete/${jobId}" style="display: inline;">
|
||
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
|
||
</form></p>`;
|
||
}
|
||
|
||
// Add skipped files section if any
|
||
if (job.skipped && job.skipped.length > 0) {
|
||
jobHTML += `
|
||
<div class="skipped-section">
|
||
<div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
|
||
⚠️ Skipped Files (${job.skipped.length}) ▶
|
||
</div>
|
||
<div class="skipped-details">
|
||
<ul>
|
||
${job.skipped.map(skipInfo => `<li><strong>${skipInfo[0]}</strong>: ${skipInfo[1]}</li>`).join('')}
|
||
</ul>
|
||
</div>
|
||
</div>`;
|
||
}
|
||
}
|
||
|
||
jobDiv.innerHTML = jobHTML;
|
||
console.log(`[updateJobs] HTML created for job ${jobId}`);
|
||
|
||
// Insert job before the script tag
|
||
if (scriptTag) {
|
||
body.insertBefore(jobDiv, scriptTag);
|
||
} else {
|
||
body.appendChild(jobDiv);
|
||
}
|
||
console.log(`[updateJobs] Job ${jobId} element inserted into DOM`);
|
||
} catch (err) {
|
||
console.error(`[updateJobs] Error creating job element for ${jobId}:`, err);
|
||
}
|
||
});
|
||
} else {
|
||
console.log('[updateJobs] No jobs to render, showing "no jobs" message');
|
||
const noJobsMsg = document.createElement('p');
|
||
noJobsMsg.innerHTML = 'No jobs yet. <a href="/">Upload some files</a> to get started.';
|
||
if (scriptTag) {
|
||
body.insertBefore(noJobsMsg, scriptTag);
|
||
} else {
|
||
body.appendChild(noJobsMsg);
|
||
}
|
||
}
|
||
console.log('[updateJobs] Update complete!');
|
||
})
|
||
.catch(function(error) {
|
||
console.error('[updateJobs] Error updating jobs:', error);
|
||
if (error.stack) {
|
||
console.error('[updateJobs] Error stack:', error.stack);
|
||
}
|
||
});
|
||
}
|
||
|
||
console.log('[Init] Setting up job update interval...');
|
||
// Update jobs every 1 second
|
||
const updateInterval = setInterval(updateJobs, 1000);
|
||
console.log('[Init] Update interval set, ID:', updateInterval);
|
||
|
||
// Initialize previous job statuses
|
||
console.log('[Init] Running initial job update...');
|
||
updateJobs(); // Run once immediately to populate initial statuses
|
||
</script>
|
||
</body>
|
||
</html>
|
||
''', jobs=job_queue)
|
||
|
||
@app.route('/api/jobs')
|
||
@login_required
|
||
def api_jobs():
|
||
return jsonify(job_queue)
|
||
|
||
@app.route('/download/<filename>')
|
||
@login_required
|
||
def download(filename):
|
||
"""Download a completed job's ZIP file with path traversal protection"""
|
||
# Prevent path traversal attacks
|
||
if '..' in filename or '/' in filename or '\\' in filename:
|
||
return "Invalid filename", 400
|
||
|
||
# Security check: only allow downloading files that belong to completed jobs
|
||
with job_lock:
|
||
for job_id, job in job_queue.items():
|
||
if job.get('status') == 'completed' and job.get('zip_name') == filename:
|
||
# Use ARCHIVE_FOLDER instead of UPLOAD_FOLDER
|
||
file_path = os.path.join(app.config['ARCHIVE_FOLDER'], filename)
|
||
# Verify the resolved path is within ARCHIVE_FOLDER
|
||
resolved_path = os.path.abspath(file_path)
|
||
archive_folder = os.path.abspath(app.config['ARCHIVE_FOLDER'])
|
||
|
||
if not resolved_path.startswith(archive_folder):
|
||
return "Access denied", 403
|
||
|
||
if os.path.exists(resolved_path):
|
||
return send_from_directory(app.config['ARCHIVE_FOLDER'], filename, as_attachment=True)
|
||
|
||
return "File not found or access denied", 404
|
||
|
||
@app.route('/delete/<job_id>', methods=['POST'])
|
||
@login_required
|
||
def delete_job(job_id):
|
||
import sys
|
||
print(f"DEBUG: delete_job called for job_id: {job_id}", file=sys.stderr, flush=True)
|
||
print(f"DEBUG: delete_job called for job_id: {job_id}", flush=True)
|
||
|
||
try:
|
||
print(f"DEBUG: Acquiring job_lock", file=sys.stderr, flush=True)
|
||
with job_lock:
|
||
print(f"DEBUG: Lock acquired, checking if job exists", file=sys.stderr, flush=True)
|
||
if job_id in job_queue:
|
||
job = job_queue[job_id]
|
||
print(f"DEBUG: Job found with status: {job.get('status')}", file=sys.stderr, flush=True)
|
||
|
||
# Only allow deletion of completed jobs
|
||
if job.get('status') == 'completed':
|
||
print(f"DEBUG: Job is completed, proceeding with deletion", file=sys.stderr, flush=True)
|
||
|
||
# Clean up the ZIP file from archives folder
|
||
if job.get('zip_name'):
|
||
print(f"DEBUG: ARCHIVE_FOLDER config: {app.config.get('ARCHIVE_FOLDER')}", file=sys.stderr, flush=True)
|
||
zip_path = os.path.join(app.config['ARCHIVE_FOLDER'], job['zip_name'])
|
||
print(f"DEBUG: Attempting to delete ZIP at: {zip_path}", file=sys.stderr, flush=True)
|
||
if os.path.exists(zip_path):
|
||
try:
|
||
os.remove(zip_path)
|
||
print(f"DEBUG: Deleted archive file: {zip_path}", file=sys.stderr, flush=True)
|
||
except Exception as e:
|
||
print(f"ERROR: Failed to delete archive: {e}", file=sys.stderr, flush=True)
|
||
import traceback
|
||
traceback.print_exc(file=sys.stderr)
|
||
else:
|
||
print(f"DEBUG: ZIP file does not exist: {zip_path}", file=sys.stderr, flush=True)
|
||
|
||
# Clean up any remaining uploaded files
|
||
print(f"DEBUG: Cleaning up {len(job.get('files', []))} uploaded files", file=sys.stderr, flush=True)
|
||
for file_info in job.get('files', []):
|
||
if 'path' in file_info and os.path.exists(file_info['path']):
|
||
try:
|
||
os.remove(file_info['path'])
|
||
print(f"DEBUG: Deleted uploaded file: {file_info['path']}", file=sys.stderr, flush=True)
|
||
except Exception as e:
|
||
print(f"ERROR: Failed to delete uploaded file: {e}", file=sys.stderr, flush=True)
|
||
|
||
# Remove the job from the queue
|
||
print(f"DEBUG: Removing job from queue", file=sys.stderr, flush=True)
|
||
del job_queue[job_id]
|
||
print(f"DEBUG: Saving jobs to disk", file=sys.stderr, flush=True)
|
||
_save_jobs_unlocked() # Use unlocked version since we already hold the lock
|
||
print(f"DEBUG: Job deletion complete", file=sys.stderr, flush=True)
|
||
else:
|
||
print(f"DEBUG: Job status is not completed: {job.get('status')}", file=sys.stderr, flush=True)
|
||
return "Only completed jobs can be deleted", 400
|
||
else:
|
||
print(f"DEBUG: Job not found in queue", file=sys.stderr, flush=True)
|
||
return "Job not found", 404
|
||
except Exception as e:
|
||
print(f"ERROR: Exception in delete_job: {e}", file=sys.stderr, flush=True)
|
||
print(f"ERROR: Exception in delete_job: {e}", flush=True)
|
||
import traceback
|
||
traceback.print_exc(file=sys.stderr)
|
||
traceback.print_exc()
|
||
return f"Error deleting job: {str(e)}", 500
|
||
|
||
print(f"DEBUG: Redirecting to /jobs", file=sys.stderr, flush=True)
|
||
try:
|
||
return redirect('/jobs')
|
||
except Exception as e:
|
||
print(f"ERROR: Redirect failed: {e}", file=sys.stderr, flush=True)
|
||
return "Job deleted but redirect failed", 200
|
||
|
||
@app.route('/completed-jobs')
|
||
@login_required
|
||
def completed_jobs():
|
||
# Show ALL jobs with their statuses for debugging
|
||
all_jobs = job_queue
|
||
return render_template_string('''
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>All Jobs Status</title>
|
||
<style>
|
||
body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
|
||
img { margin-bottom: 20px; }
|
||
h1 { color: #333; }
|
||
.job { background-color: #fff; margin: 10px auto; padding: 15px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); max-width: 1000px; text-align: left; }
|
||
.status { padding: 5px 10px; border-radius: 5px; color: white; font-weight: bold; }
|
||
.status.completed { background-color: #28a745; }
|
||
.status.processing { background-color: #17a2b8; }
|
||
.status.error { background-color: #dc3545; }
|
||
.progress-text { color: #17a2b8; font-style: italic; margin-left: 10px; }
|
||
.download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin: 5px; display: inline-block; }
|
||
.download-btn:hover { background-color: #0056b3; }
|
||
.delete-btn { background-color: #dc3545; color: white; border: none; padding: 8px 16px; border-radius: 5px; margin: 5px; cursor: pointer; }
|
||
.delete-btn:hover { background-color: #c82333; }
|
||
.flash { padding: 10px; margin: 10px 0; border-radius: 5px; text-align: center; }
|
||
.flash.success { background-color: #d4edda; color: #155724; border: 1px solid #c3e6cb; }
|
||
.flash.error { background-color: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
|
||
.logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
|
||
.logout:hover { background-color: #c82333; }
|
||
.nav-links { margin-bottom: 20px; }
|
||
.nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
|
||
.nav-links a:hover { background-color: #e9ecef; }
|
||
.nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
|
||
.nav-btn:hover { background-color: #0056b3; }
|
||
.job-stats { background-color: #e9ecef; padding: 10px; border-radius: 5px; margin-bottom: 15px; }
|
||
.file-count { color: #17a2b8; font-weight: bold; }
|
||
.zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
|
||
.zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
|
||
.zip-toggle:hover { background-color: #dee2e6; }
|
||
.zip-details { display: none; margin-top: 10px; }
|
||
.skipped-section { background-color: #fff3cd; border: 1px solid #ffeaa7; border-radius: 5px; padding: 10px; margin: 10px 0; }
|
||
.skipped-toggle { cursor: pointer; background-color: #ffeaa7; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #ffc107; font-weight: bold; color: #856404; }
|
||
.skipped-toggle:hover { background-color: #ffc107; }
|
||
.skipped-details { display: none; margin-top: 10px; }
|
||
ul { list-style-type: none; padding: 0; }
|
||
li { background-color: #ffffff; margin: 2px 0; padding: 5px; border-radius: 3px; }
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<a href="/logout" class="logout">Logout</a>
|
||
<img src="/static/trbc-logo-full.png" alt="Logo" width="200">
|
||
<h1>All Jobs Status</h1>
|
||
<div class="nav-links">
|
||
<a href="/" class="nav-btn">← Upload Files</a>
|
||
<a href="/jobs" class="nav-btn">View All Jobs →</a>
|
||
</div>
|
||
{% with messages = get_flashed_messages(with_categories=true) %}
|
||
{% if messages %}
|
||
{% for category, message in messages %}
|
||
<div class="flash {{ category }}">{{ message }}</div>
|
||
{% endfor %}
|
||
{% endif %}
|
||
{% endwith %}
|
||
{% for job_id, job in all_jobs.items() %}
|
||
<div class="job">
|
||
<h3>Job {{ job_id[:8] }} - {{ job.created_at if job.created_at else 'Unknown Date' }}</h3>
|
||
<div class="job-stats">
|
||
<span class="status {{ job.status }}">{{ job.status.title() }}</span>
|
||
<span class="file-count">{{ job.processed_files|selectattr('0', 'equalto', 'success')|list|length }} files processed</span>
|
||
{% if job.progress %}
|
||
<span class="progress-text">Progress: {{ job.progress }}</span>
|
||
{% endif %}
|
||
{% if job.skipped %}
|
||
<span style="color: #856404; font-weight: bold;">{{ job.skipped|length }} files skipped</span>
|
||
{% endif %}
|
||
</div>
|
||
|
||
{% if job.zip_contents %}
|
||
<p><strong>Original ZIP Contents:</strong></p>
|
||
{% for zip_name, pdf_files in job.zip_contents.items() %}
|
||
<div class="zip-content">
|
||
<div class="zip-toggle" onclick="toggleZipDetails(this)">
|
||
<strong>{{ zip_name }}</strong> ({{ pdf_files|length }} files) ▶
|
||
</div>
|
||
<div class="zip-details">
|
||
<ul>
|
||
{% for pdf_file in pdf_files %}
|
||
<li>{{ pdf_file }}</li>
|
||
{% endfor %}
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
{% endfor %}
|
||
{% endif %}
|
||
|
||
{% if job.zip_name %}
|
||
<p><strong>Download:</strong> <a href="/download/{{ job.zip_name }}" class="download-btn">📁 Download Organized ZIP ({{ job.zip_name }})</a></p>
|
||
{% endif %}
|
||
|
||
{% if job.skipped %}
|
||
<div class="skipped-section">
|
||
<div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
|
||
⚠️ Skipped Files ({{ job.skipped|length }}) ▶
|
||
</div>
|
||
<div class="skipped-details">
|
||
<ul>
|
||
{% for skip_info in job.skipped %}
|
||
<li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}
|
||
{% if skip_info|length > 2 and skip_info[2] %}
|
||
<br><details><summary>Extracted Text</summary><pre style="white-space: pre-wrap; background: #f8f9fa; border: 1px solid #eee; padding: 5px;">{{ skip_info[2] }}</pre></details>
|
||
{% endif %}
|
||
</li>
|
||
{% endfor %}
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
{% endif %}
|
||
|
||
<form method="post" action="/delete/{{ job_id }}" style="display: inline;">
|
||
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to permanently delete this completed job and its files?')">🗑️ Delete Job</button>
|
||
</form>
|
||
</div>
|
||
{% endfor %}
|
||
{% if not jobs %}
|
||
<p>No completed jobs yet. <a href="/">Upload some files</a> to get started.</p>
|
||
{% endif %}
|
||
<script>
|
||
// Collapsible section functions
|
||
function toggleZipDetails(element) {
|
||
const details = element.nextElementSibling;
|
||
const isVisible = details.style.display === 'block';
|
||
details.style.display = isVisible ? 'none' : 'block';
|
||
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
||
}
|
||
|
||
function toggleSkippedDetails(element) {
|
||
const details = element.nextElementSibling;
|
||
const isVisible = details.style.display === 'block';
|
||
details.style.display = isVisible ? 'none' : 'block';
|
||
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
||
}
|
||
</script>
|
||
</body>
|
||
</html>
|
||
''', all_jobs=all_jobs)
|
||
|
||
if __name__ == '__main__':
|
||
app.run(debug=True, port=3000) |