1346 lines
64 KiB
Python
1346 lines
64 KiB
Python
import os
|
|
import tempfile
|
|
import zipfile
|
|
import threading
|
|
import uuid
|
|
import json
|
|
from datetime import datetime
|
|
from functools import wraps
|
|
from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory, flash, get_flashed_messages, jsonify
|
|
from werkzeug.utils import secure_filename
|
|
from werkzeug.middleware.proxy_fix import ProxyFix
|
|
from PyPDF2 import PdfReader
|
|
from authlib.integrations.flask_client import OAuth
|
|
|
|
app = Flask(__name__)
|
|
|
|
# Trust proxy headers (since we're behind Nginx)
|
|
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
|
|
|
|
app.config['UPLOAD_FOLDER'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'processed')
|
|
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
|
|
app.config['MAX_CONTENT_LENGTH'] = 500 * 1024 * 1024 # 500MB limit
|
|
app.secret_key = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production')
|
|
|
|
# Entra ID Configuration
|
|
TENANT_ID = os.getenv('TENANT_ID', 'your-tenant-id')
|
|
CLIENT_ID = os.getenv('CLIENT_ID', 'your-client-id')
|
|
CLIENT_SECRET = os.getenv('CLIENT_SECRET', 'your-client-secret')
|
|
|
|
print(f"DEBUG: TENANT_ID = {TENANT_ID}")
|
|
print(f"DEBUG: CLIENT_ID = {CLIENT_ID}")
|
|
print(f"DEBUG: CLIENT_SECRET = {'*' * len(CLIENT_SECRET) if CLIENT_SECRET else 'None'}")
|
|
|
|
# Admin Configuration
|
|
ADMIN_USERNAME = os.getenv('ADMIN_USERNAME', 'trbcit')
|
|
ADMIN_PASSWORD = os.getenv('ADMIN_PASSWORD', 'atdrmbicn')
|
|
|
|
# Initialize OAuth
|
|
oauth = OAuth(app)
|
|
|
|
# Register Microsoft OAuth provider with proper configuration
|
|
try:
|
|
microsoft = oauth.register(
|
|
name='microsoft',
|
|
client_id=CLIENT_ID,
|
|
client_secret=CLIENT_SECRET,
|
|
server_metadata_url=f'https://login.microsoftonline.com/{TENANT_ID}/v2.0/.well-known/openid-configuration',
|
|
client_kwargs={
|
|
'scope': 'openid email profile',
|
|
'token_endpoint_auth_method': 'client_secret_post'
|
|
}
|
|
)
|
|
print("DEBUG: Microsoft OAuth provider registered successfully")
|
|
except Exception as e:
|
|
print(f"ERROR: Failed to register Microsoft OAuth provider: {e}")
|
|
microsoft = None
|
|
|
|
# Job persistence
|
|
JOBS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'jobs.json')
|
|
job_queue = {}
|
|
job_lock = threading.Lock()
|
|
|
|
def save_jobs():
|
|
"""Save jobs to persistent storage"""
|
|
try:
|
|
with job_lock:
|
|
# Convert datetime objects to strings for JSON serialization
|
|
jobs_to_save = {}
|
|
for job_id, job in job_queue.items():
|
|
jobs_to_save[job_id] = job.copy()
|
|
# Convert any datetime objects if needed
|
|
if 'created_at' in jobs_to_save[job_id]:
|
|
jobs_to_save[job_id]['created_at'] = str(jobs_to_save[job_id]['created_at'])
|
|
|
|
with open(JOBS_FILE, 'w') as f:
|
|
json.dump(jobs_to_save, f, indent=2)
|
|
except Exception as e:
|
|
print(f"ERROR: Failed to save jobs: {e}")
|
|
|
|
def load_jobs():
|
|
"""Load jobs from persistent storage"""
|
|
global job_queue
|
|
try:
|
|
if os.path.exists(JOBS_FILE):
|
|
with open(JOBS_FILE, 'r') as f:
|
|
loaded_jobs = json.load(f)
|
|
job_queue = loaded_jobs
|
|
print(f"DEBUG: Loaded {len(job_queue)} jobs from persistent storage")
|
|
else:
|
|
print("DEBUG: No existing jobs file found, starting fresh")
|
|
except Exception as e:
|
|
print(f"ERROR: Failed to load jobs: {e}")
|
|
job_queue = {}
|
|
|
|
# Load jobs on startup
|
|
load_jobs()
|
|
|
|
def update_job_status(job_id, status, **kwargs):
|
|
"""Update job status and persist changes"""
|
|
with job_lock:
|
|
if job_id in job_queue:
|
|
job_queue[job_id]['status'] = status
|
|
job_queue[job_id].update(kwargs)
|
|
job_queue[job_id]['updated_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
save_jobs()
|
|
print(f"DEBUG: Job {job_id} status updated to {status}")
|
|
|
|
def analyze_failed_extractions(log_path):
|
|
"""
|
|
Analyze failed extraction log to identify new patterns.
|
|
Returns suggestions for new regex patterns.
|
|
"""
|
|
import re
|
|
from collections import defaultdict
|
|
|
|
if not os.path.exists(log_path):
|
|
return "No failed extraction log found."
|
|
|
|
patterns_found = defaultdict(int)
|
|
|
|
try:
|
|
with open(log_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
|
|
# Split into individual failed extractions
|
|
failed_entries = content.split('-' * 50)
|
|
|
|
for entry in failed_entries:
|
|
if 'Text content:' in entry:
|
|
text_part = entry.split('Text content:')[1].strip()
|
|
lines = text_part.split('\n')
|
|
|
|
# Analyze line patterns
|
|
for i, line in enumerate(lines):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
|
|
# Look for potential patterns
|
|
# Gender + something
|
|
if re.match(r'^(Female|Male)', line):
|
|
patterns_found['gender_start'] += 1
|
|
|
|
# School + something
|
|
if re.search(r'(?:High|School|Academy|Middle|Elementary)', line):
|
|
patterns_found['school_in_line'] += 1
|
|
|
|
# Grade + something
|
|
if re.match(r'^\d{1,2}', line):
|
|
patterns_found['grade_start'] += 1
|
|
|
|
# Names (capitalized words)
|
|
name_words = re.findall(r'\b([A-Z][a-z]+)\b', line)
|
|
if len(name_words) >= 2:
|
|
patterns_found['multiple_names'] += 1
|
|
|
|
# Check for concatenated patterns
|
|
if re.match(r'^[A-Z][a-z]+[A-Z][a-z]+', line):
|
|
patterns_found['concatenated_names'] += 1
|
|
|
|
except Exception as e:
|
|
return f"Error analyzing log: {str(e)}"
|
|
|
|
# Generate suggestions
|
|
suggestions = []
|
|
if patterns_found.get('gender_start', 0) > 0:
|
|
suggestions.append("Found gender at start of lines - may need new gender-first patterns")
|
|
if patterns_found.get('school_in_line', 0) > 0:
|
|
suggestions.append("Found school names in lines - may need school-based patterns")
|
|
if patterns_found.get('grade_start', 0) > 0:
|
|
suggestions.append("Found grades at start of lines - may need grade-first patterns")
|
|
if patterns_found.get('multiple_names', 0) > 0:
|
|
suggestions.append("Found multiple capitalized words - may need better name parsing")
|
|
if patterns_found.get('concatenated_names', 0) > 0:
|
|
suggestions.append("Found concatenated names - may need better concatenation handling")
|
|
|
|
if not suggestions:
|
|
suggestions.append("No obvious patterns detected - may need manual inspection")
|
|
|
|
def map_pattern_1(match1, match2):
|
|
return {
|
|
'gender': match1.group(1),
|
|
'school': match1.group(2).strip(),
|
|
'last_name': match1.group(3),
|
|
'grade': match2.group(1),
|
|
'first_name': match2.group(2)
|
|
}
|
|
|
|
def map_pattern_2(match1, match2, match3):
|
|
return {
|
|
'first_name': match1.group(1),
|
|
'gender': match2.group(1),
|
|
'school': match2.group(2).strip(),
|
|
'last_name': match2.group(3),
|
|
'grade': match3.group(1)
|
|
}
|
|
|
|
def map_pattern_3(match1, match2, match3):
|
|
return {
|
|
'gender': match1.group(1),
|
|
'first_name': match1.group(2),
|
|
'school': match2.group(1).strip(),
|
|
'last_name': match2.group(2),
|
|
'grade': match3.group(1)
|
|
}
|
|
|
|
def map_pattern_4(match1, match2, match3):
|
|
return {
|
|
'school': match1.group(1).strip(),
|
|
'grade': match1.group(2),
|
|
'gender': match2.group(1),
|
|
'first_name': match2.group(2),
|
|
'last_name': match3.group(1)
|
|
}
|
|
|
|
def map_pattern_5(match1, match2, match3):
|
|
return {
|
|
'first_name': match1.group(1),
|
|
'gender': match2.group(1),
|
|
'grade': match2.group(2),
|
|
'school': match3.group(1).strip(),
|
|
'last_name': match3.group(2)
|
|
}
|
|
|
|
def map_pattern_6(match1, match2, match3):
|
|
return {
|
|
'first_name': match1.group(1),
|
|
'school': match2.group(1).strip(),
|
|
'grade': match2.group(2),
|
|
'last_name': match2.group(3),
|
|
'gender': match3.group(1)
|
|
}
|
|
|
|
def map_pattern_7(match1, match2):
|
|
return {
|
|
'gender': match1.group(1),
|
|
'first_name': match1.group(2),
|
|
'school': match2.group(1).strip(),
|
|
'grade': match2.group(2),
|
|
'last_name': match2.group(3)
|
|
}
|
|
|
|
def map_pattern_8(match1, match2, match3):
|
|
return {
|
|
'school': match1.group(1).strip(),
|
|
'first_name': match1.group(2),
|
|
'last_name': match2.group(1),
|
|
'gender': match3.group(1),
|
|
'grade': match3.group(2)
|
|
}
|
|
|
|
def map_pattern_9(match1, match2, match3, match4):
|
|
return {
|
|
'first_name': match1.group(2),
|
|
'last_name': match1.group(1),
|
|
'grade': match2.group(1),
|
|
'school': match3.group(1).strip(),
|
|
'gender': match4.group(1)
|
|
}
|
|
|
|
def map_pattern_10(match1, match2):
|
|
return {
|
|
'gender': match1.group(1),
|
|
'school': match1.group(2).strip(),
|
|
'last_name': match1.group(3),
|
|
'grade': match2.group(1),
|
|
'first_name': match2.group(2)
|
|
}
|
|
|
|
ALLOWED_EXTENSIONS = {'pdf', 'zip'}
|
|
|
|
# Global jobs storage (in production, use database)
|
|
job_queue = {}
|
|
|
|
def login_required(f):
|
|
@wraps(f)
|
|
def decorated_function(*args, **kwargs):
|
|
print(f"DEBUG login_required: session = {dict(session)}")
|
|
if 'user' not in session and 'admin' not in session:
|
|
print("DEBUG: No user or admin in session, redirecting to login")
|
|
return redirect(url_for('login'))
|
|
print(f"DEBUG: User authenticated: {session.get('user', session.get('admin'))}")
|
|
return f(*args, **kwargs)
|
|
return decorated_function
|
|
|
|
def allowed_file(filename):
|
|
return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
|
|
|
|
def extract_names_from_pdf(pdf_path):
|
|
try:
|
|
reader = PdfReader(pdf_path)
|
|
|
|
# All PDFs are flattened, so only try text extraction from first page
|
|
if len(reader.pages) > 0:
|
|
first_page_text = reader.pages[0].extract_text()
|
|
extracted_data = extract_student_info_from_text(first_page_text)
|
|
|
|
if extracted_data and all(extracted_data.values()):
|
|
return {
|
|
'filename': f"{extracted_data['last_name']}{extracted_data['first_name']}_{extracted_data['grade']}.pdf",
|
|
'last_name': extracted_data['last_name'],
|
|
'first_name': extracted_data['first_name'],
|
|
'grade': extracted_data['grade'],
|
|
'school': extracted_data['school'],
|
|
'gender': extracted_data['gender']
|
|
}
|
|
|
|
# If we get here, we couldn't extract the data
|
|
# Log the failed extraction for debugging
|
|
import os
|
|
failed_log_path = os.path.join(os.path.dirname(pdf_path), 'failed_extractions.log')
|
|
try:
|
|
with open(failed_log_path, 'a', encoding='utf-8') as f:
|
|
f.write(f"Failed to extract from: {os.path.basename(pdf_path)}\n")
|
|
if len(reader.pages) > 0:
|
|
text = reader.pages[0].extract_text()
|
|
f.write(f"Text content:\n{text}\n")
|
|
f.write("-" * 50 + "\n")
|
|
except Exception as log_error:
|
|
pass # Don't let logging errors break the main flow
|
|
|
|
return {'error': 'Could not extract student information from PDF. Text extraction failed.'}
|
|
|
|
except Exception as e:
|
|
return {'error': f'PDF read error: {str(e)}'}
|
|
|
|
def extract_student_info_from_text(text):
|
|
"""
|
|
Extract student information from flattened PDF text.
|
|
This function handles multiple concatenation patterns and edge cases.
|
|
"""
|
|
import re
|
|
|
|
extracted = {
|
|
'first_name': '',
|
|
'last_name': '',
|
|
'grade': '',
|
|
'school': '',
|
|
'gender': ''
|
|
}
|
|
|
|
# Split text into lines for better parsing
|
|
lines = text.split('\n')
|
|
|
|
# First pass: Look for individual fields that appear on separate lines
|
|
for line in lines:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
|
|
# Look for grade (numbers, including K-12)
|
|
grade_match = re.match(r'^(?:Grade\s*)?(\d{1,2}|K|k|\d+(?:st|nd|rd|th))$', line)
|
|
if grade_match:
|
|
grade = grade_match.group(1).upper()
|
|
if grade in ['K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12']:
|
|
extracted['grade'] = grade
|
|
|
|
# Look for gender (case insensitive, handle common variations)
|
|
if re.match(r'^(Male|Female|M|F|Boy|Girl)$', line, re.IGNORECASE):
|
|
gender = line.capitalize()
|
|
if gender in ['Male', 'Female']:
|
|
extracted['gender'] = gender
|
|
|
|
# Look for school names (more flexible pattern)
|
|
school_match = re.match(r'^(.+?)(?:\s+(?:High|School|Academy|Middle|Elementary|Christian|Catholic|Public|Private))$', line, re.IGNORECASE)
|
|
if school_match and len(school_match.group(1)) > 3: # Avoid false positives
|
|
extracted['school'] = line
|
|
|
|
# Second pass: Try various concatenation patterns
|
|
patterns = [
|
|
# Pattern 1: Gender + School + LastName, Grade + FirstName
|
|
{
|
|
'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
|
|
'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$',
|
|
'map': map_pattern_1
|
|
},
|
|
# Pattern 2: FirstName, Gender + School + LastName, Grade
|
|
{
|
|
'regex1': r'^([A-Z][a-z]+)$',
|
|
'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
|
|
'regex3': r'^(\d{1,2}|K)$',
|
|
'map': map_pattern_2
|
|
},
|
|
# Pattern 3: Gender + FirstName, School + LastName, Grade
|
|
{
|
|
'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
|
|
'regex2': r'^(.+?)([A-Z][a-z]+)$',
|
|
'regex3': r'^(\d{1,2}|K)$',
|
|
'map': map_pattern_3
|
|
},
|
|
# Pattern 4: School + Grade, Gender + FirstName, LastName
|
|
{
|
|
'regex1': r'^(.+?)(\d{1,2}|K)$',
|
|
'regex2': r'^(Female|Male)([A-Z][a-z]+)$',
|
|
'regex3': r'^([A-Z][a-z]+)$',
|
|
'map': map_pattern_4
|
|
},
|
|
# Pattern 5: FirstName, Gender + Grade, School + LastName
|
|
{
|
|
'regex1': r'^([A-Z][a-z]+)$',
|
|
'regex2': r'^(Female|Male)(\d{1,2}|K)$',
|
|
'regex3': r'^(.+?)([A-Z][a-z]+)$',
|
|
'map': map_pattern_5
|
|
},
|
|
# Pattern 6: FirstName, School + Grade + LastName, Gender
|
|
{
|
|
'regex1': r'^([A-Z][a-z]+)$',
|
|
'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
|
|
'regex3': r'^(Female|Male)$',
|
|
'map': map_pattern_6
|
|
},
|
|
# Pattern 7: Gender + FirstName, School + Grade + LastName
|
|
{
|
|
'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
|
|
'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
|
|
'map': map_pattern_7
|
|
},
|
|
# Pattern 8: School + FirstName, LastName, Gender + Grade
|
|
{
|
|
'regex1': r'^(.+?)([A-Z][a-z]+)$',
|
|
'regex2': r'^([A-Z][a-z]+)$',
|
|
'regex3': r'^(Female|Male)(\d{1,2}|K)$',
|
|
'map': map_pattern_8
|
|
},
|
|
# Pattern 9: FirstLast (concatenated), Grade, School, Gender
|
|
{
|
|
'regex1': r'^([A-Z][a-z]+)([A-Z][a-z]+)$',
|
|
'regex2': r'^(\d{1,2}|K)$',
|
|
'regex3': r'^(.+)$',
|
|
'regex4': r'^(Female|Male)$',
|
|
'map': map_pattern_9
|
|
},
|
|
# Pattern 10: Gender + School + LastName, Grade + FirstName
|
|
{
|
|
'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
|
|
'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)',
|
|
'map': map_pattern_10
|
|
}
|
|
]
|
|
|
|
# Try each pattern
|
|
for pattern in patterns:
|
|
if all(extracted.values()): # Already found all fields
|
|
break
|
|
|
|
for i, line in enumerate(lines):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
|
|
match1 = re.match(pattern['regex1'], line)
|
|
if not match1:
|
|
continue
|
|
|
|
mapped = None
|
|
|
|
if 'regex4' in pattern and i + 3 < len(lines):
|
|
# Try 4-line pattern
|
|
match2 = re.match(pattern['regex2'], lines[i + 1].strip())
|
|
match3 = re.match(pattern['regex3'], lines[i + 2].strip())
|
|
match4 = re.match(pattern['regex4'], lines[i + 3].strip())
|
|
if match2 and match3 and match4:
|
|
mapped = pattern['map'](match1, match2, match3, match4)
|
|
elif 'regex3' in pattern and i + 2 < len(lines):
|
|
# Try 3-line pattern
|
|
match2 = re.match(pattern['regex2'], lines[i + 1].strip())
|
|
match3 = re.match(pattern['regex3'], lines[i + 2].strip())
|
|
if match2 and match3:
|
|
mapped = pattern['map'](match1, match2, match3)
|
|
elif 'regex2' in pattern and i + 1 < len(lines):
|
|
# Try 2-line pattern
|
|
match2 = re.match(pattern['regex2'], lines[i + 1].strip())
|
|
if match2:
|
|
if 'regex3' in pattern or 'regex4' in pattern:
|
|
# This pattern expects more matches but only has 2 lines available
|
|
continue
|
|
mapped = pattern['map'](match1, match2)
|
|
|
|
if mapped:
|
|
extracted.update(mapped)
|
|
|
|
# Third pass: Fallback logic for concatenated names and missing fields
|
|
if not extracted['first_name'] or not extracted['last_name']:
|
|
# Find all capitalized words that could be names
|
|
all_text = ' '.join(lines)
|
|
name_candidates = re.findall(r'\b([A-Z][a-z]+)\b', all_text)
|
|
|
|
# Filter out common non-name words
|
|
common_words = {'Student', 'First', 'Last', 'Name', 'Grade', 'School', 'Gender', 'Date', 'Id', 'Envelope', 'Docusign', 'Male', 'Female', 'Details', 'Registration', 'Episode', 'Page', 'Signature', 'Participant', 'Parent', 'Guardian', 'High', 'Academy', 'Christian', 'Liberty', 'Middle', 'Elementary', 'Catholic', 'Public', 'Private', 'Other', 'Not', 'Listed'}
|
|
name_words = [w for w in name_candidates if w not in common_words and len(w) > 1]
|
|
|
|
# Try to assign names based on context
|
|
if len(name_words) >= 2:
|
|
if not extracted['first_name']:
|
|
extracted['first_name'] = name_words[0]
|
|
if not extracted['last_name']:
|
|
extracted['last_name'] = name_words[-1] # Usually last name is last
|
|
|
|
# Final validation and cleanup
|
|
required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
|
|
if all(extracted.get(field) for field in required_fields):
|
|
# Clean up school names (remove extra spaces, standardize)
|
|
extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
|
|
return extracted
|
|
|
|
return None
|
|
|
|
def process_job(job_id, files):
|
|
try:
|
|
print("=== DEBUG: process_job STARTED ===")
|
|
job_queue[job_id]['status'] = 'processing'
|
|
job_queue[job_id]['progress'] = 'Starting job processing...'
|
|
save_jobs() # Persist initial progress
|
|
save_jobs() # Persist status change
|
|
processed_files = []
|
|
temp_extract_dirs = []
|
|
|
|
# Create base Episode directory for this job
|
|
episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'Episode')
|
|
os.makedirs(episode_dir, exist_ok=True)
|
|
job_queue[job_id]['progress'] = 'Created output directory structure'
|
|
|
|
# Analyze ZIP contents before processing
|
|
zip_contents = {}
|
|
job_queue[job_id]['progress'] = 'Analyzing uploaded files...'
|
|
for file_info in files:
|
|
if file_info['name'].lower().endswith('.zip'):
|
|
try:
|
|
with zipfile.ZipFile(file_info['path'], 'r') as zip_ref:
|
|
pdf_files = [f for f in zip_ref.namelist() if f.lower().endswith('.pdf')]
|
|
zip_contents[file_info['name']] = pdf_files
|
|
except Exception as e:
|
|
zip_contents[file_info['name']] = [f"Error reading ZIP: {str(e)}"]
|
|
|
|
job_queue[job_id]['zip_contents'] = zip_contents
|
|
|
|
total_files = sum(1 for file_info in files if file_info['name'].lower().endswith('.pdf')) + \
|
|
sum(len(zip_contents.get(file_info['name'], []))
|
|
for file_info in files if file_info['name'].lower().endswith('.zip'))
|
|
|
|
job_queue[job_id]['progress'] = f'Found {total_files} files to process'
|
|
save_jobs() # Persist file count
|
|
|
|
processed_count = 0
|
|
|
|
for file_info in files:
|
|
filepath = file_info['path']
|
|
filename = file_info['name']
|
|
|
|
if filename.lower().endswith('.zip'):
|
|
try:
|
|
job_queue[job_id]['progress'] = f'Extracting ZIP archive: {filename}'
|
|
print(f"DEBUG: Starting ZIP extraction for {filename}")
|
|
# Extract ZIP
|
|
extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
|
|
os.makedirs(extract_dir, exist_ok=True)
|
|
temp_extract_dirs.append(extract_dir)
|
|
print(f"DEBUG: Created extract directory: {extract_dir}")
|
|
with zipfile.ZipFile(filepath, 'r') as zip_ref:
|
|
file_count = len(zip_ref.namelist())
|
|
print(f"DEBUG: ZIP contains {file_count} files: {zip_ref.namelist()}")
|
|
zip_ref.extractall(extract_dir)
|
|
job_queue[job_id]['progress'] = f'Extracted {file_count} files from {filename}'
|
|
print(f"DEBUG: ZIP extraction completed, progress updated: {job_queue[job_id]['progress']}")
|
|
|
|
# Process PDFs in extract_dir
|
|
pdf_files_found = []
|
|
for root, dirs, files_in_zip in os.walk(extract_dir):
|
|
for f in files_in_zip:
|
|
if f.lower().endswith('.pdf'):
|
|
pdf_files_found.append(f)
|
|
pdf_path = os.path.join(root, f)
|
|
job_queue[job_id]['progress'] = f'Processing PDF: {f}'
|
|
print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
|
|
print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
|
|
pdf_info = extract_names_from_pdf(pdf_path)
|
|
print(f"DEBUG: PDF extraction result: {pdf_info}")
|
|
if 'error' in pdf_info:
|
|
processed_files.append(('skipped', f, pdf_info['error']))
|
|
job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
|
|
print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
|
|
else:
|
|
job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
|
|
print(f"DEBUG: Successfully extracted data for {pdf_info['first_name']} {pdf_info['last_name']}")
|
|
# Create folder structure: Episode/School/Grade/Gender/
|
|
school_dir = os.path.join(episode_dir, pdf_info['school'])
|
|
grade_dir = os.path.join(school_dir, pdf_info['grade'])
|
|
gender_dir = os.path.join(grade_dir, pdf_info['gender'])
|
|
print(f"DEBUG: Creating directories: {school_dir} -> {grade_dir} -> {gender_dir}")
|
|
os.makedirs(gender_dir, exist_ok=True)
|
|
print(f"DEBUG: Directories created successfully")
|
|
|
|
# Move file to appropriate folder with new name
|
|
new_path = os.path.join(gender_dir, pdf_info['filename'])
|
|
print(f"DEBUG: Attempting to move {pdf_path} to {new_path}")
|
|
counter = 1
|
|
while os.path.exists(new_path):
|
|
base, ext = os.path.splitext(pdf_info['filename'])
|
|
new_path = os.path.join(gender_dir, f"{base}_{counter}{ext}")
|
|
counter += 1
|
|
print(f"DEBUG: File exists, trying {new_path}")
|
|
print(f"DEBUG: Moving file to {new_path}")
|
|
os.rename(pdf_path, new_path)
|
|
processed_files.append(('success', os.path.relpath(new_path, episode_dir), new_path))
|
|
processed_count += 1
|
|
job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
|
|
print(f"DEBUG: Organized file {processed_count}/{total_files}")
|
|
|
|
print(f"DEBUG: Found {len(pdf_files_found)} PDF files in ZIP: {pdf_files_found}")
|
|
|
|
except Exception as e:
|
|
processed_files.append(('skipped', filename, f'ZIP processing error: {str(e)}'))
|
|
job_queue[job_id]['progress'] = f'Failed to process ZIP {filename}: {str(e)}'
|
|
print(f"ZIP processing error for {filename}: {str(e)}")
|
|
continue
|
|
else:
|
|
# Single PDF
|
|
try:
|
|
job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
|
|
pdf_info = extract_names_from_pdf(filepath)
|
|
if 'error' in pdf_info:
|
|
processed_files.append(('skipped', filename, pdf_info['error']))
|
|
job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
|
|
else:
|
|
job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
|
|
# Create folder structure: Episode/School/Grade/Gender/
|
|
school_dir = os.path.join(episode_dir, pdf_info['school'])
|
|
grade_dir = os.path.join(school_dir, pdf_info['grade'])
|
|
gender_dir = os.path.join(grade_dir, pdf_info['gender'])
|
|
os.makedirs(gender_dir, exist_ok=True)
|
|
|
|
# Move file to appropriate folder with new name
|
|
new_filepath = os.path.join(gender_dir, pdf_info['filename'])
|
|
counter = 1
|
|
while os.path.exists(new_filepath):
|
|
base, ext = os.path.splitext(pdf_info['filename'])
|
|
new_filepath = os.path.join(gender_dir, f"{base}_{counter}{ext}")
|
|
counter += 1
|
|
os.rename(filepath, new_filepath)
|
|
processed_files.append(('success', os.path.relpath(new_filepath, episode_dir), new_filepath))
|
|
processed_count += 1
|
|
job_queue[job_id]['progress'] = f'Organized {processed_count}/{total_files} files'
|
|
except Exception as e:
|
|
processed_files.append(('skipped', filename, f'PDF processing error: {str(e)}'))
|
|
job_queue[job_id]['progress'] = f'Failed to process PDF {filename}: {str(e)}'
|
|
print(f"PDF processing error for {filename}: {str(e)}")
|
|
continue
|
|
|
|
job_queue[job_id]['progress'] = 'Creating ZIP archive...'
|
|
# Create ZIP of the entire Episode directory structure
|
|
successful_files = [p for p in processed_files if p[0] == 'success']
|
|
print(f"DEBUG: Starting ZIP creation, successful_files: {len(successful_files)}")
|
|
if successful_files:
|
|
try:
|
|
zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip')
|
|
print(f"DEBUG: Creating ZIP at {zip_output}")
|
|
with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
|
|
for root, dirs, files_in_dir in os.walk(episode_dir):
|
|
for file in files_in_dir:
|
|
file_path = os.path.join(root, file)
|
|
arcname = os.path.relpath(file_path, episode_dir)
|
|
print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
|
|
zip_out.write(file_path, arcname)
|
|
job_queue[job_id]['zip_path'] = zip_output
|
|
job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
|
|
job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
|
|
save_jobs() # Persist completion progress
|
|
print(f"DEBUG: ZIP creation completed successfully")
|
|
except Exception as e:
|
|
job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
|
|
job_queue[job_id]['status'] = 'error'
|
|
save_jobs() # Persist error status
|
|
print(f"DEBUG: ZIP creation failed: {str(e)}")
|
|
else:
|
|
job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
|
|
print(f"DEBUG: No successful files to archive")
|
|
|
|
job_queue[job_id]['status'] = 'completed'
|
|
job_queue[job_id]['processed_files'] = processed_files
|
|
job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
|
|
save_jobs() # Persist completed job
|
|
print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")
|
|
|
|
except Exception as e:
|
|
job_queue[job_id]['status'] = 'error'
|
|
job_queue[job_id]['progress'] = f'Job failed with error: {str(e)}'
|
|
save_jobs() # Persist error status
|
|
print(f"Job {job_id} failed: {str(e)}") # Debug logging
|
|
|
|
@app.route('/login')
|
|
def login():
|
|
return render_template_string('''
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Login</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
|
|
.login-container { background-color: #fff; padding: 30px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); display: inline-block; margin: 20px; }
|
|
h1 { color: #333; }
|
|
.login-options { display: flex; justify-content: space-around; flex-wrap: wrap; }
|
|
.login-option { margin: 10px; }
|
|
a { color: #007bff; text-decoration: none; padding: 15px 30px; background-color: #007bff; color: white; border-radius: 5px; display: inline-block; }
|
|
a:hover { background-color: #0056b3; }
|
|
form { margin-top: 20px; }
|
|
input[type="text"], input[type="password"] { padding: 10px; margin: 5px; width: 200px; border: 1px solid #ccc; border-radius: 5px; }
|
|
input[type="submit"] { background-color: #28a745; color: white; border: none; padding: 10px 20px; border-radius: 5px; cursor: pointer; }
|
|
input[type="submit"]:hover { background-color: #218838; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<img src="/static/trbc-logo-full.png" alt="Logo" width="200">
|
|
<h1>PDF Renamer Login</h1>
|
|
<div class="login-options">
|
|
<div class="login-option">
|
|
<h2>Microsoft Entra ID</h2>
|
|
<a href="/entra-login">Login with Microsoft</a>
|
|
</div>
|
|
<div class="login-option">
|
|
<h2>Admin Login</h2>
|
|
<form method="post" action="/admin-login">
|
|
<input type="text" name="username" placeholder="Username" required><br>
|
|
<input type="password" name="password" placeholder="Password" required><br>
|
|
<input type="submit" value="Admin Login">
|
|
</form>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
''')
|
|
|
|
@app.route('/entra-login')
|
|
def entra_login():
|
|
# Check if Microsoft OAuth provider is available
|
|
if microsoft is None:
|
|
flash('Microsoft Entra ID authentication is not configured.', 'error')
|
|
return redirect(url_for('login'))
|
|
|
|
# Check if Microsoft Entra ID is configured
|
|
if TENANT_ID in ['your-tenant-id', 'YOUR_MICROSOFT_ENTRA_TENANT_ID'] or \
|
|
CLIENT_ID in ['your-client-id', 'YOUR_MICROSOFT_ENTRA_CLIENT_ID'] or \
|
|
CLIENT_SECRET in ['your-client-secret', 'YOUR_MICROSOFT_ENTRA_CLIENT_SECRET']:
|
|
flash('Microsoft Entra ID is not properly configured. Please check your environment variables.', 'error')
|
|
return redirect(url_for('login'))
|
|
|
|
try:
|
|
redirect_uri = url_for('auth', _external=True)
|
|
print(f"DEBUG: Initiating OAuth with redirect_uri: {redirect_uri}")
|
|
return microsoft.authorize_redirect(redirect_uri)
|
|
except Exception as e:
|
|
print(f"ERROR: Failed to initiate OAuth: {e}")
|
|
import traceback
|
|
print(f"Traceback: {traceback.format_exc()}")
|
|
flash('Failed to initiate Microsoft authentication. Please try again or use admin login.', 'error')
|
|
return redirect(url_for('login'))
|
|
|
|
@app.route('/admin-login', methods=['POST'])
|
|
def admin_login():
|
|
username = request.form.get('username')
|
|
password = request.form.get('password')
|
|
if username == ADMIN_USERNAME and password == ADMIN_PASSWORD:
|
|
session['admin'] = True
|
|
return redirect(url_for('upload_files'))
|
|
else:
|
|
return render_template_string('''
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Login Failed</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; background-color: #f4f4f4; text-align: center; padding: 50px; }
|
|
.error { color: #dc3545; background-color: #f8d7da; padding: 20px; border-radius: 5px; border: 1px solid #f5c6cb; }
|
|
a { color: #007bff; text-decoration: none; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="error">
|
|
<h1>Login Failed</h1>
|
|
<p>Invalid username or password.</p>
|
|
<a href="/login">Try Again</a>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
''')
|
|
|
|
@app.route('/auth')
|
|
def auth():
|
|
try:
|
|
print("DEBUG: Starting OAuth callback processing")
|
|
token = microsoft.authorize_access_token()
|
|
print(f"DEBUG: Got token: {token}")
|
|
|
|
# Get user info from the token's userinfo (already parsed by Authlib)
|
|
user_info = token.get('userinfo')
|
|
|
|
if not user_info:
|
|
print("ERROR: No userinfo in token")
|
|
flash('Failed to get user information from Microsoft.', 'error')
|
|
return redirect(url_for('login'))
|
|
|
|
print(f"DEBUG: Parsed user info: {user_info}")
|
|
|
|
# Store user information in session
|
|
session['user'] = {
|
|
'name': user_info.get('name', 'Unknown'),
|
|
'email': user_info.get('email') or user_info.get('preferred_username', 'unknown@example.com'),
|
|
'oid': user_info.get('oid'),
|
|
}
|
|
session.permanent = True
|
|
|
|
print(f"DEBUG: User logged in successfully: {session['user']}")
|
|
return redirect(url_for('upload_files'))
|
|
|
|
except Exception as e:
|
|
print(f"OAuth error: {e}") # Log the error for debugging
|
|
import traceback
|
|
print(f"Full traceback: {traceback.format_exc()}")
|
|
flash('Authentication failed. Please try again.', 'error')
|
|
return redirect(url_for('login'))
|
|
|
|
@app.route('/logout')
|
|
@login_required
|
|
def logout():
|
|
session.pop('user', None)
|
|
session.pop('admin', None)
|
|
return redirect(url_for('login'))
|
|
|
|
@app.route('/', methods=['GET', 'POST'])
|
|
@login_required
|
|
def upload_files():
|
|
if request.method == 'POST':
|
|
files = request.files.getlist('files')
|
|
if not files or all(not f.filename for f in files):
|
|
flash('No files selected.', 'error')
|
|
return redirect(url_for('upload_files'))
|
|
|
|
# Create job ID using timestamp format: YYYY-MM-DD_HH-MM-SS
|
|
job_id = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
|
|
job_queue[job_id] = {
|
|
'status': 'queued',
|
|
'files': [],
|
|
'zip_path': None,
|
|
'zip_name': None,
|
|
'processed_files': [],
|
|
'skipped': [],
|
|
'created_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
|
'job_id': job_id # Store the job ID for reference
|
|
}
|
|
|
|
file_list = []
|
|
for file in files:
|
|
if file and allowed_file(file.filename):
|
|
filename = secure_filename(file.filename)
|
|
filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
|
|
file.save(filepath)
|
|
file_list.append({'name': filename, 'path': filepath})
|
|
print(f"Uploaded file: {filename} -> {filepath}") # Debug logging
|
|
|
|
if not file_list:
|
|
flash('No valid files uploaded. Only PDF and ZIP files are allowed.', 'error')
|
|
job_queue.pop(job_id, None) # Remove empty job
|
|
return redirect(url_for('upload_files'))
|
|
|
|
job_queue[job_id]['files'] = file_list
|
|
print(f"Created job {job_id} with {len(file_list)} files") # Debug logging
|
|
|
|
# Save job to persistent storage
|
|
save_jobs()
|
|
|
|
# Start processing in background
|
|
def process_with_context():
|
|
with app.app_context():
|
|
process_job(job_id, file_list)
|
|
|
|
thread = threading.Thread(target=process_with_context)
|
|
thread.daemon = True
|
|
thread.start()
|
|
|
|
flash(f'Job created successfully. Processing {len(file_list)} files...', 'success')
|
|
return redirect(url_for('jobs'))
|
|
|
|
return render_template_string('''
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>PDF Renamer</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
|
|
img { margin-bottom: 20px; }
|
|
h1 { color: #333; }
|
|
form { background-color: #fff; padding: 20px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); display: inline-block; }
|
|
input[type="file"] { margin: 10px 0; }
|
|
input[type="submit"] { background-color: #28a745; color: white; border: none; padding: 10px 20px; border-radius: 5px; cursor: pointer; }
|
|
input[type="submit"]:hover { background-color: #218838; }
|
|
.logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
|
|
.logout:hover { background-color: #c82333; }
|
|
.view-jobs-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 10px 0; }
|
|
.view-jobs-btn:hover { background-color: #0056b3; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<a href="/logout" class="logout">Logout</a>
|
|
<img src="/static/trbc-logo-full.png" alt="Logo" width="200">
|
|
<h1>Upload PDF Files or ZIP to Rename</h1>
|
|
<a href="/jobs" class="view-jobs-btn">View Jobs</a>
|
|
<br><br>
|
|
<form method="post" enctype="multipart/form-data">
|
|
<input type="file" name="files" multiple accept=".pdf,.zip">
|
|
<br><br>
|
|
<input type="submit" value="Process PDFs">
|
|
</form>
|
|
</body>
|
|
</html>
|
|
''')
|
|
|
|
@app.route('/jobs')
|
|
@login_required
|
|
def jobs():
|
|
return render_template_string('''
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Jobs</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
|
|
img { margin-bottom: 20px; }
|
|
h1 { color: #333; }
|
|
.job { background-color: #fff; margin: 10px auto; padding: 15px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); max-width: 800px; text-align: left; }
|
|
.status { padding: 5px 10px; border-radius: 5px; color: white; font-weight: bold; }
|
|
.status.queued { background-color: #ffc107; }
|
|
.status.processing { background-color: #17a2b8; }
|
|
.status.completed { background-color: #28a745; }
|
|
.status.error { background-color: #dc3545; }
|
|
.progress-text { color: #17a2b8; font-style: italic; }
|
|
.download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; }
|
|
.download-btn:hover { background-color: #0056b3; }
|
|
.delete-btn { background-color: #dc3545; color: white; border: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; cursor: pointer; }
|
|
.delete-btn:hover { background-color: #c82333; }
|
|
.flash { padding: 10px; margin: 10px 0; border-radius: 5px; text-align: center; }
|
|
.flash.success { background-color: #d4edda; color: #155724; border: 1px solid #c3e6cb; }
|
|
.flash.error { background-color: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
|
|
.logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
|
|
.logout:hover { background-color: #c82333; }
|
|
.nav-links { margin-bottom: 20px; }
|
|
.nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
|
|
.nav-links a:hover { background-color: #e9ecef; }
|
|
.nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
|
|
.nav-btn:hover { background-color: #0056b3; }
|
|
.zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
|
|
.zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
|
|
.zip-toggle:hover { background-color: #dee2e6; }
|
|
.zip-details { display: none; margin-top: 10px; }
|
|
.skipped-section { background-color: #fff3cd; border: 1px solid #ffeaa7; border-radius: 5px; padding: 10px; margin: 10px 0; }
|
|
.skipped-toggle { cursor: pointer; background-color: #ffeaa7; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #ffc107; font-weight: bold; color: #856404; }
|
|
.skipped-toggle:hover { background-color: #ffc107; }
|
|
.skipped-details { display: none; margin-top: 10px; }
|
|
ul { list-style-type: none; padding: 0; }
|
|
li { background-color: #ffffff; margin: 2px 0; padding: 5px; border-radius: 3px; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<a href="/logout" class="logout">Logout</a>
|
|
<img src="/static/trbc-logo-full.png" alt="Logo" width="200">
|
|
<h1>Processing Jobs</h1>
|
|
<div class="nav-links" style="margin-bottom: 20px;">
|
|
<a href="/" class="nav-btn">← Upload Files</a>
|
|
<a href="/completed-jobs" class="nav-btn">View Completed Jobs Archive →</a>
|
|
</div>
|
|
{% with messages = get_flashed_messages(with_categories=true) %}
|
|
{% if messages %}
|
|
{% for category, message in messages %}
|
|
<div class="flash {{ category }}">{{ message }}</div>
|
|
{% endfor %}
|
|
{% endif %}
|
|
{% endwith %}
|
|
{% for job_id, job in jobs.items() %}
|
|
<div class="job">
|
|
<h3>Job {{ job_id[:8] }}</h3>
|
|
<p><strong>Status:</strong> <span class="status {{ job.status }}">{{ job.status.title() }}</span></p>
|
|
{% if job.status == 'processing' and job.progress %}
|
|
<p><strong>Progress:</strong> <span class="progress-text">{{ job.progress }}</span></p>
|
|
{% endif %}
|
|
<p><strong>Files:</strong> {{ job.files|length }}</p>
|
|
{% if job.zip_contents %}
|
|
<p><strong>ZIP Contents:</strong></p>
|
|
{% for zip_name, pdf_files in job.zip_contents.items() %}
|
|
<div class="zip-content">
|
|
<div class="zip-toggle" onclick="toggleZipDetails(this)">
|
|
<strong>{{ zip_name }}</strong> ({{ pdf_files|length }} files) ▶
|
|
</div>
|
|
<div class="zip-details">
|
|
<ul>
|
|
{% for pdf_file in pdf_files %}
|
|
<li>{{ pdf_file }}</li>
|
|
{% endfor %}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
{% endfor %}
|
|
{% endif %}
|
|
{% if job.status == 'completed' %}
|
|
{% if job.zip_name %}
|
|
<p><a href="/download/{{ job.zip_name }}" class="download-btn">Download ZIP</a>
|
|
<form method="post" action="/delete/{{ job_id }}" style="display: inline;">
|
|
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
|
|
</form></p>
|
|
{% else %}
|
|
<p><em>No files were successfully processed to create a ZIP archive.</em>
|
|
<form method="post" action="/delete/{{ job_id }}" style="display: inline;">
|
|
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
|
|
</form></p>
|
|
{% endif %}
|
|
{% if job.skipped %}
|
|
<div class="skipped-section">
|
|
<div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
|
|
⚠️ Skipped Files ({{ job.skipped|length }}) ▶
|
|
</div>
|
|
<div class="skipped-details">
|
|
<ul>
|
|
{% for skip_info in job.skipped %}
|
|
<li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}</li>
|
|
{% endfor %}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
{% endif %}
|
|
{% endif %}
|
|
</div>
|
|
{% endfor %}
|
|
{% if not jobs %}
|
|
<p>No jobs yet. <a href="/">Upload some files</a> to get started.</p>
|
|
{% endif %}
|
|
<script>
|
|
// Track previous job statuses to detect completion
|
|
let previousJobStatuses = {};
|
|
|
|
function updateJobs() {
|
|
fetch('/api/jobs', { credentials: 'same-origin' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
console.log('Job update received:', data);
|
|
const jobs = Object.entries(data);
|
|
|
|
// Check for job completions
|
|
let hasNewCompletion = false;
|
|
if (jobs.length > 0) {
|
|
jobs.forEach(([jobId, job]) => {
|
|
console.log(`Job ${jobId}: status=${job.status}, progress=${job.progress || 'none'}`);
|
|
|
|
// Check if job just completed
|
|
if (job.status === 'completed' && previousJobStatuses[jobId] === 'processing') {
|
|
hasNewCompletion = true;
|
|
console.log(`Job ${jobId} just completed!`);
|
|
}
|
|
|
|
// Update previous status
|
|
previousJobStatuses[jobId] = job.status;
|
|
});
|
|
}
|
|
|
|
// Redirect to completed jobs if a job just finished
|
|
if (hasNewCompletion) {
|
|
console.log('Redirecting to completed jobs page...');
|
|
setTimeout(() => {
|
|
window.location.href = '/completed-jobs';
|
|
}, 500); // Small delay to show completion briefly
|
|
return; // Stop processing to avoid UI updates during redirect
|
|
}
|
|
const jobsContainer = document.querySelector('h1').parentNode;
|
|
const existingJobs = jobsContainer.querySelectorAll('.job');
|
|
const existingNoJobsMsg = jobsContainer.querySelectorAll('p');
|
|
|
|
// Remove existing job elements
|
|
existingJobs.forEach(job => job.remove());
|
|
|
|
// Remove existing "No jobs yet" messages
|
|
existingNoJobsMsg.forEach(p => {
|
|
if (p.textContent.includes('No jobs yet')) {
|
|
p.remove();
|
|
}
|
|
});
|
|
|
|
// Add updated job elements
|
|
const jobs = Object.entries(data);
|
|
if (jobs.length > 0) {
|
|
jobs.forEach(([jobId, job]) => {
|
|
const jobDiv = document.createElement('div');
|
|
jobDiv.className = 'job';
|
|
jobDiv.innerHTML = `
|
|
<h3>Job ${jobId.substring(0, 8)}</h3>
|
|
<p><strong>Status:</strong> <span class="status ${job.status}">${job.status.charAt(0).toUpperCase() + job.status.slice(1)}</span></p>
|
|
${job.status === 'processing' && job.progress ? `<p><strong>Progress:</strong> <span class="progress-text">${job.progress}</span></p>` : ''}
|
|
<p><strong>Files:</strong> ${job.files ? job.files.length : 0}</p>
|
|
${job.zip_contents ? `
|
|
<p><strong>ZIP Contents:</strong></p>
|
|
${Object.entries(job.zip_contents).map(([zipName, pdfFiles]) => `
|
|
<div class="zip-content">
|
|
<div class="zip-toggle" onclick="toggleZipDetails(this)">
|
|
<strong>${zipName}</strong> (${pdfFiles.length} files) ▶
|
|
</div>
|
|
<div class="zip-details">
|
|
<ul>
|
|
${pdfFiles.map(pdfFile => `<li>${pdfFile}</li>`).join('')}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
`).join('')}
|
|
` : ''}
|
|
${job.status === 'completed' ? `
|
|
${job.zip_name ? `
|
|
<p><a href="/download/${job.zip_name}" class="download-btn">Download ZIP</a>
|
|
<form method="post" action="/delete/${jobId}" style="display: inline;">
|
|
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
|
|
</form></p>
|
|
` : `
|
|
<p><em>No files were successfully processed to create a ZIP archive.</em>
|
|
<form method="post" action="/delete/${jobId}" style="display: inline;">
|
|
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to delete this job?')">Delete Job</button>
|
|
</form></p>
|
|
`}
|
|
${job.skipped ? `
|
|
<div class="skipped-section">
|
|
<div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
|
|
⚠️ Skipped Files (${job.skipped.length}) ▶
|
|
</div>
|
|
<div class="skipped-details">
|
|
<ul>
|
|
${job.skipped.map(skipInfo => `<li><strong>${skipInfo[0]}</strong>: ${skipInfo[1]}</li>`).join('')}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
` : ''}
|
|
` : ''}
|
|
` : ''}
|
|
`;
|
|
jobsContainer.insertBefore(jobDiv, jobsContainer.querySelector('script') || jobsContainer.lastElementChild);
|
|
});
|
|
} else {
|
|
const noJobsMsg = document.createElement('p');
|
|
noJobsMsg.innerHTML = 'No jobs yet. <a href="/">Upload some files</a> to get started.';
|
|
jobsContainer.insertBefore(noJobsMsg, jobsContainer.querySelector('script') || jobsContainer.lastElementChild);
|
|
}
|
|
})
|
|
.catch(error => console.error('Error updating jobs:', error));
|
|
}
|
|
|
|
// Update jobs every 1 second
|
|
setInterval(updateJobs, 1000);
|
|
|
|
// Initialize previous job statuses
|
|
updateJobs(); // Run once immediately to populate initial statuses
|
|
|
|
// Collapsible section functions
|
|
function toggleZipDetails(element) {
|
|
const details = element.nextElementSibling;
|
|
const isVisible = details.style.display === 'block';
|
|
details.style.display = isVisible ? 'none' : 'block';
|
|
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
|
}
|
|
|
|
function toggleSkippedDetails(element) {
|
|
const details = element.nextElementSibling;
|
|
const isVisible = details.style.display === 'block';
|
|
details.style.display = isVisible ? 'none' : 'block';
|
|
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|
|
''', jobs=job_queue)
|
|
|
|
@app.route('/api/jobs')
|
|
@login_required
|
|
def api_jobs():
|
|
return jsonify(job_queue)
|
|
|
|
@app.route('/download/<filename>')
|
|
@login_required
|
|
def download(filename):
|
|
# Security check: only allow downloading files that belong to completed jobs
|
|
for job_id, job in job_queue.items():
|
|
if job.get('status') == 'completed' and job.get('zip_name') == filename:
|
|
file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
|
|
if os.path.exists(file_path):
|
|
return send_from_directory(app.config['UPLOAD_FOLDER'], filename, as_attachment=True)
|
|
return "File not found or access denied", 404
|
|
|
|
@app.route('/delete/<job_id>', methods=['POST'])
|
|
@login_required
|
|
def delete_job(job_id):
|
|
if job_id in job_queue:
|
|
job = job_queue[job_id]
|
|
# Only allow deletion of completed jobs
|
|
if job.get('status') == 'completed':
|
|
# Clean up the ZIP file if it exists
|
|
if job.get('zip_name'):
|
|
zip_path = os.path.join(app.config['UPLOAD_FOLDER'], job['zip_name'])
|
|
if os.path.exists(zip_path):
|
|
os.remove(zip_path)
|
|
# Remove the job from the queue
|
|
del job_queue[job_id]
|
|
save_jobs() # Persist job deletion
|
|
flash('Job deleted successfully.', 'success')
|
|
else:
|
|
flash('Only completed jobs can be deleted.', 'error')
|
|
else:
|
|
flash('Job not found.', 'error')
|
|
return redirect('/jobs')
|
|
|
|
@app.route('/completed-jobs')
|
|
@login_required
|
|
def completed_jobs():
|
|
# Filter only completed jobs
|
|
completed_jobs = {job_id: job for job_id, job in job_queue.items() if job.get('status') == 'completed'}
|
|
return render_template_string('''
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Completed Jobs Archive</title>
|
|
<style>
|
|
body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
|
|
img { margin-bottom: 20px; }
|
|
h1 { color: #333; }
|
|
.job { background-color: #fff; margin: 10px auto; padding: 15px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); max-width: 1000px; text-align: left; }
|
|
.status { padding: 5px 10px; border-radius: 5px; color: white; font-weight: bold; background-color: #28a745; }
|
|
.download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin: 5px; display: inline-block; }
|
|
.download-btn:hover { background-color: #0056b3; }
|
|
.delete-btn { background-color: #dc3545; color: white; border: none; padding: 8px 16px; border-radius: 5px; margin: 5px; cursor: pointer; }
|
|
.delete-btn:hover { background-color: #c82333; }
|
|
.flash { padding: 10px; margin: 10px 0; border-radius: 5px; text-align: center; }
|
|
.flash.success { background-color: #d4edda; color: #155724; border: 1px solid #c3e6cb; }
|
|
.flash.error { background-color: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
|
|
.logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
|
|
.logout:hover { background-color: #c82333; }
|
|
.nav-links { margin-bottom: 20px; }
|
|
.nav-links a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #f8f9fa; border-radius: 5px; margin: 0 10px; }
|
|
.nav-links a:hover { background-color: #e9ecef; }
|
|
.nav-btn { background-color: #007bff; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; margin: 0 10px; }
|
|
.nav-btn:hover { background-color: #0056b3; }
|
|
.job-stats { background-color: #e9ecef; padding: 10px; border-radius: 5px; margin-bottom: 15px; }
|
|
.file-count { color: #17a2b8; font-weight: bold; }
|
|
.zip-content { background-color: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 5px; border-left: 4px solid #007bff; }
|
|
.zip-toggle { cursor: pointer; background-color: #e9ecef; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #dee2e6; }
|
|
.zip-toggle:hover { background-color: #dee2e6; }
|
|
.zip-details { display: none; margin-top: 10px; }
|
|
.skipped-section { background-color: #fff3cd; border: 1px solid #ffeaa7; border-radius: 5px; padding: 10px; margin: 10px 0; }
|
|
.skipped-toggle { cursor: pointer; background-color: #ffeaa7; padding: 8px; margin: 5px 0; border-radius: 5px; border: 1px solid #ffc107; font-weight: bold; color: #856404; }
|
|
.skipped-toggle:hover { background-color: #ffc107; }
|
|
.skipped-details { display: none; margin-top: 10px; }
|
|
ul { list-style-type: none; padding: 0; }
|
|
li { background-color: #ffffff; margin: 2px 0; padding: 5px; border-radius: 3px; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<a href="/logout" class="logout">Logout</a>
|
|
<img src="/static/trbc-logo-full.png" alt="Logo" width="200">
|
|
<h1>Completed Jobs Archive</h1>
|
|
<div class="nav-links">
|
|
<a href="/" class="nav-btn">← Upload Files</a>
|
|
<a href="/jobs" class="nav-btn">View All Jobs →</a>
|
|
</div>
|
|
{% with messages = get_flashed_messages(with_categories=true) %}
|
|
{% if messages %}
|
|
{% for category, message in messages %}
|
|
<div class="flash {{ category }}">{{ message }}</div>
|
|
{% endfor %}
|
|
{% endif %}
|
|
{% endwith %}
|
|
{% for job_id, job in jobs.items() %}
|
|
<div class="job">
|
|
<h3>Job {{ job_id[:8] }} - {{ job.created_at if job.created_at else 'Unknown Date' }}</h3>
|
|
<div class="job-stats">
|
|
<span class="status">Completed</span>
|
|
<span class="file-count">{{ job.processed_files|selectattr('0', 'equalto', 'success')|list|length }} files processed</span>
|
|
{% if job.skipped %}
|
|
<span style="color: #856404; font-weight: bold;">{{ job.skipped|length }} files skipped</span>
|
|
{% endif %}
|
|
</div>
|
|
|
|
{% if job.zip_contents %}
|
|
<p><strong>Original ZIP Contents:</strong></p>
|
|
{% for zip_name, pdf_files in job.zip_contents.items() %}
|
|
<div class="zip-content">
|
|
<div class="zip-toggle" onclick="toggleZipDetails(this)">
|
|
<strong>{{ zip_name }}</strong> ({{ pdf_files|length }} files) ▶
|
|
</div>
|
|
<div class="zip-details">
|
|
<ul>
|
|
{% for pdf_file in pdf_files %}
|
|
<li>{{ pdf_file }}</li>
|
|
{% endfor %}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
{% endfor %}
|
|
{% endif %}
|
|
|
|
{% if job.zip_name %}
|
|
<p><strong>Download:</strong> <a href="/download/{{ job.zip_name }}" class="download-btn">📁 Download Organized ZIP ({{ job.zip_name }})</a></p>
|
|
{% endif %}
|
|
|
|
{% if job.skipped %}
|
|
<div class="skipped-section">
|
|
<div class="skipped-toggle" onclick="toggleSkippedDetails(this)">
|
|
⚠️ Skipped Files ({{ job.skipped|length }}) ▶
|
|
</div>
|
|
<div class="skipped-details">
|
|
<ul>
|
|
{% for skip_info in job.skipped %}
|
|
<li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}</li>
|
|
{% endfor %}
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
{% endif %}
|
|
|
|
<form method="post" action="/delete/{{ job_id }}" style="display: inline;">
|
|
<button type="submit" class="delete-btn" onclick="return confirm('Are you sure you want to permanently delete this completed job and its files?')">🗑️ Delete Job</button>
|
|
</form>
|
|
</div>
|
|
{% endfor %}
|
|
{% if not jobs %}
|
|
<p>No completed jobs yet. <a href="/">Upload some files</a> to get started.</p>
|
|
{% endif %}
|
|
<script>
|
|
// Collapsible section functions
|
|
function toggleZipDetails(element) {
|
|
const details = element.nextElementSibling;
|
|
const isVisible = details.style.display === 'block';
|
|
details.style.display = isVisible ? 'none' : 'block';
|
|
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
|
}
|
|
|
|
function toggleSkippedDetails(element) {
|
|
const details = element.nextElementSibling;
|
|
const isVisible = details.style.display === 'block';
|
|
details.style.display = isVisible ? 'none' : 'block';
|
|
element.innerHTML = element.innerHTML.replace(isVisible ? ' ▼' : ' ▶', isVisible ? ' ▶' : ' ▼');
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|
|
''', jobs=completed_jobs)
|
|
|
|
if __name__ == '__main__':
|
|
app.run(debug=True, port=3000) |