nate/nextgenpdf
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve PDF renamer: better extraction, skip handling, and logging for failed PDFs

Browse Source
This commit is contained in:
Nate Emmert committed 2026-02-02 07:56:58 -05:00
1 parent 34d8778bc4
commit a1454bf966
2 files changed
+4959 -4945
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

jobs.json
+4755 -4735
View File
File diff suppressed because it is too large. Load diff
pdf_renamer.py
+204 -210
View File
@@ -427,6 +427,7 @@ def extract_names_from_pdf(pdf_path):
reader = PdfReader(pdf_path)
# All PDFs are flattened, so only try text extraction from first page
if len(reader.pages) > 0:
first_page_text = reader.pages[0].extract_text()
extracted_data = extract_student_info_from_text(first_page_text)
@@ -440,22 +441,13 @@ def extract_names_from_pdf(pdf_path):
'school': extracted_data['school'],
'gender': extracted_data['gender']
}
# If we get here, we couldn't extract the data
# Log the failed extraction for debugging
import os
failed_log_path = os.path.join(os.path.dirname(pdf_path), 'failed_extractions.log')
try:
with open(failed_log_path, 'a', encoding='utf-8') as f:
f.write(f"Failed to extract from: {os.path.basename(pdf_path)}\n")
if len(reader.pages) > 0:
text = reader.pages[0].extract_text()
f.write(f"Text content:\n{text}\n")
f.write("-" * 50 + "\n")
except Exception as log_error:
pass # Don't let logging errors break the main flow
return {'error': 'Could not extract student information from PDF. Text extraction failed.'}
# If extraction failed, include raw text for logging
return {
'error': 'Could not extract student information from PDF. Text extraction failed.',
'raw_text': first_page_text or ''
}
# If we get here, we couldn't extract the data or text
return {'error': 'Could not extract student information from PDF. No text found.'}
except Exception as e:
return {'error': f'PDF read error: {str(e)}'}
@@ -468,12 +460,7 @@ def extract_names_from_pdf(pdf_path):
pass
def extract_student_info_from_text(text):
"""
Extract student information from flattened PDF text.
This function handles multiple concatenation patterns and edge cases.
"""
import re
extracted = {
'first_name': '',
'last_name': '',
@@ -481,10 +468,48 @@ def extract_student_info_from_text(text):
'school': '',
'gender': ''
}
# Split text into lines for better parsing
lines = text.split('\n')
# --- Custom patterns for failed extractions ---
# Pattern: GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
for i in range(len(lines) - 1):
m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', lines[i])
m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', lines[i+1])
if m1 and m2:
extracted['gender'] = m1.group(1)
extracted['school'] = m1.group(2).strip()
extracted['last_name'] = m1.group(3)
extracted['grade'] = m2.group(1)
extracted['first_name'] = m2.group(2)
break
# Pattern: ConcatenatedNames, Grade, School, Gender (e.g., MaynardJenny, 11, Liberty Christian Academy, Female)
for i in range(len(lines) - 3):
m1 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i])
m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
m3 = lines[i+2].strip()
m4 = re.match(r'^(Female|Male)$', lines[i+3])
if m1 and m2 and m3 and m4:
extracted['last_name'] = m1.group(1)
extracted['first_name'] = m1.group(2)
extracted['grade'] = m2.group(1)
extracted['school'] = m3
extracted['gender'] = m4.group(1)
break
# Pattern: SchoolFirstName, LastName, GenderGrade (e.g., Liberty Christian AcademySally, Sue, Female9)
for i in range(len(lines) - 2):
m1 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i])
m2 = re.match(r'^([A-Z][a-z]+)$', lines[i+1])
m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2])
if m1 and m2 and m3:
extracted['school'] = m1.group(1).strip()
extracted['first_name'] = m1.group(2)
extracted['last_name'] = m2.group(1)
extracted['gender'] = m3.group(1)
extracted['grade'] = m3.group(2)
break
# Enhanced preprocessing: aggressively split lines with multiple fields stuck together
processed_lines = []
skip_headers = [
@@ -492,36 +517,30 @@ def extract_student_info_from_text(text):
'student school:', 'details', 'gender:', 'docusign envelope id:',
'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian'
]
school_suffixes = [
'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
valid_schools = [
'Amherst', 'Appomattox', 'Brookville', 'Dunbar', 'E.C. Glass', 'Forest', 'Heritage', 'Home School',
'Jefferson Forest', 'Liberty Christian Academy', 'Linkhorne', 'Monelison', 'Rustburg', 'Sandusky',
'Temple Christian', 'Timberlake Christian'
]
school_suffixes = [' High', ' Middle', ' Academy', ' Elementary', ' School', ' Christian']
all_school_names = sorted([base + suffix for base in valid_schools for suffix in [''] + school_suffixes], key=len, reverse=True)
for line in lines:
orig_line = line.strip()
# Ignore header/title/label lines
if not orig_line or any(h in orig_line.lower() for h in skip_headers):
continue
# Special: Handle lines like '6HoskinsMary Emma' (grade, last name, first name) ONLY if the line is exactly that pattern
m_special = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)([A-Z][a-z]+\s[A-Z][a-z]+)$', orig_line)
if m_special and orig_line == m_special.group(0):
processed_lines.append(m_special.group(1))
processed_lines.append(m_special.group(2))
# Split first and last name if possible
names = m_special.group(3).strip().split()
if len(names) == 2:
processed_lines.append(names[0])
processed_lines.append(names[1])
else:
processed_lines.append(m_special.group(3).strip())
continue
# Special: Handle lines like 'FemaleNew Covenant' (gender, school) ONLY if the line is exactly that pattern
m_special2 = re.match(r'^(Female|Male)([A-Z][a-z]+ [A-Z][a-z]+.*)$', orig_line)
if m_special2 and orig_line == m_special2.group(0):
processed_lines.append(m_special2.group(1))
processed_lines.append(m_special2.group(2).strip())
continue
# Never treat a number or 'K' as a school
if re.fullmatch(r'(\d{1,2}|K)', orig_line):
processed_lines.append(orig_line)
# Aggressive split: if line starts with a valid school name (with suffix) and is immediately followed by a capitalized name or digit, split
split_done = False
for school_name in all_school_names:
if orig_line.startswith(school_name):
rest = orig_line[len(school_name):]
if rest and (re.match(r'^[A-Z]', rest) or re.match(r'^\d', rest)):
processed_lines.append(school_name)
processed_lines.append(rest)
split_done = True
break
if split_done:
continue
# --- Always split off leading gender ---
m_gender = re.match(r'^(Female|Male)(.+)', orig_line)
@@ -545,9 +564,6 @@ def extract_student_info_from_text(text):
for part in re.findall(r"[\w'\-]+", trailing):
processed_lines.append(part)
continue
# Otherwise, keep line as is
processed_lines.append(orig_line)
# ...existing code for other patterns...
# Pattern: Gender + School + LastName (e.g., 'FemaleBrookville HighMouse')
m = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)$', orig_line)
if m:
@@ -588,7 +604,7 @@ def extract_student_info_from_text(text):
processed_lines.append(m5.group(1))
processed_lines.append(m5.group(2).strip())
processed_lines.append(m5.group(3))
processed_lines.append(m5.group(4))
processed_lines.append(m4.group(4))
continue
# Pattern: Grade + Gender + School + FirstName + LastName (e.g., '10FemaleHomeSchoolAbigailDober')
m6 = re.match(r'^(\d{1,2}|K)(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
@@ -638,207 +654,154 @@ def extract_student_info_from_text(text):
{
'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$',
'map': map_pattern_1
'map': lambda m1, m2: {
'gender': m1.group(1),
'school': m1.group(2).strip(),
'last_name': m1.group(3),
'grade': m2.group(1),
'first_name': m2.group(2)
}
},
# Pattern 2: FirstName, Gender + School + LastName, Grade
{
'regex1': r'^([A-Z][a-z]+)$',
'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
'regex3': r'^(\d{1,2}|K)$',
'map': map_pattern_2
'map': lambda m1, m2, m3: {
'first_name': m1.group(1),
'gender': m2.group(1),
'school': m2.group(2).strip(),
'last_name': m2.group(3),
'grade': m3.group(1)
}
},
# Pattern 3: Gender + FirstName, School + LastName, Grade
{
'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
'regex2': r'^(.+?)([A-Z][a-z]+)$',
'regex3': r'^(\d{1,2}|K)$',
'map': map_pattern_3
'map': lambda m1, m2, m3: {
'gender': m1.group(1),
'first_name': m1.group(2),
'school': m2.group(1).strip(),
'last_name': m2.group(2),
'grade': m3.group(1)
}
},
# Pattern 4: School + Grade, Gender + FirstName, LastName
{
'regex1': r'^(.+?)(\d{1,2}|K)$',
'regex2': r'^(Female|Male)([A-Z][a-z]+)$',
'regex3': r'^([A-Z][a-z]+)$',
'map': map_pattern_4
'map': lambda m1, m2, m3: {
'school': m1.group(1).strip(),
'grade': m1.group(2),
'gender': m2.group(1),
'first_name': m2.group(2),
'last_name': m3.group(1)
}
},
# Pattern 5: FirstName, Gender + Grade, School + LastName
{
'regex1': r'^([A-Z][a-z]+)$',
'regex2': r'^(Female|Male)(\d{1,2}|K)$',
'regex3': r'^(.+?)([A-Z][a-z]+)$',
'map': map_pattern_5
'map': lambda m1, m2, m3: {
'first_name': m1.group(1),
'gender': m2.group(1),
'grade': m2.group(2),
'school': m3.group(1).strip(),
'last_name': m3.group(2)
}
},
# Pattern 6: FirstName, School + Grade + LastName, Gender
{
'regex1': r'^([A-Z][a-z]+)$',
'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
'regex3': r'^(Female|Male)$',
'map': map_pattern_6
'map': lambda m1, m2, m3: {
'first_name': m1.group(1),
'school': m2.group(1).strip(),
'grade': m2.group(2),
'last_name': m2.group(3),
'gender': m3.group(1)
}
},
# Pattern 7: Gender + FirstName, School + Grade + LastName
{
'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
'map': map_pattern_7
},
# Pattern 8: School + FirstName, LastName, Gender + Grade
{
'regex1': r'^(.+?)([A-Z][a-z]+)$',
'regex2': r'^([A-Z][a-z]+)$',
'regex3': r'^(Female|Male)(\d{1,2}|K)$',
'map': map_pattern_8
},
# Pattern 9: FirstLast (concatenated), Grade, School, Gender
{
'regex1': r'^([A-Z][a-z]+)([A-Z][a-z]+)$',
'regex2': r'^(\d{1,2}|K)$',
'regex3': r'^(.+)$',
'regex4': r'^(Female|Male)$',
'map': map_pattern_9
},
# Pattern 10: Gender + School + LastName, Grade + FirstName
{
'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)',
'map': map_pattern_10
'map': lambda m1, m2: {
'gender': m1.group(1),
'first_name': m1.group(2),
'school': m2.group(1).strip(),
'grade': m2.group(2),
'last_name': m2.group(3)
}
},
]
# Try each pattern
for pattern in patterns:
if all(extracted.values()): # Already found all fields
break
for i, line in enumerate(lines):
line = line.strip()
if not line:
continue
match1 = re.match(pattern['regex1'], line)
if not match1:
continue
mapped = None
if 'regex4' in pattern and i + 3 < len(lines):
# Try 4-line pattern
match2 = re.match(pattern['regex2'], lines[i + 1].strip())
match3 = re.match(pattern['regex3'], lines[i + 2].strip())
match4 = re.match(pattern['regex4'], lines[i + 3].strip())
if match2 and match3 and match4:
mapped = pattern['map'](match1, match2, match3, match4)
elif 'regex3' in pattern and i + 2 < len(lines):
# Try 3-line pattern
match2 = re.match(pattern['regex2'], lines[i + 1].strip())
match3 = re.match(pattern['regex3'], lines[i + 2].strip())
if match2 and match3:
mapped = pattern['map'](match1, match2, match3)
elif 'regex2' in pattern and i + 1 < len(lines):
# Try 2-line pattern
match2 = re.match(pattern['regex2'], lines[i + 1].strip())
if match2:
if 'regex3' in pattern or 'regex4' in pattern:
# This pattern expects more matches but only has 2 lines available
continue
mapped = pattern['map'](match1, match2)
if mapped:
if 'regex3' in pattern:
for i in range(len(lines) - 2):
m1 = re.match(pattern['regex1'], lines[i])
m2 = re.match(pattern['regex2'], lines[i+1])
m3 = re.match(pattern['regex3'], lines[i+2])
if m1 and m2 and m3:
mapped = pattern['map'](m1, m2, m3)
extracted.update(mapped)
# Fourth pass: Try to extract from sequential lines (for cases like: NameName, Grade, School, Gender)
if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
for i in range(len(lines) - 3):
# Look for pattern: NameName, Grade, School, Gender
name_line = lines[i]
grade_line = lines[i+1]
school_line = lines[i+2]
gender_line = lines[i+3]
# NameName: two capitalized names concatenated
m = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', name_line)
if m and re.match(r'^(\d{1,2}|K)$', grade_line) and len(school_line) > 2 and gender_line in ['Male', 'Female']:
extracted['first_name'] = m.group(1)
extracted['last_name'] = m.group(2)
extracted['grade'] = grade_line
extracted['school'] = school_line
extracted['gender'] = gender_line
break
# Fifth pass: Try to extract from GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
else:
for i in range(len(lines) - 1):
line1 = lines[i]
line2 = lines[i+1]
m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', line1)
m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', line2)
m1 = re.match(pattern['regex1'], lines[i])
m2 = re.match(pattern['regex2'], lines[i+1])
if m1 and m2:
extracted['gender'] = m1.group(1)
extracted['school'] = m1.group(2).strip()
extracted['last_name'] = m1.group(3)
extracted['grade'] = m2.group(1)
extracted['first_name'] = m2.group(2)
mapped = pattern['map'](m1, m2)
extracted.update(mapped)
break
if all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
break
# Final validation and cleanup
# Post-processing and validation
required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
if all(extracted.get(field) for field in required_fields):
# Clean up school names (remove extra spaces, standardize)
extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
# --- Improved post-processing for stuck-together school/grade/last name ---
school_suffixes = [
'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
]
# Find the last occurrence of any school suffix and split there
import re
school = extracted['school']
last_pos = -1
last_suffix = ''
for suffix in school_suffixes:
idx = school.rfind(suffix)
if idx != -1 and idx + len(suffix) > last_pos:
last_pos = idx + len(suffix)
last_suffix = suffix
if last_pos != -1:
# School name is everything up to and including the suffix
extracted['school'] = school[:last_pos].strip()
trailing = school[last_pos:].strip().replace(' ', '')
# Regex: (grade?)(all capitalized names?)(anything else?)
m = re.match(r'^(\d{1,2}|K)?((?:[A-Z][a-z]+)+)(.*)$', trailing)
if m:
grade = m.group(1)
names_str = m.group(2)
leftover = m.group(3)
if grade and not extracted['grade']:
extracted['grade'] = grade
# Split names_str into tokens of capitalized names
names = re.findall(r'[A-Z][a-z]+', names_str)
if names:
extracted['last_name'] = names[0]
if len(names) > 1:
extracted['first_name'] = ' '.join(names[1:])
# If there is leftover text, append to first name
if leftover:
if extracted['first_name']:
extracted['first_name'] += leftover
else:
extracted['first_name'] = leftover
# Clean up school name: remove any trailing capitalized names or numbers
extracted['school'] = re.sub(r'([A-Z][a-z]+|\d+)$', '', extracted['school']).strip()
# Remove any names from school if they match first/last name
for n in [extracted['first_name'], extracted['last_name']]:
if n:
extracted['school'] = re.sub(rf'\b{re.escape(n)}\b', '', extracted['school']).strip()
# Final check: if school is just a number or a name, discard it
if re.fullmatch(r'(\d{1,2}|K)', extracted['school']) or re.fullmatch(r'([A-Z][a-z]+|[A-Z\-]+|[A-Z][a-z]+\s[A-Z][a-z]+)+', extracted['school']):
extracted['school'] = ''
extracted['school'] = extracted['school'].strip()
return extracted
# Fallback: Try to extract fields from any order in the lines (for highly concatenated or out-of-order data)
# Look for all capitalized words (possible names)
forbidden_names = set(['School', 'Home', 'High', 'Middle', 'Academy', 'Christian', 'Temple', 'Rustburg', 'Liberty', 'Forest', 'Heritage', 'Amherst', 'Appomattox', 'Dunbar', 'Glass', 'Jefferson', 'Linkhorne', 'Monelison', 'Sandusky', 'Timberlake'])
gender_words = set(['Male', 'Female'])
grade_words = set([str(i) for i in range(1,13)] + ['K'])
school_name = extracted['school'].lower()
names = []
for line in lines:
for word in line.split():
if re.match(r'^[A-Z][a-z]+$', word):
w_lower = word.lower()
# Exclude if word matches or contains any part of the school name
if (re.match(r'^[A-Z][a-z]+$', word)
and word not in forbidden_names
and word not in gender_words
and word not in grade_words
and school_name not in w_lower
and w_lower not in school_name):
names.append(word)
if len(names) >= 2:
# Require two distinct names
if len(names) >= 2 and names[0] != names[1]:
if not extracted['first_name']:
extracted['first_name'] = names[0]
if not extracted['last_name']:
extracted['last_name'] = names[1]
else:
return None
# Look for grade
for line in lines:
m = re.search(r'(\d{1,2}|K)', line)
@@ -869,11 +832,13 @@ def process_job(job_id, files):
job_queue[job_id]['status'] = 'processing'
job_queue[job_id]['progress'] = 'Starting job processing...'
save_jobs() # Persist initial progress
processed_files = []
# Create base Episode directory for this job
episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'Episode')
os.makedirs(episode_dir, exist_ok=True)
processed_files = []
skipped_dir = os.path.join(episode_dir, 'Skipped')
os.makedirs(skipped_dir, exist_ok=True)
skipped_log_path = os.path.join(skipped_dir, 'skipped_files.log')
job_queue[job_id]['progress'] = 'Created output directory structure'
# Analyze ZIP contents before processing
@@ -931,10 +896,27 @@ def process_job(job_id, files):
save_jobs() # Persist current file progress
print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
# Skip MacOS resource fork files (._*)
if f.startswith('._'):
print(f"DEBUG: Skipping MacOS resource fork file: {f}")
continue
pdf_info = extract_names_from_pdf(pdf_path)
print(f"DEBUG: PDF extraction result: {pdf_info}")
if 'error' in pdf_info:
processed_files.append(('skipped', f, pdf_info['error']))
# Move skipped file to Skipped/
skipped_pdf_path = os.path.join(skipped_dir, f)
try:
os.rename(pdf_path, skipped_pdf_path)
except Exception as move_err:
print(f"WARNING: Could not move skipped PDF {f}: {move_err}")
# Log skip reason and raw text if available
with open(skipped_log_path, 'a', encoding='utf-8') as slog:
slog.write(f"File: {f}\nReason: {pdf_info['error']}\n")
if 'raw_text' in pdf_info:
slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
slog.write("-"*50 + "\n")
# Store reason and raw_text for web GUI
processed_files.append(('skipped', f, pdf_info['error'], pdf_info.get('raw_text', '')))
job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
else:
@@ -979,9 +961,25 @@ def process_job(job_id, files):
try:
job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
save_jobs() # Persist current file progress
# Skip MacOS resource fork files (._*)
if filename.startswith('._'):
print(f"DEBUG: Skipping MacOS resource fork file: {filename}")
continue
pdf_info = extract_names_from_pdf(filepath)
if 'error' in pdf_info:
processed_files.append(('skipped', filename, pdf_info['error']))
# Move skipped file to Skipped/
skipped_pdf_path = os.path.join(skipped_dir, filename)
try:
os.rename(filepath, skipped_pdf_path)
except Exception as move_err:
print(f"WARNING: Could not move skipped PDF {filename}: {move_err}")
# Log skip reason and raw text if available
with open(skipped_log_path, 'a', encoding='utf-8') as slog:
slog.write(f"File: {filename}\nReason: {pdf_info['error']}\n")
if 'raw_text' in pdf_info:
slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
slog.write("-"*50 + "\n")
processed_files.append(('skipped', filename, pdf_info['error'], pdf_info.get('raw_text', '')))
job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
else:
job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
@@ -1013,28 +1011,24 @@ def process_job(job_id, files):
job_queue[job_id]['progress'] = 'Creating ZIP archive...'
# Create ZIP of the entire Episode directory structure
successful_files = [p for p in processed_files if p[0] == 'success']
print(f"DEBUG: Starting ZIP creation, successful_files: {len(successful_files)}")
if successful_files:
# Always create ZIP, including Skipped/ folder if present
try:
# Create ZIP in archives folder
zip_output = os.path.join(app.config['ARCHIVE_FOLDER'], f'renamed_{job_id}.zip')
print(f"DEBUG: Creating ZIP at {zip_output}")
with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
for root, dirs, files_in_dir in os.walk(episode_dir):
for file in files_in_dir:
if file.startswith('.'):
continue # Ignore dotfiles
if file.startswith('.') or file.startswith('._'):
continue # Ignore dotfiles and MacOS resource forks
file_path = os.path.join(root, file)
arcname = os.path.relpath(file_path, episode_dir)
print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
zip_out.write(file_path, arcname)
job_queue[job_id]['zip_path'] = zip_output
job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
job_queue[job_id]['progress'] = f'Created ZIP archive with {len([p for p in processed_files if p[0] == 'success'])} organized files and {len([p for p in processed_files if p[0] == 'skipped'])} skipped files'
save_jobs() # Persist completion progress
print(f"DEBUG: ZIP creation completed successfully")
# Clean up Episode directory after successful ZIP creation
try:
import shutil
@@ -1042,7 +1036,6 @@ def process_job(job_id, files):
print(f"DEBUG: Cleaned up Episode directory: {episode_dir}")
except Exception as cleanup_error:
print(f"WARNING: Failed to clean up Episode directory: {cleanup_error}")
# Clean up original uploaded files
for file_info in files:
try:
@@ -1051,20 +1044,17 @@ def process_job(job_id, files):
print(f"DEBUG: Removed uploaded file: {file_info['path']}")
except Exception as file_cleanup_error:
print(f"WARNING: Failed to remove uploaded file: {file_cleanup_error}")
except Exception as e:
job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
job_queue[job_id]['status'] = 'error'
save_jobs() # Persist error status
print(f"DEBUG: ZIP creation failed: {str(e)}")
else:
job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
print(f"DEBUG: No successful files to archive")
with job_lock:
job_queue[job_id]['status'] = 'completed'
job_queue[job_id]['processed_files'] = processed_files
job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
# Store skipped as (filename, reason, raw_text)
job_queue[job_id]['skipped'] = [(p[1], p[2], p[3]) for p in processed_files if p[0] == 'skipped']
save_jobs() # Persist completed job
print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")
@@ -1870,7 +1860,11 @@ def completed_jobs():
<div class="skipped-details">
<ul>
{% for skip_info in job.skipped %}
<li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}</li>
<li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}
{% if skip_info|length > 2 and skip_info[2] %}
<br><details><summary>Extracted Text</summary><pre style="white-space: pre-wrap; background: #f8f9fa; border: 1px solid #eee; padding: 5px;">{{ skip_info[2] }}</pre></details>
{% endif %}
</li>
{% endfor %}
</ul>
</div>