Improve PDF renamer: better extraction, skip handling, and logging for failed PDFs

pdf_renamer.py

@@ -427,6 +427,7 @@ def extract_names_from_pdf(pdf_path):
         reader = PdfReader(pdf_path)
 
         # All PDFs are flattened, so only try text extraction from first page
+
         if len(reader.pages) > 0:
             first_page_text = reader.pages[0].extract_text()
             extracted_data = extract_student_info_from_text(first_page_text)
@@ -440,22 +441,13 @@ def extract_names_from_pdf(pdf_path):
                     'school': extracted_data['school'],
                     'gender': extracted_data['gender']
                 }
-
+            # If extraction failed, include raw text for logging
-        # If we get here, we couldn't extract the data
+            return {
-        # Log the failed extraction for debugging
+                'error': 'Could not extract student information from PDF. Text extraction failed.',
-        import os
+                'raw_text': first_page_text or ''
-        failed_log_path = os.path.join(os.path.dirname(pdf_path), 'failed_extractions.log')
+            }
-        try:
+        # If we get here, we couldn't extract the data or text
-            with open(failed_log_path, 'a', encoding='utf-8') as f:
+        return {'error': 'Could not extract student information from PDF. No text found.'}
-                f.write(f"Failed to extract from: {os.path.basename(pdf_path)}\n")
-                if len(reader.pages) > 0:
-                    text = reader.pages[0].extract_text()
-                    f.write(f"Text content:\n{text}\n")
-                    f.write("-" * 50 + "\n")
-        except Exception as log_error:
-            pass  # Don't let logging errors break the main flow
-
-        return {'error': 'Could not extract student information from PDF. Text extraction failed.'}
 
     except Exception as e:
         return {'error': f'PDF read error: {str(e)}'}
@@ -468,12 +460,7 @@ def extract_names_from_pdf(pdf_path):
                 pass
 
 def extract_student_info_from_text(text):
-    """
-    Extract student information from flattened PDF text.
-    This function handles multiple concatenation patterns and edge cases.
-    """
     import re
-
     extracted = {
         'first_name': '',
         'last_name': '',
@@ -481,9 +468,47 @@ def extract_student_info_from_text(text):
         'school': '',
         'gender': ''
     }
-
-    # Split text into lines for better parsing
     lines = text.split('\n')
+    
+    # --- Custom patterns for failed extractions ---
+    # Pattern: GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
+    for i in range(len(lines) - 1):
+        m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', lines[i])
+        m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', lines[i+1])
+        if m1 and m2:
+            extracted['gender'] = m1.group(1)
+            extracted['school'] = m1.group(2).strip()
+            extracted['last_name'] = m1.group(3)
+            extracted['grade'] = m2.group(1)
+            extracted['first_name'] = m2.group(2)
+            break
+
+    # Pattern: ConcatenatedNames, Grade, School, Gender (e.g., MaynardJenny, 11, Liberty Christian Academy, Female)
+    for i in range(len(lines) - 3):
+        m1 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', lines[i])
+        m2 = re.match(r'^(\d{1,2}|K)$', lines[i+1])
+        m3 = lines[i+2].strip()
+        m4 = re.match(r'^(Female|Male)$', lines[i+3])
+        if m1 and m2 and m3 and m4:
+            extracted['last_name'] = m1.group(1)
+            extracted['first_name'] = m1.group(2)
+            extracted['grade'] = m2.group(1)
+            extracted['school'] = m3
+            extracted['gender'] = m4.group(1)
+            break
+
+    # Pattern: SchoolFirstName, LastName, GenderGrade (e.g., Liberty Christian AcademySally, Sue, Female9)
+    for i in range(len(lines) - 2):
+        m1 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i])
+        m2 = re.match(r'^([A-Z][a-z]+)$', lines[i+1])
+        m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2])
+        if m1 and m2 and m3:
+            extracted['school'] = m1.group(1).strip()
+            extracted['first_name'] = m1.group(2)
+            extracted['last_name'] = m2.group(1)
+            extracted['gender'] = m3.group(1)
+            extracted['grade'] = m3.group(2)
+            break
 
     # Enhanced preprocessing: aggressively split lines with multiple fields stuck together
     processed_lines = []
@@ -492,36 +517,30 @@ def extract_student_info_from_text(text):
         'student school:', 'details', 'gender:', 'docusign envelope id:',
         'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian'
     ]
-    school_suffixes = [
+    valid_schools = [
-        'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
+        'Amherst', 'Appomattox', 'Brookville', 'Dunbar', 'E.C. Glass', 'Forest', 'Heritage', 'Home School',
+        'Jefferson Forest', 'Liberty Christian Academy', 'Linkhorne', 'Monelison', 'Rustburg', 'Sandusky',
+        'Temple Christian', 'Timberlake Christian'
     ]
+    school_suffixes = [' High', ' Middle', ' Academy', ' Elementary', ' School', ' Christian']
+    all_school_names = sorted([base + suffix for base in valid_schools for suffix in [''] + school_suffixes], key=len, reverse=True)
+    
     for line in lines:
         orig_line = line.strip()
         # Ignore header/title/label lines
         if not orig_line or any(h in orig_line.lower() for h in skip_headers):
             continue
-        # Special: Handle lines like '6HoskinsMary Emma' (grade, last name, first name) ONLY if the line is exactly that pattern
+        # Aggressive split: if line starts with a valid school name (with suffix) and is immediately followed by a capitalized name or digit, split
-        m_special = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)([A-Z][a-z]+\s[A-Z][a-z]+)$', orig_line)
+        split_done = False
-        if m_special and orig_line == m_special.group(0):
+        for school_name in all_school_names:
-            processed_lines.append(m_special.group(1))
+            if orig_line.startswith(school_name):
-            processed_lines.append(m_special.group(2))
+                rest = orig_line[len(school_name):]
-            # Split first and last name if possible
+                if rest and (re.match(r'^[A-Z]', rest) or re.match(r'^\d', rest)):
-            names = m_special.group(3).strip().split()
+                    processed_lines.append(school_name)
-            if len(names) == 2:
+                    processed_lines.append(rest)
-                processed_lines.append(names[0])
+                    split_done = True
-                processed_lines.append(names[1])
+                    break
-            else:
+        if split_done:
-                processed_lines.append(m_special.group(3).strip())
-            continue
-        # Special: Handle lines like 'FemaleNew Covenant' (gender, school) ONLY if the line is exactly that pattern
-        m_special2 = re.match(r'^(Female|Male)([A-Z][a-z]+ [A-Z][a-z]+.*)$', orig_line)
-        if m_special2 and orig_line == m_special2.group(0):
-            processed_lines.append(m_special2.group(1))
-            processed_lines.append(m_special2.group(2).strip())
-            continue
-        # Never treat a number or 'K' as a school
-        if re.fullmatch(r'(\d{1,2}|K)', orig_line):
-            processed_lines.append(orig_line)
             continue
         # --- Always split off leading gender ---
         m_gender = re.match(r'^(Female|Male)(.+)', orig_line)
@@ -545,9 +564,6 @@ def extract_student_info_from_text(text):
                 for part in re.findall(r"[\w'\-]+", trailing):
                     processed_lines.append(part)
             continue
-        # Otherwise, keep line as is
-        processed_lines.append(orig_line)
-        # ...existing code for other patterns...
         # Pattern: Gender + School + LastName (e.g., 'FemaleBrookville HighMouse')
         m = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)$', orig_line)
         if m:
@@ -588,7 +604,7 @@ def extract_student_info_from_text(text):
             processed_lines.append(m5.group(1))
             processed_lines.append(m5.group(2).strip())
             processed_lines.append(m5.group(3))
-            processed_lines.append(m5.group(4))
+            processed_lines.append(m4.group(4))
             continue
         # Pattern: Grade + Gender + School + FirstName + LastName (e.g., '10FemaleHomeSchoolAbigailDober')
         m6 = re.match(r'^(\d{1,2}|K)(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
@@ -638,207 +654,154 @@ def extract_student_info_from_text(text):
         {
             'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
             'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)$',
-            'map': map_pattern_1
+            'map': lambda m1, m2: {
+                'gender': m1.group(1),
+                'school': m1.group(2).strip(),
+                'last_name': m1.group(3),
+                'grade': m2.group(1),
+                'first_name': m2.group(2)
+            }
         },
         # Pattern 2: FirstName, Gender + School + LastName, Grade
         {
             'regex1': r'^([A-Z][a-z]+)$',
             'regex2': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
             'regex3': r'^(\d{1,2}|K)$',
-            'map': map_pattern_2
+            'map': lambda m1, m2, m3: {
+                'first_name': m1.group(1),
+                'gender': m2.group(1),
+                'school': m2.group(2).strip(),
+                'last_name': m2.group(3),
+                'grade': m3.group(1)
+            }
         },
         # Pattern 3: Gender + FirstName, School + LastName, Grade
         {
             'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
             'regex2': r'^(.+?)([A-Z][a-z]+)$',
             'regex3': r'^(\d{1,2}|K)$',
-            'map': map_pattern_3
+            'map': lambda m1, m2, m3: {
+                'gender': m1.group(1),
+                'first_name': m1.group(2),
+                'school': m2.group(1).strip(),
+                'last_name': m2.group(2),
+                'grade': m3.group(1)
+            }
         },
         # Pattern 4: School + Grade, Gender + FirstName, LastName
         {
             'regex1': r'^(.+?)(\d{1,2}|K)$',
             'regex2': r'^(Female|Male)([A-Z][a-z]+)$',
             'regex3': r'^([A-Z][a-z]+)$',
-            'map': map_pattern_4
+            'map': lambda m1, m2, m3: {
+                'school': m1.group(1).strip(),
+                'grade': m1.group(2),
+                'gender': m2.group(1),
+                'first_name': m2.group(2),
+                'last_name': m3.group(1)
+            }
         },
         # Pattern 5: FirstName, Gender + Grade, School + LastName
         {
             'regex1': r'^([A-Z][a-z]+)$',
             'regex2': r'^(Female|Male)(\d{1,2}|K)$',
             'regex3': r'^(.+?)([A-Z][a-z]+)$',
-            'map': map_pattern_5
+            'map': lambda m1, m2, m3: {
+                'first_name': m1.group(1),
+                'gender': m2.group(1),
+                'grade': m2.group(2),
+                'school': m3.group(1).strip(),
+                'last_name': m3.group(2)
+            }
         },
         # Pattern 6: FirstName, School + Grade + LastName, Gender
         {
             'regex1': r'^([A-Z][a-z]+)$',
             'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
             'regex3': r'^(Female|Male)$',
-            'map': map_pattern_6
+            'map': lambda m1, m2, m3: {
+                'first_name': m1.group(1),
+                'school': m2.group(1).strip(),
+                'grade': m2.group(2),
+                'last_name': m2.group(3),
+                'gender': m3.group(1)
+            }
         },
         # Pattern 7: Gender + FirstName, School + Grade + LastName
         {
             'regex1': r'^(Female|Male)([A-Z][a-z]+)$',
             'regex2': r'^(.+?)(\d{1,2}|K)([A-Z][a-z]+)$',
-            'map': map_pattern_7
+            'map': lambda m1, m2: {
+                'gender': m1.group(1),
+                'first_name': m1.group(2),
+                'school': m2.group(1).strip(),
+                'grade': m2.group(2),
+                'last_name': m2.group(3)
+            }
         },
-        # Pattern 8: School + FirstName, LastName, Gender + Grade
-        {
-            'regex1': r'^(.+?)([A-Z][a-z]+)$',
-            'regex2': r'^([A-Z][a-z]+)$',
-            'regex3': r'^(Female|Male)(\d{1,2}|K)$',
-            'map': map_pattern_8
-        },
-        # Pattern 9: FirstLast (concatenated), Grade, School, Gender
-        {
-            'regex1': r'^([A-Z][a-z]+)([A-Z][a-z]+)$',
-            'regex2': r'^(\d{1,2}|K)$',
-            'regex3': r'^(.+)$',
-            'regex4': r'^(Female|Male)$',
-            'map': map_pattern_9
-        },
-        # Pattern 10: Gender + School + LastName, Grade + FirstName
-        {
-            'regex1': r'^(Female|Male)(.+?)([A-Z][a-z]+)$',
-            'regex2': r'^(\d{1,2}|K)([A-Z][a-z]+)',
-            'map': map_pattern_10
-        }
     ]
 
-    # Try each pattern
     for pattern in patterns:
-        if all(extracted.values()):  # Already found all fields
+        if 'regex3' in pattern:
+            for i in range(len(lines) - 2):
+                m1 = re.match(pattern['regex1'], lines[i])
+                m2 = re.match(pattern['regex2'], lines[i+1])
+                m3 = re.match(pattern['regex3'], lines[i+2])
+                if m1 and m2 and m3:
+                    mapped = pattern['map'](m1, m2, m3)
+                    extracted.update(mapped)
+                    break
+        else:
+            for i in range(len(lines) - 1):
+                m1 = re.match(pattern['regex1'], lines[i])
+                m2 = re.match(pattern['regex2'], lines[i+1])
+                if m1 and m2:
+                    mapped = pattern['map'](m1, m2)
+                    extracted.update(mapped)
+                    break
+        if all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
             break
 
-        for i, line in enumerate(lines):
+    # Post-processing and validation
-            line = line.strip()
-            if not line:
-                continue
-
-            match1 = re.match(pattern['regex1'], line)
-            if not match1:
-                continue
-
-            mapped = None
-
-            if 'regex4' in pattern and i + 3 < len(lines):
-                # Try 4-line pattern
-                match2 = re.match(pattern['regex2'], lines[i + 1].strip())
-                match3 = re.match(pattern['regex3'], lines[i + 2].strip())
-                match4 = re.match(pattern['regex4'], lines[i + 3].strip())
-                if match2 and match3 and match4:
-                    mapped = pattern['map'](match1, match2, match3, match4)
-            elif 'regex3' in pattern and i + 2 < len(lines):
-                # Try 3-line pattern
-                match2 = re.match(pattern['regex2'], lines[i + 1].strip())
-                match3 = re.match(pattern['regex3'], lines[i + 2].strip())
-                if match2 and match3:
-                    mapped = pattern['map'](match1, match2, match3)
-            elif 'regex2' in pattern and i + 1 < len(lines):
-                # Try 2-line pattern
-                match2 = re.match(pattern['regex2'], lines[i + 1].strip())
-                if match2:
-                    if 'regex3' in pattern or 'regex4' in pattern:
-                        # This pattern expects more matches but only has 2 lines available
-                        continue
-                    mapped = pattern['map'](match1, match2)
-
-            if mapped:
-                extracted.update(mapped)
-
-
-    # Fourth pass: Try to extract from sequential lines (for cases like: NameName, Grade, School, Gender)
-    if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
-        for i in range(len(lines) - 3):
-            # Look for pattern: NameName, Grade, School, Gender
-            name_line = lines[i]
-            grade_line = lines[i+1]
-            school_line = lines[i+2]
-            gender_line = lines[i+3]
-            # NameName: two capitalized names concatenated
-            m = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', name_line)
-            if m and re.match(r'^(\d{1,2}|K)$', grade_line) and len(school_line) > 2 and gender_line in ['Male', 'Female']:
-                extracted['first_name'] = m.group(1)
-                extracted['last_name'] = m.group(2)
-                extracted['grade'] = grade_line
-                extracted['school'] = school_line
-                extracted['gender'] = gender_line
-                break
-
-    # Fifth pass: Try to extract from GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
-    if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
-        for i in range(len(lines) - 1):
-            line1 = lines[i]
-            line2 = lines[i+1]
-            m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', line1)
-            m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', line2)
-            if m1 and m2:
-                extracted['gender'] = m1.group(1)
-                extracted['school'] = m1.group(2).strip()
-                extracted['last_name'] = m1.group(3)
-                extracted['grade'] = m2.group(1)
-                extracted['first_name'] = m2.group(2)
-                break
-
-    # Final validation and cleanup
     required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
     if all(extracted.get(field) for field in required_fields):
-        # Clean up school names (remove extra spaces, standardize)
+        # Clean up school name: remove any trailing capitalized names or numbers
-        extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
+        extracted['school'] = re.sub(r'([A-Z][a-z]+|\d+)$', '', extracted['school']).strip()
-        # --- Improved post-processing for stuck-together school/grade/last name ---
+        # Remove any names from school if they match first/last name
-        school_suffixes = [
+        for n in [extracted['first_name'], extracted['last_name']]:
-            'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
+            if n:
-        ]
+                extracted['school'] = re.sub(rf'\b{re.escape(n)}\b', '', extracted['school']).strip()
-        # Find the last occurrence of any school suffix and split there
-        import re
-        school = extracted['school']
-        last_pos = -1
-        last_suffix = ''
-        for suffix in school_suffixes:
-            idx = school.rfind(suffix)
-            if idx != -1 and idx + len(suffix) > last_pos:
-                last_pos = idx + len(suffix)
-                last_suffix = suffix
-        if last_pos != -1:
-            # School name is everything up to and including the suffix
-            extracted['school'] = school[:last_pos].strip()
-            trailing = school[last_pos:].strip().replace(' ', '')
-            # Regex: (grade?)(all capitalized names?)(anything else?)
-            m = re.match(r'^(\d{1,2}|K)?((?:[A-Z][a-z]+)+)(.*)$', trailing)
-            if m:
-                grade = m.group(1)
-                names_str = m.group(2)
-                leftover = m.group(3)
-                if grade and not extracted['grade']:
-                    extracted['grade'] = grade
-                # Split names_str into tokens of capitalized names
-                names = re.findall(r'[A-Z][a-z]+', names_str)
-                if names:
-                    extracted['last_name'] = names[0]
-                    if len(names) > 1:
-                        extracted['first_name'] = ' '.join(names[1:])
-                # If there is leftover text, append to first name
-                if leftover:
-                    if extracted['first_name']:
-                        extracted['first_name'] += leftover
-                    else:
-                        extracted['first_name'] = leftover
         # Final check: if school is just a number or a name, discard it
         if re.fullmatch(r'(\d{1,2}|K)', extracted['school']) or re.fullmatch(r'([A-Z][a-z]+|[A-Z\-]+|[A-Z][a-z]+\s[A-Z][a-z]+)+', extracted['school']):
             extracted['school'] = ''
+        extracted['school'] = extracted['school'].strip()
         return extracted
 
     # Fallback: Try to extract fields from any order in the lines (for highly concatenated or out-of-order data)
-    # Look for all capitalized words (possible names)
+    forbidden_names = set(['School', 'Home', 'High', 'Middle', 'Academy', 'Christian', 'Temple', 'Rustburg', 'Liberty', 'Forest', 'Heritage', 'Amherst', 'Appomattox', 'Dunbar', 'Glass', 'Jefferson', 'Linkhorne', 'Monelison', 'Sandusky', 'Timberlake'])
+    gender_words = set(['Male', 'Female'])
+    grade_words = set([str(i) for i in range(1,13)] + ['K'])
+    school_name = extracted['school'].lower()
     names = []
     for line in lines:
         for word in line.split():
-            if re.match(r'^[A-Z][a-z]+$', word):
+            w_lower = word.lower()
+            # Exclude if word matches or contains any part of the school name
+            if (re.match(r'^[A-Z][a-z]+$', word)
+                and word not in forbidden_names
+                and word not in gender_words
+                and word not in grade_words
+                and school_name not in w_lower
+                and w_lower not in school_name):
                 names.append(word)
-    if len(names) >= 2:
+    # Require two distinct names
+    if len(names) >= 2 and names[0] != names[1]:
         if not extracted['first_name']:
             extracted['first_name'] = names[0]
         if not extracted['last_name']:
             extracted['last_name'] = names[1]
+    else:
+        return None
     # Look for grade
     for line in lines:
         m = re.search(r'(\d{1,2}|K)', line)
@@ -869,11 +832,13 @@ def process_job(job_id, files):
             job_queue[job_id]['status'] = 'processing'
             job_queue[job_id]['progress'] = 'Starting job processing...'
         save_jobs()  # Persist initial progress
-        processed_files = []
-        
         # Create base Episode directory for this job
         episode_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'Episode')
         os.makedirs(episode_dir, exist_ok=True)
+        processed_files = []
+        skipped_dir = os.path.join(episode_dir, 'Skipped')
+        os.makedirs(skipped_dir, exist_ok=True)
+        skipped_log_path = os.path.join(skipped_dir, 'skipped_files.log')
         job_queue[job_id]['progress'] = 'Created output directory structure'
         
         # Analyze ZIP contents before processing
@@ -931,10 +896,27 @@ def process_job(job_id, files):
                                 save_jobs()  # Persist current file progress
                                 print(f"DEBUG: Processing PDF: {f} at {pdf_path}")
                                 print(f"DEBUG: File exists: {os.path.exists(pdf_path)}")
+                                # Skip MacOS resource fork files (._*)
+                                if f.startswith('._'):
+                                    print(f"DEBUG: Skipping MacOS resource fork file: {f}")
+                                    continue
                                 pdf_info = extract_names_from_pdf(pdf_path)
                                 print(f"DEBUG: PDF extraction result: {pdf_info}")
                                 if 'error' in pdf_info:
-                                    processed_files.append(('skipped', f, pdf_info['error']))
+                                    # Move skipped file to Skipped/
+                                    skipped_pdf_path = os.path.join(skipped_dir, f)
+                                    try:
+                                        os.rename(pdf_path, skipped_pdf_path)
+                                    except Exception as move_err:
+                                        print(f"WARNING: Could not move skipped PDF {f}: {move_err}")
+                                    # Log skip reason and raw text if available
+                                    with open(skipped_log_path, 'a', encoding='utf-8') as slog:
+                                        slog.write(f"File: {f}\nReason: {pdf_info['error']}\n")
+                                        if 'raw_text' in pdf_info:
+                                            slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
+                                        slog.write("-"*50 + "\n")
+                                    # Store reason and raw_text for web GUI
+                                    processed_files.append(('skipped', f, pdf_info['error'], pdf_info.get('raw_text', '')))
                                     job_queue[job_id]['progress'] = f'Skipped {f} - {pdf_info["error"]}'
                                     print(f"DEBUG: Skipped PDF {f}: {pdf_info['error']}")
                                 else:
@@ -979,9 +961,25 @@ def process_job(job_id, files):
                 try:
                     job_queue[job_id]['progress'] = f'Processing PDF: {filename}'
                     save_jobs()  # Persist current file progress
+                    # Skip MacOS resource fork files (._*)
+                    if filename.startswith('._'):
+                        print(f"DEBUG: Skipping MacOS resource fork file: {filename}")
+                        continue
                     pdf_info = extract_names_from_pdf(filepath)
                     if 'error' in pdf_info:
-                        processed_files.append(('skipped', filename, pdf_info['error']))
+                        # Move skipped file to Skipped/
+                        skipped_pdf_path = os.path.join(skipped_dir, filename)
+                        try:
+                            os.rename(filepath, skipped_pdf_path)
+                        except Exception as move_err:
+                            print(f"WARNING: Could not move skipped PDF {filename}: {move_err}")
+                        # Log skip reason and raw text if available
+                        with open(skipped_log_path, 'a', encoding='utf-8') as slog:
+                            slog.write(f"File: {filename}\nReason: {pdf_info['error']}\n")
+                            if 'raw_text' in pdf_info:
+                                slog.write(f"Raw Text:\n{pdf_info['raw_text']}\n")
+                            slog.write("-"*50 + "\n")
+                        processed_files.append(('skipped', filename, pdf_info['error'], pdf_info.get('raw_text', '')))
                         job_queue[job_id]['progress'] = f'Skipped {filename} - {pdf_info["error"]}'
                     else:
                         job_queue[job_id]['progress'] = f'Extracted data for {pdf_info["first_name"]} {pdf_info["last_name"]}'
@@ -1013,58 +1011,50 @@ def process_job(job_id, files):
         
         job_queue[job_id]['progress'] = 'Creating ZIP archive...'
         # Create ZIP of the entire Episode directory structure
-        successful_files = [p for p in processed_files if p[0] == 'success']
+        # Always create ZIP, including Skipped/ folder if present
-        print(f"DEBUG: Starting ZIP creation, successful_files: {len(successful_files)}")
+        try:
-        if successful_files:
+            zip_output = os.path.join(app.config['ARCHIVE_FOLDER'], f'renamed_{job_id}.zip')
+            print(f"DEBUG: Creating ZIP at {zip_output}")
+            with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
+                for root, dirs, files_in_dir in os.walk(episode_dir):
+                    for file in files_in_dir:
+                        if file.startswith('.') or file.startswith('._'):
+                            continue  # Ignore dotfiles and MacOS resource forks
+                        file_path = os.path.join(root, file)
+                        arcname = os.path.relpath(file_path, episode_dir)
+                        print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
+                        zip_out.write(file_path, arcname)
+            job_queue[job_id]['zip_path'] = zip_output
+            job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
+            job_queue[job_id]['progress'] = f'Created ZIP archive with {len([p for p in processed_files if p[0] == 'success'])} organized files and {len([p for p in processed_files if p[0] == 'skipped'])} skipped files'
+            save_jobs()  # Persist completion progress
+            print(f"DEBUG: ZIP creation completed successfully")
+            # Clean up Episode directory after successful ZIP creation
             try:
-                # Create ZIP in archives folder
+                import shutil
-                zip_output = os.path.join(app.config['ARCHIVE_FOLDER'], f'renamed_{job_id}.zip')
+                shutil.rmtree(episode_dir)
-                print(f"DEBUG: Creating ZIP at {zip_output}")
+                print(f"DEBUG: Cleaned up Episode directory: {episode_dir}")
-                with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
+            except Exception as cleanup_error:
-                    for root, dirs, files_in_dir in os.walk(episode_dir):
+                print(f"WARNING: Failed to clean up Episode directory: {cleanup_error}")
-                        for file in files_in_dir:
+            # Clean up original uploaded files
-                            if file.startswith('.'):
+            for file_info in files:
-                                continue  # Ignore dotfiles
-                            file_path = os.path.join(root, file)
-                            arcname = os.path.relpath(file_path, episode_dir)
-                            print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
-                            zip_out.write(file_path, arcname)
-                job_queue[job_id]['zip_path'] = zip_output
-                job_queue[job_id]['zip_name'] = f'renamed_{job_id}.zip'
-                job_queue[job_id]['progress'] = f'Created ZIP archive with {len(successful_files)} organized files'
-                save_jobs()  # Persist completion progress
-                print(f"DEBUG: ZIP creation completed successfully")
-                
-                # Clean up Episode directory after successful ZIP creation
                 try:
-                    import shutil
+                    if os.path.exists(file_info['path']):
-                    shutil.rmtree(episode_dir)
+                        os.remove(file_info['path'])
-                    print(f"DEBUG: Cleaned up Episode directory: {episode_dir}")
+                        print(f"DEBUG: Removed uploaded file: {file_info['path']}")
-                except Exception as cleanup_error:
+                except Exception as file_cleanup_error:
-                    print(f"WARNING: Failed to clean up Episode directory: {cleanup_error}")
+                    print(f"WARNING: Failed to remove uploaded file: {file_cleanup_error}")
-                
+        except Exception as e:
-                # Clean up original uploaded files
+            job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
-                for file_info in files:
+            job_queue[job_id]['status'] = 'error'
-                    try:
+            save_jobs()  # Persist error status
-                        if os.path.exists(file_info['path']):
+            print(f"DEBUG: ZIP creation failed: {str(e)}")
-                            os.remove(file_info['path'])
-                            print(f"DEBUG: Removed uploaded file: {file_info['path']}")
-                    except Exception as file_cleanup_error:
-                        print(f"WARNING: Failed to remove uploaded file: {file_cleanup_error}")
-                
-            except Exception as e:
-                job_queue[job_id]['progress'] = f'Error creating ZIP: {str(e)}'
-                job_queue[job_id]['status'] = 'error'
-                save_jobs()  # Persist error status
-                print(f"DEBUG: ZIP creation failed: {str(e)}")
-        else:
-            job_queue[job_id]['progress'] = 'No files to archive - all files were skipped'
-            print(f"DEBUG: No successful files to archive")
         
         with job_lock:
             job_queue[job_id]['status'] = 'completed'
             job_queue[job_id]['processed_files'] = processed_files
-            job_queue[job_id]['skipped'] = [(p[1], p[2]) for p in processed_files if p[0] == 'skipped']
+            # Store skipped as (filename, reason, raw_text)
+            job_queue[job_id]['skipped'] = [(p[1], p[2], p[3]) for p in processed_files if p[0] == 'skipped']
         save_jobs()  # Persist completed job
         print(f"DEBUG: Job {job_id} completed successfully. Processed: {len(processed_files)}, Skipped: {len(job_queue[job_id]['skipped'])}")
 
@@ -1870,7 +1860,11 @@ def completed_jobs():
                 <div class="skipped-details">
                     <ul>
                     {% for skip_info in job.skipped %}
-                        <li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}</li>
+                        <li><strong>{{ skip_info[0] }}</strong>: {{ skip_info[1] }}
+                        {% if skip_info|length > 2 and skip_info[2] %}
+                            <br><details><summary>Extracted Text</summary><pre style="white-space: pre-wrap; background: #f8f9fa; border: 1px solid #eee; padding: 5px;">{{ skip_info[2] }}</pre></details>
+                        {% endif %}
+                        </li>
                     {% endfor %}
                     </ul>
                 </div>