Fix school parsing edge cases

.gitignore

@@ -30,4 +30,8 @@ Thumbs.db
 
 # Temporary files
 *.tmp
-*.tempEpisode Waivers Jan 29.zip
+*.temp
+
+# Local test ZIPs and archive outputs
+Episode Waivers Jan 29.zip
+archives/*.zip

pdf_renamer.py

@@ -445,6 +445,9 @@ def extract_names_from_pdf(pdf_path):
             extracted_data = extract_student_info_from_text(first_page_text)
 
             if extracted_data and all(extracted_data.values()):
+                school_val = extracted_data.get('school', '').strip()
+                if school_val.isdigit() or school_val.lower().startswith('docusign envelope id'):
+                    extracted_data['school'] = ''
                 return {
                     'filename': f"{extracted_data['last_name']}_{extracted_data['first_name']}_{extracted_data['grade']}.pdf",
                     'last_name': extracted_data['last_name'],
@@ -485,8 +488,190 @@ def extract_student_info_from_text(text):
     name_re = r"[A-Z][a-zA-Z]+(?:[-'][A-Za-zA-Z]+)*"
     def norm_name(value):
         return value.strip().replace('"', '').title()
+    school_suffix_words = ['High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private']
+    school_suffix_re = r'(?:' + '|'.join(school_suffix_words) + r')'
+    skip_headers = [
+        'student first name:', 'student last name:', 'student grade:',
+        'student school:', 'details', 'gender:', 'docusign envelope id:',
+        'docusign envelope id', 'episode registration', 'form', 'page', 'date',
+        'signature', 'parent', 'guardian'
+    ]
+    def is_school_only(line):
+        if not line:
+            return False
+        if line.strip().lower() == 'other (not listed)':
+            return True
+        words = re.sub(r'[^A-Za-z ]', '', line).split()
+        return len(words) >= 2 and words[-1] in school_suffix_words
+    def split_school_first(line):
+        m = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', line)
+        if m:
+            return m.group(1).strip(), m.group(2)
+        return None
+    # Pre-clean lines: remove headers, normalize lowercase names, split gender/school concatenations
+    preprocessed_lines = []
+    for line in lines:
+        orig_line = line.strip()
+        if not orig_line:
+            continue
+        lower_line = orig_line.lower()
+        if any(h in lower_line for h in skip_headers):
+            continue
+        if orig_line.islower() and orig_line.isalpha():
+            orig_line = orig_line.title()
+        m_gender = re.match(r'^(Female|Male)(.+)', orig_line)
+        if m_gender:
+            rest = m_gender.group(2).strip()
+            rest_is_title_words = re.match(r'^[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)+$', rest)
+            rest_is_other = rest.lower().startswith('other (not listed)')
+            # Keep combined gender+school/grade lines for existing patterns
+            if re.search(rf'\b{school_suffix_re}\b', rest) or re.search(r'\d', rest) or rest_is_title_words or rest_is_other:
+                orig_line = m_gender.group(1) + rest
+            else:
+                preprocessed_lines.append(m_gender.group(1))
+                orig_line = rest
+        split_school = split_school_first(orig_line)
+        if split_school:
+            preprocessed_lines.append(split_school[0])
+            preprocessed_lines.append(split_school[1])
+            continue
+        preprocessed_lines.append(orig_line)
+    lines = preprocessed_lines
     
     # --- Custom patterns for failed extractions ---
+    # Pattern: Uppercase LASTNAME, School, Uppercase FIRSTNAME, GenderGrade (after preprocessing)
+    for i in range(len(lines) - 3):
+        m1 = re.match(r'^([A-Z][A-Z]+)$', lines[i])
+        m2 = re.match(rf'^(.*\b{school_suffix_re})$', lines[i+1])
+        m3 = re.match(r'^([A-Z][A-Z]+)$', lines[i+2])
+        m4 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+3])
+        if m1 and m2 and m3 and m4:
+            extracted['last_name'] = norm_name(m1.group(1))
+            extracted['school'] = m2.group(1).strip()
+            extracted['first_name'] = norm_name(m3.group(1))
+            extracted['gender'] = m4.group(1)
+            extracted['grade'] = m4.group(2)
+            return extracted
+
+    # Pattern: School, Last, GenderGradeFirst (after preprocessing splits SchoolLast)
+    for i in range(len(lines) - 2):
+        m1 = re.match(rf'^(.*\b{school_suffix_re})$', lines[i])
+        m2 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i+1])
+        m3 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+2])
+        if m1 and m2 and m3 and not is_school_only(lines[i+1]):
+            extracted['school'] = m1.group(1).strip()
+            extracted['last_name'] = norm_name(m2.group(1))
+            extracted['gender'] = m3.group(1)
+            extracted['grade'] = m3.group(2)
+            extracted['first_name'] = m3.group(3)
+            return extracted
+
+    # Pattern: School, First, GradeLast, Gender (after preprocessing splits SchoolFirst)
+    for i in range(len(lines) - 3):
+        m1 = re.match(rf'^(.*\b{school_suffix_re})$', lines[i])
+        m2 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i+1])
+        m3 = re.match(r'^(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+2])
+        m4 = re.match(r'^(Female|Male)$', lines[i+3])
+        if m1 and m2 and m3 and m4 and not is_school_only(lines[i+1]):
+            extracted['school'] = m1.group(1).strip()
+            extracted['first_name'] = norm_name(m2.group(1))
+            extracted['grade'] = m3.group(1)
+            extracted['last_name'] = m3.group(2)
+            extracted['gender'] = m4.group(1)
+            return extracted
+
+    # Pattern: SchoolLast, GenderGradeFirst (e.g., Home SchoolGillum, Female11Karis)
+    for i in range(len(lines) - 1):
+        m1 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i])
+        m2 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+1])
+        if m1 and m2:
+            extracted['school'] = m1.group(1).strip()
+            extracted['last_name'] = norm_name(m1.group(2))
+            extracted['gender'] = m2.group(1)
+            extracted['grade'] = m2.group(2)
+            extracted['first_name'] = m2.group(3)
+            return extracted
+
+    # Pattern: SchoolFirst, GradeLast, Gender (e.g., Heritage HighKailee, 12Lawhorne, Female)
+    for i in range(len(lines) - 2):
+        m1 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i])
+        m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+1])
+        m3 = re.match(r'^(Female|Male)$', lines[i+2])
+        if m1 and m2 and m3:
+            extracted['school'] = m1.group(1).strip()
+            extracted['first_name'] = norm_name(m1.group(2))
+            extracted['grade'] = m2.group(1)
+            extracted['last_name'] = m2.group(2)
+            extracted['gender'] = m3.group(1)
+            return extracted
+
+    # Pattern: Uppercase LASTNAME, SchoolFirstName, GenderGrade (e.g., NIKITINA, Linkhorne MiddleLILLIAN, Female6)
+    for i in range(len(lines) - 2):
+        m1 = re.match(r'^([A-Z][A-Z]+)$', lines[i])  # All uppercase last name
+        m2 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i+1])
+        m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2])
+        if m1 and m2 and m3:
+            extracted['last_name'] = norm_name(m1.group(1))
+            extracted['school'] = m2.group(1).strip()
+            extracted['first_name'] = norm_name(m2.group(2))
+            extracted['gender'] = m3.group(1)
+            extracted['grade'] = m3.group(2)
+            return extracted
+
+    # Pattern: FirstName, GenderGradeLast, School (e.g., Noah, Male11Smith, Liberty Christian Academy)
+    for i in range(len(lines) - 2):
+        m1 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i])
+        m2 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+1])
+        m3 = re.match(r'^(.+)$', lines[i+2])
+        if m1 and m2 and m3 and not is_school_only(lines[i]):
+            extracted['first_name'] = m1.group(1)
+            extracted['gender'] = m2.group(1)
+            extracted['grade'] = m2.group(2)
+            extracted['last_name'] = m2.group(3)
+            extracted['school'] = m3.group(1).strip()
+            return extracted
+
+    # Pattern: GenderGradeLastFirst, School (e.g., Female11SpearAva, Home School)
+    for i in range(len(lines) - 1):
+        m1 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)([A-Z][a-zA-Z]+)$', lines[i])
+        m2 = re.match(r'^(.+)$', lines[i+1])
+        if m1 and m2:
+            extracted['gender'] = m1.group(1)
+            extracted['grade'] = m1.group(2)
+            extracted['last_name'] = m1.group(3)
+            extracted['first_name'] = m1.group(4)
+            extracted['school'] = m2.group(1).strip()
+            return extracted
+
+    # Pattern: SchoolFirstName, GenderGrade (e.g., Linkhorne MiddleLILLIAN, Female6)
+    for i in range(len(lines) - 1):
+        m1 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i])
+        m2 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+1])
+        if m1 and m2:
+            extracted['school'] = m1.group(1).strip()
+            extracted['first_name'] = norm_name(m1.group(2))
+            extracted['gender'] = m2.group(1)
+            extracted['grade'] = m2.group(2)
+            # Look for last name on next line
+            if i + 2 < len(lines):
+                m3 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i+2])
+                if m3:
+                    extracted['last_name'] = m3.group(1)
+            return extracted
+
+    # Pattern: GenderGradeFirstName, SchoolLastName (e.g., Male11Dwight, Heritage HighWilliams IV)
+    for i in range(len(lines) - 1):
+        m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i])
+        m2 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)(?:\s+(I|II|III|IV|V|VI|Jr|Sr))?$', lines[i+1])
+        if m1 and m2:
+            extracted['gender'] = m1.group(1)
+            extracted['grade'] = m1.group(2)
+            extracted['first_name'] = m1.group(3)
+            suffix = f" {m2.group(3)}" if m2.group(3) else ''
+            extracted['last_name'] = f"{m2.group(2)}{suffix}".strip()
+            extracted['school'] = m2.group(1).strip()
+            return extracted
+
     # Pattern: GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
     for i in range(len(lines) - 1):
         m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', lines[i])
@@ -513,19 +698,108 @@ def extract_student_info_from_text(text):
             extracted['gender'] = m4.group(1)
             break
 
+    # Pattern: School, GenderGradeFirstName, LastName (e.g., Liberty Christian Academy, Female11Ava, Lawrenson)
+    for i in range(len(lines) - 2):
+        if not is_school_only(lines[i]):
+            continue
+        m1 = re.match(r'^(.+)$', lines[i])
+        m2 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
+        m3 = re.match(r'^(' + name_re + r')$', lines[i+2])
+        if m1 and m2 and m3:
+            extracted['school'] = m1.group(1).strip()
+            extracted['gender'] = m2.group(1)
+            extracted['grade'] = m2.group(2)
+            extracted['first_name'] = m2.group(3)
+            extracted['last_name'] = m3.group(1)
+            return extracted
+
     # Pattern: SchoolFirstName, LastName, GenderGrade (e.g., Liberty Christian AcademySally, Sue, Female9)
     for i in range(len(lines) - 2):
+        split = split_school_first(lines[i])
         m1 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i])
         m2 = re.match(r'^([A-Z][a-z]+)$', lines[i+1])
         m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2])
-        if m1 and m2 and m3:
+        if m2 and m3:
-            extracted['school'] = m1.group(1).strip()
+            if split:
-            extracted['first_name'] = m1.group(2)
+                school_val, first_val = split
+            elif m1 and not is_school_only(lines[i]) and ' ' in lines[i]:
+                school_val, first_val = m1.group(1).strip(), m1.group(2)
+            else:
+                continue
+            extracted['school'] = school_val
+            extracted['first_name'] = first_val
             extracted['last_name'] = m2.group(1)
             extracted['gender'] = m3.group(1)
             extracted['grade'] = m3.group(2)
             break
 
+    # Pattern: GenderGradeFirstName, SchoolLastName (e.g., Male11Dwight, Heritage HighWilliams IV)
+    for i in range(len(lines) - 1):
+        m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i])
+        m2 = re.match(r'^(.+)$', lines[i+1])
+        if m1 and m2:
+            m_school = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)(?:\s+(I|II|III|IV|V|VI|Jr|Sr))?$', m2.group(1).strip())
+            if m_school:
+                extracted['gender'] = m1.group(1)
+                extracted['grade'] = m1.group(2)
+                extracted['first_name'] = m1.group(3)
+                suffix = f" {m_school.group(3)}" if m_school.group(3) else ''
+                extracted['last_name'] = f"{m_school.group(2)}{suffix}".strip()
+                extracted['school'] = m_school.group(1).strip()
+                break
+
+    # Pattern: SchoolFirstName, GenderGradeLastName (e.g., Forest MiddleCheyanne, Female8Henderson)
+    for i in range(len(lines) - 1):
+        split = split_school_first(lines[i])
+        m1 = re.match(r'^(.+?)([A-Z][a-zA-Z]+)$', lines[i])
+        m2 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
+        if m2:
+            if split:
+                school_val, first_val = split
+            elif m1 and not is_school_only(lines[i]) and ' ' in lines[i]:
+                school_val, first_val = m1.group(1).strip(), m1.group(2)
+            else:
+                continue
+            extracted['school'] = school_val
+            extracted['first_name'] = first_val
+            extracted['gender'] = m2.group(1)
+            extracted['grade'] = m2.group(2)
+            extracted['last_name'] = m2.group(3)
+            break
+
+    # Pattern: FirstName, GenderGradeLastName, School (e.g., Myla, Female8Littlejohn, Home School)
+    for i in range(len(lines) - 2):
+        m1 = re.match(r'^(' + name_re + r')$', lines[i])
+        m2 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i+1])
+        m3 = re.match(r'^(.+)$', lines[i+2])
+        if m1 and m2 and m3:
+            extracted['first_name'] = m1.group(1)
+            extracted['gender'] = m2.group(1)
+            extracted['grade'] = m2.group(2)
+            extracted['last_name'] = m2.group(3)
+            extracted['school'] = m3.group(1).strip()
+            break
+
+    # Pattern: LastName, SchoolFirstName, GenderGrade (e.g., NIKITINA, Linkhorne MiddleLILLIAN, Female6)
+    for i in range(len(lines) - 2):
+        m1 = re.match(r'^([A-Z]{2,})$', lines[i])
+        split = split_school_first(lines[i+1])
+        m2 = re.match(r'^(.+?)([A-Z][a-zA-Z]+)$', lines[i+1])
+        m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2])
+        if m1 and m3:
+            if split:
+                school_val, first_val = split
+            elif m2:
+                school_val, first_val = m2.group(1).strip(), m2.group(2)
+            else:
+                continue
+            extracted['last_name'] = norm_name(m1.group(1))
+            extracted['school'] = school_val
+            extracted['first_name'] = norm_name(first_val)
+            extracted['gender'] = m3.group(1)
+            extracted['grade'] = m3.group(2)
+            break
+
     # Pattern: FirstName, SchoolGradeLastName, Gender (e.g., Michael, Home School7Shuck, Male)
     for i in range(len(lines) - 2):
         m1 = re.match(r'^([A-Z][a-z]+)$', lines[i])
@@ -582,11 +856,18 @@ def extract_student_info_from_text(text):
         m1 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-z]+)$', lines[i])
         m2 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i+1])
         if m1 and m2:
+            split = split_school_first(lines[i+1])
+            if split:
+                school_val, first_val = split
+            elif is_school_only(lines[i+1]):
+                continue
+            else:
+                school_val, first_val = m2.group(1).strip(), m2.group(2)
             extracted['gender'] = m1.group(1)
             extracted['grade'] = m1.group(2)
             extracted['last_name'] = m1.group(3)
-            extracted['school'] = m2.group(1).strip()
+            extracted['school'] = school_val
-            extracted['first_name'] = m2.group(2)
+            extracted['first_name'] = first_val
             break
 
     # Pattern: GenderLastName, GradeFirstName, School (e.g., MaleKirk, 10Jaihden "JJ", Home School)
@@ -723,11 +1004,18 @@ def extract_student_info_from_text(text):
         m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i])
         m2 = re.match(r'^(.+?)(' + name_re + r')$', lines[i+1])
         if m1 and m2:
+            split = split_school_first(lines[i+1])
+            if split:
+                school_val, first_val = split
+            elif is_school_only(lines[i+1]):
+                continue
+            else:
+                school_val, first_val = m2.group(1).strip(), m2.group(2)
             extracted['gender'] = m1.group(1)
             extracted['grade'] = m1.group(2)
             extracted['last_name'] = m1.group(3)
-            extracted['school'] = m2.group(1).strip()
+            extracted['school'] = school_val
-            extracted['first_name'] = m2.group(2)
+            extracted['first_name'] = first_val
             break
 
     # Pattern: SchoolLastNameFirstName, GenderGrade (e.g., Home SchoolBethelAbigail, Female7)
@@ -886,6 +1174,8 @@ def extract_student_info_from_text(text):
 
     # Pattern: SchoolFirstName (case-insensitive), GenderGradeLastName (case-insensitive)
     for i in range(len(lines) - 1):
+        if is_school_only(lines[i]) or (' ' not in lines[i] and not split_school_first(lines[i])):
+            continue
         m1 = re.match(r'^(.+?)([A-Za-z]+)$', lines[i], re.IGNORECASE)
         m2 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Za-z]+)$', lines[i+1], re.IGNORECASE)
         if m1 and m2:
@@ -1040,11 +1330,6 @@ def extract_student_info_from_text(text):
 
     # Enhanced preprocessing: aggressively split lines with multiple fields stuck together
     processed_lines = []
-    skip_headers = [
-        'student first name:', 'student last name:', 'student grade:',
-        'student school:', 'details', 'gender:', 'docusign envelope id:',
-        'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian'
-    ]
     valid_schools = [
         'Amherst', 'Appomattox', 'Brookville', 'Dunbar', 'E.C. Glass', 'Forest', 'Heritage', 'Home School',
         'Jefferson Forest', 'Liberty Christian Academy', 'Linkhorne', 'Monelison', 'Rustburg', 'Sandusky',
@@ -1058,6 +1343,15 @@ def extract_student_info_from_text(text):
         # Ignore header/title/label lines
         if not orig_line or any(h in orig_line.lower() for h in skip_headers):
             continue
+        # Normalize single-word lowercase names
+        if orig_line.islower() and orig_line.isalpha():
+            orig_line = orig_line.title()
+        # Split school+name concatenations (e.g., "Liberty Christian AcademyWhorley")
+        split_school = split_school_first(orig_line)
+        if split_school:
+            processed_lines.append(split_school[0])
+            processed_lines.append(split_school[1])
+            continue
         # Aggressive split: if line starts with a valid school name (with suffix) and is immediately followed by a capitalized name or digit, split
         split_done = False
         for school_name in all_school_names:
@@ -1313,8 +1607,11 @@ def extract_student_info_from_text(text):
             if n:
                 school_clean = re.sub(rf'\b{re.escape(n)}\b', '', school_clean).strip()
         extracted['school'] = school_clean
-        # Final check: discard invalid schools (pure grade or same as student name)
+        # Final check: discard invalid schools (pure grade, header text, or same as student name)
         school_clean = extracted['school'].strip()
+        if any(x in school_clean.lower() for x in ['docusign envelope id', 'episode registration']):
+            extracted['school'] = ''
+            return extracted
         if re.fullmatch(r'(\d{1,2}|K)', school_clean):
             extracted['school'] = ''
         else: