From ca05eb8fa538e74ca86f91f31a4e3ba5f990e946 Mon Sep 17 00:00:00 2001 From: nmemmert Date: Wed, 4 Feb 2026 11:42:36 -0500 Subject: [PATCH] Fix school parsing edge cases --- .gitignore | 6 +- pdf_renamer.py | 323 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 315 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 893472a..0c08d89 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,8 @@ Thumbs.db # Temporary files *.tmp -*.tempEpisode Waivers Jan 29.zip +*.temp + +# Local test ZIPs and archive outputs +Episode Waivers Jan 29.zip +archives/*.zip diff --git a/pdf_renamer.py b/pdf_renamer.py index c5f2616..c594cec 100644 --- a/pdf_renamer.py +++ b/pdf_renamer.py @@ -445,6 +445,9 @@ def extract_names_from_pdf(pdf_path): extracted_data = extract_student_info_from_text(first_page_text) if extracted_data and all(extracted_data.values()): + school_val = extracted_data.get('school', '').strip() + if school_val.isdigit() or school_val.lower().startswith('docusign envelope id'): + extracted_data['school'] = '' return { 'filename': f"{extracted_data['last_name']}_{extracted_data['first_name']}_{extracted_data['grade']}.pdf", 'last_name': extracted_data['last_name'], @@ -485,8 +488,190 @@ def extract_student_info_from_text(text): name_re = r"[A-Z][a-zA-Z]+(?:[-'][A-Za-zA-Z]+)*" def norm_name(value): return value.strip().replace('"', '').title() + school_suffix_words = ['High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'] + school_suffix_re = r'(?:' + '|'.join(school_suffix_words) + r')' + skip_headers = [ + 'student first name:', 'student last name:', 'student grade:', + 'student school:', 'details', 'gender:', 'docusign envelope id:', + 'docusign envelope id', 'episode registration', 'form', 'page', 'date', + 'signature', 'parent', 'guardian' + ] + def is_school_only(line): + if not line: + return False + if line.strip().lower() == 'other (not listed)': + return True + words = re.sub(r'[^A-Za-z ]', '', line).split() + return len(words) >= 2 and words[-1] in school_suffix_words + def split_school_first(line): + m = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', line) + if m: + return m.group(1).strip(), m.group(2) + return None + # Pre-clean lines: remove headers, normalize lowercase names, split gender/school concatenations + preprocessed_lines = [] + for line in lines: + orig_line = line.strip() + if not orig_line: + continue + lower_line = orig_line.lower() + if any(h in lower_line for h in skip_headers): + continue + if orig_line.islower() and orig_line.isalpha(): + orig_line = orig_line.title() + m_gender = re.match(r'^(Female|Male)(.+)', orig_line) + if m_gender: + rest = m_gender.group(2).strip() + rest_is_title_words = re.match(r'^[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)+$', rest) + rest_is_other = rest.lower().startswith('other (not listed)') + # Keep combined gender+school/grade lines for existing patterns + if re.search(rf'\b{school_suffix_re}\b', rest) or re.search(r'\d', rest) or rest_is_title_words or rest_is_other: + orig_line = m_gender.group(1) + rest + else: + preprocessed_lines.append(m_gender.group(1)) + orig_line = rest + split_school = split_school_first(orig_line) + if split_school: + preprocessed_lines.append(split_school[0]) + preprocessed_lines.append(split_school[1]) + continue + preprocessed_lines.append(orig_line) + lines = preprocessed_lines # --- Custom patterns for failed extractions --- + # Pattern: Uppercase LASTNAME, School, Uppercase FIRSTNAME, GenderGrade (after preprocessing) + for i in range(len(lines) - 3): + m1 = re.match(r'^([A-Z][A-Z]+)$', lines[i]) + m2 = re.match(rf'^(.*\b{school_suffix_re})$', lines[i+1]) + m3 = re.match(r'^([A-Z][A-Z]+)$', lines[i+2]) + m4 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+3]) + if m1 and m2 and m3 and m4: + extracted['last_name'] = norm_name(m1.group(1)) + extracted['school'] = m2.group(1).strip() + extracted['first_name'] = norm_name(m3.group(1)) + extracted['gender'] = m4.group(1) + extracted['grade'] = m4.group(2) + return extracted + + # Pattern: School, Last, GenderGradeFirst (after preprocessing splits SchoolLast) + for i in range(len(lines) - 2): + m1 = re.match(rf'^(.*\b{school_suffix_re})$', lines[i]) + m2 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i+1]) + m3 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+2]) + if m1 and m2 and m3 and not is_school_only(lines[i+1]): + extracted['school'] = m1.group(1).strip() + extracted['last_name'] = norm_name(m2.group(1)) + extracted['gender'] = m3.group(1) + extracted['grade'] = m3.group(2) + extracted['first_name'] = m3.group(3) + return extracted + + # Pattern: School, First, GradeLast, Gender (after preprocessing splits SchoolFirst) + for i in range(len(lines) - 3): + m1 = re.match(rf'^(.*\b{school_suffix_re})$', lines[i]) + m2 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i+1]) + m3 = re.match(r'^(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+2]) + m4 = re.match(r'^(Female|Male)$', lines[i+3]) + if m1 and m2 and m3 and m4 and not is_school_only(lines[i+1]): + extracted['school'] = m1.group(1).strip() + extracted['first_name'] = norm_name(m2.group(1)) + extracted['grade'] = m3.group(1) + extracted['last_name'] = m3.group(2) + extracted['gender'] = m4.group(1) + return extracted + + # Pattern: SchoolLast, GenderGradeFirst (e.g., Home SchoolGillum, Female11Karis) + for i in range(len(lines) - 1): + m1 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i]) + m2 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+1]) + if m1 and m2: + extracted['school'] = m1.group(1).strip() + extracted['last_name'] = norm_name(m1.group(2)) + extracted['gender'] = m2.group(1) + extracted['grade'] = m2.group(2) + extracted['first_name'] = m2.group(3) + return extracted + + # Pattern: SchoolFirst, GradeLast, Gender (e.g., Heritage HighKailee, 12Lawhorne, Female) + for i in range(len(lines) - 2): + m1 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i]) + m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+1]) + m3 = re.match(r'^(Female|Male)$', lines[i+2]) + if m1 and m2 and m3: + extracted['school'] = m1.group(1).strip() + extracted['first_name'] = norm_name(m1.group(2)) + extracted['grade'] = m2.group(1) + extracted['last_name'] = m2.group(2) + extracted['gender'] = m3.group(1) + return extracted + + # Pattern: Uppercase LASTNAME, SchoolFirstName, GenderGrade (e.g., NIKITINA, Linkhorne MiddleLILLIAN, Female6) + for i in range(len(lines) - 2): + m1 = re.match(r'^([A-Z][A-Z]+)$', lines[i]) # All uppercase last name + m2 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i+1]) + m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2]) + if m1 and m2 and m3: + extracted['last_name'] = norm_name(m1.group(1)) + extracted['school'] = m2.group(1).strip() + extracted['first_name'] = norm_name(m2.group(2)) + extracted['gender'] = m3.group(1) + extracted['grade'] = m3.group(2) + return extracted + + # Pattern: FirstName, GenderGradeLast, School (e.g., Noah, Male11Smith, Liberty Christian Academy) + for i in range(len(lines) - 2): + m1 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i]) + m2 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)$', lines[i+1]) + m3 = re.match(r'^(.+)$', lines[i+2]) + if m1 and m2 and m3 and not is_school_only(lines[i]): + extracted['first_name'] = m1.group(1) + extracted['gender'] = m2.group(1) + extracted['grade'] = m2.group(2) + extracted['last_name'] = m2.group(3) + extracted['school'] = m3.group(1).strip() + return extracted + + # Pattern: GenderGradeLastFirst, School (e.g., Female11SpearAva, Home School) + for i in range(len(lines) - 1): + m1 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-zA-Z]+)([A-Z][a-zA-Z]+)$', lines[i]) + m2 = re.match(r'^(.+)$', lines[i+1]) + if m1 and m2: + extracted['gender'] = m1.group(1) + extracted['grade'] = m1.group(2) + extracted['last_name'] = m1.group(3) + extracted['first_name'] = m1.group(4) + extracted['school'] = m2.group(1).strip() + return extracted + + # Pattern: SchoolFirstName, GenderGrade (e.g., Linkhorne MiddleLILLIAN, Female6) + for i in range(len(lines) - 1): + m1 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)$', lines[i]) + m2 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+1]) + if m1 and m2: + extracted['school'] = m1.group(1).strip() + extracted['first_name'] = norm_name(m1.group(2)) + extracted['gender'] = m2.group(1) + extracted['grade'] = m2.group(2) + # Look for last name on next line + if i + 2 < len(lines): + m3 = re.match(r'^([A-Z][a-zA-Z]+)$', lines[i+2]) + if m3: + extracted['last_name'] = m3.group(1) + return extracted + + # Pattern: GenderGradeFirstName, SchoolLastName (e.g., Male11Dwight, Heritage HighWilliams IV) + for i in range(len(lines) - 1): + m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i]) + m2 = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)(?:\s+(I|II|III|IV|V|VI|Jr|Sr))?$', lines[i+1]) + if m1 and m2: + extracted['gender'] = m1.group(1) + extracted['grade'] = m1.group(2) + extracted['first_name'] = m1.group(3) + suffix = f" {m2.group(3)}" if m2.group(3) else '' + extracted['last_name'] = f"{m2.group(2)}{suffix}".strip() + extracted['school'] = m2.group(1).strip() + return extracted + # Pattern: GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey) for i in range(len(lines) - 1): m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', lines[i]) @@ -513,19 +698,108 @@ def extract_student_info_from_text(text): extracted['gender'] = m4.group(1) break + # Pattern: School, GenderGradeFirstName, LastName (e.g., Liberty Christian Academy, Female11Ava, Lawrenson) + for i in range(len(lines) - 2): + if not is_school_only(lines[i]): + continue + m1 = re.match(r'^(.+)$', lines[i]) + m2 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i+1]) + m3 = re.match(r'^(' + name_re + r')$', lines[i+2]) + if m1 and m2 and m3: + extracted['school'] = m1.group(1).strip() + extracted['gender'] = m2.group(1) + extracted['grade'] = m2.group(2) + extracted['first_name'] = m2.group(3) + extracted['last_name'] = m3.group(1) + return extracted + # Pattern: SchoolFirstName, LastName, GenderGrade (e.g., Liberty Christian AcademySally, Sue, Female9) for i in range(len(lines) - 2): + split = split_school_first(lines[i]) m1 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i]) m2 = re.match(r'^([A-Z][a-z]+)$', lines[i+1]) m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2]) - if m1 and m2 and m3: - extracted['school'] = m1.group(1).strip() - extracted['first_name'] = m1.group(2) + if m2 and m3: + if split: + school_val, first_val = split + elif m1 and not is_school_only(lines[i]) and ' ' in lines[i]: + school_val, first_val = m1.group(1).strip(), m1.group(2) + else: + continue + extracted['school'] = school_val + extracted['first_name'] = first_val extracted['last_name'] = m2.group(1) extracted['gender'] = m3.group(1) extracted['grade'] = m3.group(2) break + # Pattern: GenderGradeFirstName, SchoolLastName (e.g., Male11Dwight, Heritage HighWilliams IV) + for i in range(len(lines) - 1): + m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i]) + m2 = re.match(r'^(.+)$', lines[i+1]) + if m1 and m2: + m_school = re.match(rf'^(.*\b{school_suffix_re})([A-Z][a-zA-Z]+)(?:\s+(I|II|III|IV|V|VI|Jr|Sr))?$', m2.group(1).strip()) + if m_school: + extracted['gender'] = m1.group(1) + extracted['grade'] = m1.group(2) + extracted['first_name'] = m1.group(3) + suffix = f" {m_school.group(3)}" if m_school.group(3) else '' + extracted['last_name'] = f"{m_school.group(2)}{suffix}".strip() + extracted['school'] = m_school.group(1).strip() + break + + # Pattern: SchoolFirstName, GenderGradeLastName (e.g., Forest MiddleCheyanne, Female8Henderson) + for i in range(len(lines) - 1): + split = split_school_first(lines[i]) + m1 = re.match(r'^(.+?)([A-Z][a-zA-Z]+)$', lines[i]) + m2 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i+1]) + if m2: + if split: + school_val, first_val = split + elif m1 and not is_school_only(lines[i]) and ' ' in lines[i]: + school_val, first_val = m1.group(1).strip(), m1.group(2) + else: + continue + extracted['school'] = school_val + extracted['first_name'] = first_val + extracted['gender'] = m2.group(1) + extracted['grade'] = m2.group(2) + extracted['last_name'] = m2.group(3) + break + + # Pattern: FirstName, GenderGradeLastName, School (e.g., Myla, Female8Littlejohn, Home School) + for i in range(len(lines) - 2): + m1 = re.match(r'^(' + name_re + r')$', lines[i]) + m2 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i+1]) + m3 = re.match(r'^(.+)$', lines[i+2]) + if m1 and m2 and m3: + extracted['first_name'] = m1.group(1) + extracted['gender'] = m2.group(1) + extracted['grade'] = m2.group(2) + extracted['last_name'] = m2.group(3) + extracted['school'] = m3.group(1).strip() + break + + # Pattern: LastName, SchoolFirstName, GenderGrade (e.g., NIKITINA, Linkhorne MiddleLILLIAN, Female6) + for i in range(len(lines) - 2): + m1 = re.match(r'^([A-Z]{2,})$', lines[i]) + split = split_school_first(lines[i+1]) + m2 = re.match(r'^(.+?)([A-Z][a-zA-Z]+)$', lines[i+1]) + m3 = re.match(r'^(Female|Male)(\d{1,2}|K)$', lines[i+2]) + if m1 and m3: + if split: + school_val, first_val = split + elif m2: + school_val, first_val = m2.group(1).strip(), m2.group(2) + else: + continue + extracted['last_name'] = norm_name(m1.group(1)) + extracted['school'] = school_val + extracted['first_name'] = norm_name(first_val) + extracted['gender'] = m3.group(1) + extracted['grade'] = m3.group(2) + break + # Pattern: FirstName, SchoolGradeLastName, Gender (e.g., Michael, Home School7Shuck, Male) for i in range(len(lines) - 2): m1 = re.match(r'^([A-Z][a-z]+)$', lines[i]) @@ -582,11 +856,18 @@ def extract_student_info_from_text(text): m1 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Z][a-z]+)$', lines[i]) m2 = re.match(r'^(.+?)([A-Z][a-z]+)$', lines[i+1]) if m1 and m2: + split = split_school_first(lines[i+1]) + if split: + school_val, first_val = split + elif is_school_only(lines[i+1]): + continue + else: + school_val, first_val = m2.group(1).strip(), m2.group(2) extracted['gender'] = m1.group(1) extracted['grade'] = m1.group(2) extracted['last_name'] = m1.group(3) - extracted['school'] = m2.group(1).strip() - extracted['first_name'] = m2.group(2) + extracted['school'] = school_val + extracted['first_name'] = first_val break # Pattern: GenderLastName, GradeFirstName, School (e.g., MaleKirk, 10Jaihden "JJ", Home School) @@ -723,11 +1004,18 @@ def extract_student_info_from_text(text): m1 = re.match(r'^(Female|Male)(\d{1,2}|K)(' + name_re + r')$', lines[i]) m2 = re.match(r'^(.+?)(' + name_re + r')$', lines[i+1]) if m1 and m2: + split = split_school_first(lines[i+1]) + if split: + school_val, first_val = split + elif is_school_only(lines[i+1]): + continue + else: + school_val, first_val = m2.group(1).strip(), m2.group(2) extracted['gender'] = m1.group(1) extracted['grade'] = m1.group(2) extracted['last_name'] = m1.group(3) - extracted['school'] = m2.group(1).strip() - extracted['first_name'] = m2.group(2) + extracted['school'] = school_val + extracted['first_name'] = first_val break # Pattern: SchoolLastNameFirstName, GenderGrade (e.g., Home SchoolBethelAbigail, Female7) @@ -886,6 +1174,8 @@ def extract_student_info_from_text(text): # Pattern: SchoolFirstName (case-insensitive), GenderGradeLastName (case-insensitive) for i in range(len(lines) - 1): + if is_school_only(lines[i]) or (' ' not in lines[i] and not split_school_first(lines[i])): + continue m1 = re.match(r'^(.+?)([A-Za-z]+)$', lines[i], re.IGNORECASE) m2 = re.match(r'^(Female|Male)(\d{1,2}|K)([A-Za-z]+)$', lines[i+1], re.IGNORECASE) if m1 and m2: @@ -1040,11 +1330,6 @@ def extract_student_info_from_text(text): # Enhanced preprocessing: aggressively split lines with multiple fields stuck together processed_lines = [] - skip_headers = [ - 'student first name:', 'student last name:', 'student grade:', - 'student school:', 'details', 'gender:', 'docusign envelope id:', - 'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian' - ] valid_schools = [ 'Amherst', 'Appomattox', 'Brookville', 'Dunbar', 'E.C. Glass', 'Forest', 'Heritage', 'Home School', 'Jefferson Forest', 'Liberty Christian Academy', 'Linkhorne', 'Monelison', 'Rustburg', 'Sandusky', @@ -1058,6 +1343,15 @@ def extract_student_info_from_text(text): # Ignore header/title/label lines if not orig_line or any(h in orig_line.lower() for h in skip_headers): continue + # Normalize single-word lowercase names + if orig_line.islower() and orig_line.isalpha(): + orig_line = orig_line.title() + # Split school+name concatenations (e.g., "Liberty Christian AcademyWhorley") + split_school = split_school_first(orig_line) + if split_school: + processed_lines.append(split_school[0]) + processed_lines.append(split_school[1]) + continue # Aggressive split: if line starts with a valid school name (with suffix) and is immediately followed by a capitalized name or digit, split split_done = False for school_name in all_school_names: @@ -1313,8 +1607,11 @@ def extract_student_info_from_text(text): if n: school_clean = re.sub(rf'\b{re.escape(n)}\b', '', school_clean).strip() extracted['school'] = school_clean - # Final check: discard invalid schools (pure grade or same as student name) + # Final check: discard invalid schools (pure grade, header text, or same as student name) school_clean = extracted['school'].strip() + if any(x in school_clean.lower() for x in ['docusign envelope id', 'episode registration']): + extracted['school'] = '' + return extracted if re.fullmatch(r'(\d{1,2}|K)', school_clean): extracted['school'] = '' else: