WIP: Improve PDF renaming and extraction logic, aggressive school/name/grade splitting

pdf_renamer.py

@@ -485,6 +485,125 @@ def extract_student_info_from_text(text):
     # Split text into lines for better parsing
     lines = text.split('\n')
 
+    # Enhanced preprocessing: aggressively split lines with multiple fields stuck together
+    processed_lines = []
+    skip_headers = [
+        'student first name:', 'student last name:', 'student grade:',
+        'student school:', 'details', 'gender:', 'docusign envelope id:',
+        'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian'
+    ]
+    school_suffixes = [
+        'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
+    ]
+    for line in lines:
+        orig_line = line.strip()
+        # Ignore header/title/label lines
+        if not orig_line or any(h in orig_line.lower() for h in skip_headers):
+            continue
+        # Special: Handle lines like '6HoskinsMary Emma' (grade, last name, first name) ONLY if the line is exactly that pattern
+        m_special = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)([A-Z][a-z]+\s[A-Z][a-z]+)$', orig_line)
+        if m_special and orig_line == m_special.group(0):
+            processed_lines.append(m_special.group(1))
+            processed_lines.append(m_special.group(2))
+            # Split first and last name if possible
+            names = m_special.group(3).strip().split()
+            if len(names) == 2:
+                processed_lines.append(names[0])
+                processed_lines.append(names[1])
+            else:
+                processed_lines.append(m_special.group(3).strip())
+            continue
+        # Special: Handle lines like 'FemaleNew Covenant' (gender, school) ONLY if the line is exactly that pattern
+        m_special2 = re.match(r'^(Female|Male)([A-Z][a-z]+ [A-Z][a-z]+.*)$', orig_line)
+        if m_special2 and orig_line == m_special2.group(0):
+            processed_lines.append(m_special2.group(1))
+            processed_lines.append(m_special2.group(2).strip())
+            continue
+        # Never treat a number or 'K' as a school
+        if re.fullmatch(r'(\d{1,2}|K)', orig_line):
+            processed_lines.append(orig_line)
+            continue
+        # --- Always split off leading gender ---
+        m_gender = re.match(r'^(Female|Male)(.+)', orig_line)
+        if m_gender:
+            processed_lines.append(m_gender.group(1))
+            orig_line = m_gender.group(2).strip()
+        # --- Split at last school suffix if followed by capitalized name or number ---
+        last_pos = -1
+        last_suffix = ''
+        for suffix in school_suffixes:
+            idx = orig_line.rfind(suffix)
+            if idx != -1 and idx + len(suffix) > last_pos:
+                last_pos = idx + len(suffix)
+                last_suffix = suffix
+        if last_pos != -1 and last_pos < len(orig_line):
+            # Always split at the last suffix: everything after is not part of the school name
+            processed_lines.append(orig_line[:last_pos].strip())
+            trailing = orig_line[last_pos:].strip()
+            if trailing:
+                # Split all words (including lowercase, apostrophes, spaces, etc.) as separate lines
+                for part in re.findall(r"[\w'\-]+", trailing):
+                    processed_lines.append(part)
+            continue
+        # Otherwise, keep line as is
+        processed_lines.append(orig_line)
+        # ...existing code for other patterns...
+        # Pattern: Gender + School + LastName (e.g., 'FemaleBrookville HighMouse')
+        m = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)$', orig_line)
+        if m:
+            processed_lines.append(m.group(1))
+            processed_lines.append(m.group(2).strip())
+            processed_lines.append(m.group(3))
+            continue
+        # Pattern: Grade + FirstName (e.g., '11Mickey')
+        m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', orig_line)
+        if m2:
+            processed_lines.append(m2.group(1))
+            processed_lines.append(m2.group(2))
+            continue
+        # Pattern: Two concatenated names (e.g., 'MaynardJenny')
+        m3 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
+        if m3 and orig_line not in processed_lines:
+            processed_lines.append(m3.group(1))
+            processed_lines.append(m3.group(2))
+            continue
+        # Pattern: Gender + FirstName (e.g., 'MaleJoshua')
+        m7 = re.match(r'^(Female|Male)([A-Z][a-z]+)$', orig_line)
+        if m7:
+            processed_lines.append(m7.group(1))
+            processed_lines.append(m7.group(2))
+            continue
+        # Pattern: Gender + School + FirstName + LastName + Grade (all concatenated)
+        m4 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)(\d{1,2}|K)$', orig_line)
+        if m4:
+            processed_lines.append(m4.group(1))
+            processed_lines.append(m4.group(2).strip())
+            processed_lines.append(m4.group(3))
+            processed_lines.append(m4.group(4))
+            processed_lines.append(m4.group(5))
+            continue
+        # Pattern: Gender + School + FirstName + LastName (no grade)
+        m5 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
+        if m5:
+            processed_lines.append(m5.group(1))
+            processed_lines.append(m5.group(2).strip())
+            processed_lines.append(m5.group(3))
+            processed_lines.append(m5.group(4))
+            continue
+        # Pattern: Grade + Gender + School + FirstName + LastName (e.g., '10FemaleHomeSchoolAbigailDober')
+        m6 = re.match(r'^(\d{1,2}|K)(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
+        if m6:
+            processed_lines.append(m6.group(1))
+            processed_lines.append(m6.group(2))
+            processed_lines.append(m6.group(3).strip())
+            processed_lines.append(m6.group(4))
+            processed_lines.append(m6.group(5))
+            continue
+        # Otherwise, keep line as is
+        processed_lines.append(orig_line)
+    # Remove empty lines
+    lines = [l for l in processed_lines if l.strip()]
+
     # First pass: Look for individual fields that appear on separate lines
     for line in lines:
         line = line.strip()
@@ -509,6 +628,10 @@ def extract_student_info_from_text(text):
         if school_match and len(school_match.group(1)) > 3:  # Avoid false positives
             extracted['school'] = line
 
+        # If line is all uppercase and not a known field, treat as possible school
+        if line.isupper() and len(line) > 5 and not any(x in line for x in ['STUDENT', 'NAME', 'GRADE', 'GENDER', 'DETAILS', 'DOCUSIGN', 'ENVELOPE', 'ID']):
+            extracted['school'] = line.title()
+
     # Second pass: Try various concatenation patterns
     patterns = [
         # Pattern 1: Gender + School + LastName, Grade + FirstName
@@ -622,27 +745,117 @@ def extract_student_info_from_text(text):
             if mapped:
                 extracted.update(mapped)
 
-    # Third pass: Fallback logic for concatenated names and missing fields
-    if not extracted['first_name'] or not extracted['last_name']:
-        # Find all capitalized words that could be names
-        all_text = ' '.join(lines)
-        name_candidates = re.findall(r'\b([A-Z][a-z]+)\b', all_text)
 
-        # Filter out common non-name words
+    # Fourth pass: Try to extract from sequential lines (for cases like: NameName, Grade, School, Gender)
-        common_words = {'Student', 'First', 'Last', 'Name', 'Grade', 'School', 'Gender', 'Date', 'Id', 'Envelope', 'Docusign', 'Male', 'Female', 'Details', 'Registration', 'Episode', 'Page', 'Signature', 'Participant', 'Parent', 'Guardian', 'High', 'Academy', 'Christian', 'Liberty', 'Middle', 'Elementary', 'Catholic', 'Public', 'Private', 'Other', 'Not', 'Listed'}
+    if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
-        name_words = [w for w in name_candidates if w not in common_words and len(w) > 1]
+        for i in range(len(lines) - 3):
+            # Look for pattern: NameName, Grade, School, Gender
+            name_line = lines[i]
+            grade_line = lines[i+1]
+            school_line = lines[i+2]
+            gender_line = lines[i+3]
+            # NameName: two capitalized names concatenated
+            m = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', name_line)
+            if m and re.match(r'^(\d{1,2}|K)$', grade_line) and len(school_line) > 2 and gender_line in ['Male', 'Female']:
+                extracted['first_name'] = m.group(1)
+                extracted['last_name'] = m.group(2)
+                extracted['grade'] = grade_line
+                extracted['school'] = school_line
+                extracted['gender'] = gender_line
+                break
 
-        # Try to assign names based on context
+    # Fifth pass: Try to extract from GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
-        if len(name_words) >= 2:
+    if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
-            if not extracted['first_name']:
+        for i in range(len(lines) - 1):
-                extracted['first_name'] = name_words[0]
+            line1 = lines[i]
-            if not extracted['last_name']:
+            line2 = lines[i+1]
-                extracted['last_name'] = name_words[-1]  # Usually last name is last
+            m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', line1)
+            m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', line2)
+            if m1 and m2:
+                extracted['gender'] = m1.group(1)
+                extracted['school'] = m1.group(2).strip()
+                extracted['last_name'] = m1.group(3)
+                extracted['grade'] = m2.group(1)
+                extracted['first_name'] = m2.group(2)
+                break
 
     # Final validation and cleanup
     required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
     if all(extracted.get(field) for field in required_fields):
         # Clean up school names (remove extra spaces, standardize)
+        extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
+        # --- Improved post-processing for stuck-together school/grade/last name ---
+        school_suffixes = [
+            'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
+        ]
+        # Find the last occurrence of any school suffix and split there
+        import re
+        school = extracted['school']
+        last_pos = -1
+        last_suffix = ''
+        for suffix in school_suffixes:
+            idx = school.rfind(suffix)
+            if idx != -1 and idx + len(suffix) > last_pos:
+                last_pos = idx + len(suffix)
+                last_suffix = suffix
+        if last_pos != -1:
+            # School name is everything up to and including the suffix
+            extracted['school'] = school[:last_pos].strip()
+            trailing = school[last_pos:].strip().replace(' ', '')
+            # Regex: (grade?)(all capitalized names?)(anything else?)
+            m = re.match(r'^(\d{1,2}|K)?((?:[A-Z][a-z]+)+)(.*)$', trailing)
+            if m:
+                grade = m.group(1)
+                names_str = m.group(2)
+                leftover = m.group(3)
+                if grade and not extracted['grade']:
+                    extracted['grade'] = grade
+                # Split names_str into tokens of capitalized names
+                names = re.findall(r'[A-Z][a-z]+', names_str)
+                if names:
+                    extracted['last_name'] = names[0]
+                    if len(names) > 1:
+                        extracted['first_name'] = ' '.join(names[1:])
+                # If there is leftover text, append to first name
+                if leftover:
+                    if extracted['first_name']:
+                        extracted['first_name'] += leftover
+                    else:
+                        extracted['first_name'] = leftover
+        # Final check: if school is just a number or a name, discard it
+        if re.fullmatch(r'(\d{1,2}|K)', extracted['school']) or re.fullmatch(r'([A-Z][a-z]+|[A-Z\-]+|[A-Z][a-z]+\s[A-Z][a-z]+)+', extracted['school']):
+            extracted['school'] = ''
+        return extracted
+
+    # Fallback: Try to extract fields from any order in the lines (for highly concatenated or out-of-order data)
+    # Look for all capitalized words (possible names)
+    names = []
+    for line in lines:
+        for word in line.split():
+            if re.match(r'^[A-Z][a-z]+$', word):
+                names.append(word)
+    if len(names) >= 2:
+        if not extracted['first_name']:
+            extracted['first_name'] = names[0]
+        if not extracted['last_name']:
+            extracted['last_name'] = names[1]
+    # Look for grade
+    for line in lines:
+        m = re.search(r'(\d{1,2}|K)', line)
+        if m and not extracted['grade']:
+            extracted['grade'] = m.group(1)
+    # Look for gender
+    for line in lines:
+        if 'female' in line.lower() and not extracted['gender']:
+            extracted['gender'] = 'Female'
+        elif 'male' in line.lower() and not extracted['gender']:
+            extracted['gender'] = 'Male'
+    # Look for school (any line with 'school', 'academy', 'high', etc.)
+    for line in lines:
+        if any(x in line.lower() for x in ['school', 'academy', 'high', 'middle', 'elementary', 'christian', 'catholic', 'public', 'private']):
+            extracted['school'] = line.strip()
+            break
+    if all(extracted.get(field) for field in required_fields):
         extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
         return extracted
 
@@ -810,6 +1023,8 @@ def process_job(job_id, files):
                 with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
                     for root, dirs, files_in_dir in os.walk(episode_dir):
                         for file in files_in_dir:
+                            if file.startswith('.'):
+                                continue  # Ignore dotfiles
                             file_path = os.path.join(root, file)
                             arcname = os.path.relpath(file_path, episode_dir)
                             print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
@@ -1032,7 +1247,7 @@ def upload_files():
         
         file_list = []
         for file in files:
-            if file and allowed_file(file.filename):
+            if file and not file.filename.startswith('.') and allowed_file(file.filename):
                 filename = secure_filename(file.filename)
                 filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
                 file.save(filepath)