WIP: Improve PDF renaming and extraction logic, aggressive school/name/grade splitting
This commit is contained in:
nmemmert committed
1 parent
4a0dad00e0
commit
f6e6101ade
2 files changed
+6039
-26
No files matched your search
+230
-15
@@ -485,6 +485,125 @@ def extract_student_info_from_text(text):
|
||||
# Split text into lines for better parsing
|
||||
lines = text.split('\n')
|
||||
|
||||
# Enhanced preprocessing: aggressively split lines with multiple fields stuck together
|
||||
processed_lines = []
|
||||
skip_headers = [
|
||||
'student first name:', 'student last name:', 'student grade:',
|
||||
'student school:', 'details', 'gender:', 'docusign envelope id:',
|
||||
'episode registration', 'form', 'page', 'date', 'signature', 'parent', 'guardian'
|
||||
]
|
||||
school_suffixes = [
|
||||
'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
|
||||
]
|
||||
for line in lines:
|
||||
orig_line = line.strip()
|
||||
# Ignore header/title/label lines
|
||||
if not orig_line or any(h in orig_line.lower() for h in skip_headers):
|
||||
continue
|
||||
# Special: Handle lines like '6HoskinsMary Emma' (grade, last name, first name) ONLY if the line is exactly that pattern
|
||||
m_special = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)([A-Z][a-z]+\s[A-Z][a-z]+)$', orig_line)
|
||||
if m_special and orig_line == m_special.group(0):
|
||||
processed_lines.append(m_special.group(1))
|
||||
processed_lines.append(m_special.group(2))
|
||||
# Split first and last name if possible
|
||||
names = m_special.group(3).strip().split()
|
||||
if len(names) == 2:
|
||||
processed_lines.append(names[0])
|
||||
processed_lines.append(names[1])
|
||||
else:
|
||||
processed_lines.append(m_special.group(3).strip())
|
||||
continue
|
||||
# Special: Handle lines like 'FemaleNew Covenant' (gender, school) ONLY if the line is exactly that pattern
|
||||
m_special2 = re.match(r'^(Female|Male)([A-Z][a-z]+ [A-Z][a-z]+.*)$', orig_line)
|
||||
if m_special2 and orig_line == m_special2.group(0):
|
||||
processed_lines.append(m_special2.group(1))
|
||||
processed_lines.append(m_special2.group(2).strip())
|
||||
continue
|
||||
# Never treat a number or 'K' as a school
|
||||
if re.fullmatch(r'(\d{1,2}|K)', orig_line):
|
||||
processed_lines.append(orig_line)
|
||||
continue
|
||||
# --- Always split off leading gender ---
|
||||
m_gender = re.match(r'^(Female|Male)(.+)', orig_line)
|
||||
if m_gender:
|
||||
processed_lines.append(m_gender.group(1))
|
||||
orig_line = m_gender.group(2).strip()
|
||||
# --- Split at last school suffix if followed by capitalized name or number ---
|
||||
last_pos = -1
|
||||
last_suffix = ''
|
||||
for suffix in school_suffixes:
|
||||
idx = orig_line.rfind(suffix)
|
||||
if idx != -1 and idx + len(suffix) > last_pos:
|
||||
last_pos = idx + len(suffix)
|
||||
last_suffix = suffix
|
||||
if last_pos != -1 and last_pos < len(orig_line):
|
||||
# Always split at the last suffix: everything after is not part of the school name
|
||||
processed_lines.append(orig_line[:last_pos].strip())
|
||||
trailing = orig_line[last_pos:].strip()
|
||||
if trailing:
|
||||
# Split all words (including lowercase, apostrophes, spaces, etc.) as separate lines
|
||||
for part in re.findall(r"[\w'\-]+", trailing):
|
||||
processed_lines.append(part)
|
||||
continue
|
||||
# Otherwise, keep line as is
|
||||
processed_lines.append(orig_line)
|
||||
# ...existing code for other patterns...
|
||||
# Pattern: Gender + School + LastName (e.g., 'FemaleBrookville HighMouse')
|
||||
m = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)$', orig_line)
|
||||
if m:
|
||||
processed_lines.append(m.group(1))
|
||||
processed_lines.append(m.group(2).strip())
|
||||
processed_lines.append(m.group(3))
|
||||
continue
|
||||
# Pattern: Grade + FirstName (e.g., '11Mickey')
|
||||
m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', orig_line)
|
||||
if m2:
|
||||
processed_lines.append(m2.group(1))
|
||||
processed_lines.append(m2.group(2))
|
||||
continue
|
||||
# Pattern: Two concatenated names (e.g., 'MaynardJenny')
|
||||
m3 = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
|
||||
if m3 and orig_line not in processed_lines:
|
||||
processed_lines.append(m3.group(1))
|
||||
processed_lines.append(m3.group(2))
|
||||
continue
|
||||
# Pattern: Gender + FirstName (e.g., 'MaleJoshua')
|
||||
m7 = re.match(r'^(Female|Male)([A-Z][a-z]+)$', orig_line)
|
||||
if m7:
|
||||
processed_lines.append(m7.group(1))
|
||||
processed_lines.append(m7.group(2))
|
||||
continue
|
||||
# Pattern: Gender + School + FirstName + LastName + Grade (all concatenated)
|
||||
m4 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)(\d{1,2}|K)$', orig_line)
|
||||
if m4:
|
||||
processed_lines.append(m4.group(1))
|
||||
processed_lines.append(m4.group(2).strip())
|
||||
processed_lines.append(m4.group(3))
|
||||
processed_lines.append(m4.group(4))
|
||||
processed_lines.append(m4.group(5))
|
||||
continue
|
||||
# Pattern: Gender + School + FirstName + LastName (no grade)
|
||||
m5 = re.match(r'^(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
|
||||
if m5:
|
||||
processed_lines.append(m5.group(1))
|
||||
processed_lines.append(m5.group(2).strip())
|
||||
processed_lines.append(m5.group(3))
|
||||
processed_lines.append(m5.group(4))
|
||||
continue
|
||||
# Pattern: Grade + Gender + School + FirstName + LastName (e.g., '10FemaleHomeSchoolAbigailDober')
|
||||
m6 = re.match(r'^(\d{1,2}|K)(Female|Male)([A-Z][a-z]+(?: [A-Z][a-z]+)*)([A-Z][a-z]+)([A-Z][a-z]+)$', orig_line)
|
||||
if m6:
|
||||
processed_lines.append(m6.group(1))
|
||||
processed_lines.append(m6.group(2))
|
||||
processed_lines.append(m6.group(3).strip())
|
||||
processed_lines.append(m6.group(4))
|
||||
processed_lines.append(m6.group(5))
|
||||
continue
|
||||
# Otherwise, keep line as is
|
||||
processed_lines.append(orig_line)
|
||||
# Remove empty lines
|
||||
lines = [l for l in processed_lines if l.strip()]
|
||||
|
||||
# First pass: Look for individual fields that appear on separate lines
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
@@ -509,6 +628,10 @@ def extract_student_info_from_text(text):
|
||||
if school_match and len(school_match.group(1)) > 3: # Avoid false positives
|
||||
extracted['school'] = line
|
||||
|
||||
# If line is all uppercase and not a known field, treat as possible school
|
||||
if line.isupper() and len(line) > 5 and not any(x in line for x in ['STUDENT', 'NAME', 'GRADE', 'GENDER', 'DETAILS', 'DOCUSIGN', 'ENVELOPE', 'ID']):
|
||||
extracted['school'] = line.title()
|
||||
|
||||
# Second pass: Try various concatenation patterns
|
||||
patterns = [
|
||||
# Pattern 1: Gender + School + LastName, Grade + FirstName
|
||||
@@ -622,27 +745,117 @@ def extract_student_info_from_text(text):
|
||||
if mapped:
|
||||
extracted.update(mapped)
|
||||
|
||||
# Third pass: Fallback logic for concatenated names and missing fields
|
||||
if not extracted['first_name'] or not extracted['last_name']:
|
||||
# Find all capitalized words that could be names
|
||||
all_text = ' '.join(lines)
|
||||
name_candidates = re.findall(r'\b([A-Z][a-z]+)\b', all_text)
|
||||
|
||||
# Filter out common non-name words
|
||||
common_words = {'Student', 'First', 'Last', 'Name', 'Grade', 'School', 'Gender', 'Date', 'Id', 'Envelope', 'Docusign', 'Male', 'Female', 'Details', 'Registration', 'Episode', 'Page', 'Signature', 'Participant', 'Parent', 'Guardian', 'High', 'Academy', 'Christian', 'Liberty', 'Middle', 'Elementary', 'Catholic', 'Public', 'Private', 'Other', 'Not', 'Listed'}
|
||||
name_words = [w for w in name_candidates if w not in common_words and len(w) > 1]
|
||||
# Fourth pass: Try to extract from sequential lines (for cases like: NameName, Grade, School, Gender)
|
||||
if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
|
||||
for i in range(len(lines) - 3):
|
||||
# Look for pattern: NameName, Grade, School, Gender
|
||||
name_line = lines[i]
|
||||
grade_line = lines[i+1]
|
||||
school_line = lines[i+2]
|
||||
gender_line = lines[i+3]
|
||||
# NameName: two capitalized names concatenated
|
||||
m = re.match(r'^([A-Z][a-z]+)([A-Z][a-z]+)$', name_line)
|
||||
if m and re.match(r'^(\d{1,2}|K)$', grade_line) and len(school_line) > 2 and gender_line in ['Male', 'Female']:
|
||||
extracted['first_name'] = m.group(1)
|
||||
extracted['last_name'] = m.group(2)
|
||||
extracted['grade'] = grade_line
|
||||
extracted['school'] = school_line
|
||||
extracted['gender'] = gender_line
|
||||
break
|
||||
|
||||
# Try to assign names based on context
|
||||
if len(name_words) >= 2:
|
||||
if not extracted['first_name']:
|
||||
extracted['first_name'] = name_words[0]
|
||||
if not extracted['last_name']:
|
||||
extracted['last_name'] = name_words[-1] # Usually last name is last
|
||||
# Fifth pass: Try to extract from GenderSchoolLastName, GradeFirstName (e.g., FemaleBrookville HighMouse, 11Mickey)
|
||||
if not all(extracted.get(field) for field in ['first_name', 'last_name', 'grade', 'school', 'gender']):
|
||||
for i in range(len(lines) - 1):
|
||||
line1 = lines[i]
|
||||
line2 = lines[i+1]
|
||||
m1 = re.match(r'^(Female|Male)(.+?)([A-Z][a-z]+)$', line1)
|
||||
m2 = re.match(r'^(\d{1,2}|K)([A-Z][a-z]+)$', line2)
|
||||
if m1 and m2:
|
||||
extracted['gender'] = m1.group(1)
|
||||
extracted['school'] = m1.group(2).strip()
|
||||
extracted['last_name'] = m1.group(3)
|
||||
extracted['grade'] = m2.group(1)
|
||||
extracted['first_name'] = m2.group(2)
|
||||
break
|
||||
|
||||
# Final validation and cleanup
|
||||
required_fields = ['first_name', 'last_name', 'grade', 'school', 'gender']
|
||||
if all(extracted.get(field) for field in required_fields):
|
||||
# Clean up school names (remove extra spaces, standardize)
|
||||
extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
|
||||
# --- Improved post-processing for stuck-together school/grade/last name ---
|
||||
school_suffixes = [
|
||||
'High', 'School', 'Academy', 'Middle', 'Elementary', 'Christian', 'Catholic', 'Public', 'Private'
|
||||
]
|
||||
# Find the last occurrence of any school suffix and split there
|
||||
import re
|
||||
school = extracted['school']
|
||||
last_pos = -1
|
||||
last_suffix = ''
|
||||
for suffix in school_suffixes:
|
||||
idx = school.rfind(suffix)
|
||||
if idx != -1 and idx + len(suffix) > last_pos:
|
||||
last_pos = idx + len(suffix)
|
||||
last_suffix = suffix
|
||||
if last_pos != -1:
|
||||
# School name is everything up to and including the suffix
|
||||
extracted['school'] = school[:last_pos].strip()
|
||||
trailing = school[last_pos:].strip().replace(' ', '')
|
||||
# Regex: (grade?)(all capitalized names?)(anything else?)
|
||||
m = re.match(r'^(\d{1,2}|K)?((?:[A-Z][a-z]+)+)(.*)$', trailing)
|
||||
if m:
|
||||
grade = m.group(1)
|
||||
names_str = m.group(2)
|
||||
leftover = m.group(3)
|
||||
if grade and not extracted['grade']:
|
||||
extracted['grade'] = grade
|
||||
# Split names_str into tokens of capitalized names
|
||||
names = re.findall(r'[A-Z][a-z]+', names_str)
|
||||
if names:
|
||||
extracted['last_name'] = names[0]
|
||||
if len(names) > 1:
|
||||
extracted['first_name'] = ' '.join(names[1:])
|
||||
# If there is leftover text, append to first name
|
||||
if leftover:
|
||||
if extracted['first_name']:
|
||||
extracted['first_name'] += leftover
|
||||
else:
|
||||
extracted['first_name'] = leftover
|
||||
# Final check: if school is just a number or a name, discard it
|
||||
if re.fullmatch(r'(\d{1,2}|K)', extracted['school']) or re.fullmatch(r'([A-Z][a-z]+|[A-Z\-]+|[A-Z][a-z]+\s[A-Z][a-z]+)+', extracted['school']):
|
||||
extracted['school'] = ''
|
||||
return extracted
|
||||
|
||||
# Fallback: Try to extract fields from any order in the lines (for highly concatenated or out-of-order data)
|
||||
# Look for all capitalized words (possible names)
|
||||
names = []
|
||||
for line in lines:
|
||||
for word in line.split():
|
||||
if re.match(r'^[A-Z][a-z]+$', word):
|
||||
names.append(word)
|
||||
if len(names) >= 2:
|
||||
if not extracted['first_name']:
|
||||
extracted['first_name'] = names[0]
|
||||
if not extracted['last_name']:
|
||||
extracted['last_name'] = names[1]
|
||||
# Look for grade
|
||||
for line in lines:
|
||||
m = re.search(r'(\d{1,2}|K)', line)
|
||||
if m and not extracted['grade']:
|
||||
extracted['grade'] = m.group(1)
|
||||
# Look for gender
|
||||
for line in lines:
|
||||
if 'female' in line.lower() and not extracted['gender']:
|
||||
extracted['gender'] = 'Female'
|
||||
elif 'male' in line.lower() and not extracted['gender']:
|
||||
extracted['gender'] = 'Male'
|
||||
# Look for school (any line with 'school', 'academy', 'high', etc.)
|
||||
for line in lines:
|
||||
if any(x in line.lower() for x in ['school', 'academy', 'high', 'middle', 'elementary', 'christian', 'catholic', 'public', 'private']):
|
||||
extracted['school'] = line.strip()
|
||||
break
|
||||
if all(extracted.get(field) for field in required_fields):
|
||||
extracted['school'] = re.sub(r'\s+', ' ', extracted['school']).strip()
|
||||
return extracted
|
||||
|
||||
@@ -810,6 +1023,8 @@ def process_job(job_id, files):
|
||||
with zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) as zip_out:
|
||||
for root, dirs, files_in_dir in os.walk(episode_dir):
|
||||
for file in files_in_dir:
|
||||
if file.startswith('.'):
|
||||
continue # Ignore dotfiles
|
||||
file_path = os.path.join(root, file)
|
||||
arcname = os.path.relpath(file_path, episode_dir)
|
||||
print(f"DEBUG: Adding to ZIP: {file_path} as {arcname}")
|
||||
@@ -1032,7 +1247,7 @@ def upload_files():
|
||||
|
||||
file_list = []
|
||||
for file in files:
|
||||
if file and allowed_file(file.filename):
|
||||
if file and not file.filename.startswith('.') and allowed_file(file.filename):
|
||||
filename = secure_filename(file.filename)
|
||||
filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
|
||||
file.save(filepath)
|
||||
|
||||
Reference in new issue
Block a user