diff --git a/.env b/.env new file mode 100644 index 0000000..c5dbdc8 --- /dev/null +++ b/.env @@ -0,0 +1,24 @@ +# Microsoft Entra (Azure AD) Configuration +# Get these values from your Azure AD app registration in the Azure portal + +# Azure AD Tenant ID (Directory ID) +AZURE_TENANT_ID=98367e69-bd4c-47c6-90ef-683461397b64 + +# Azure AD Client ID (Application ID) +AZURE_CLIENT_ID=5b2137d0-b143-4f1a-85b0-d740d62faebc + +# Azure AD Client Secret +AZURE_CLIENT_SECRET=Ql88Q~1wo-_L8vg-csgL4GFhLR_A8klPvphHUcyx + +# Azure AD Authority URL (usually https://login.microsoftonline.com/{tenant_id}) +AZURE_AUTHORITY=https://login.microsoftonline.com/98367e69-bd4c-47c6-90ef-683461397b64 + +# Redirect URI (must match what's configured in Azure AD app registration) +# For local development, use: http://localhost:8080/auth/callback +AZURE_REDIRECT_URI=http://localhost:5000/auth/callback + +# Scopes to request (openid, profile, email are common) +AZURE_SCOPES=openid profile email + +# Flask Configuration +SECRET_KEY=your-secret-key-here-change-this-in-production \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b1f18b5 --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Microsoft Entra (Azure AD) Configuration +# Copy this file to .env and fill in your actual values + +# Azure AD Tenant ID (Directory ID) +AZURE_TENANT_ID= + +# Azure AD Client ID (Application ID) +AZURE_CLIENT_ID= + +# Azure AD Client Secret +AZURE_CLIENT_SECRET= + +# Azure AD Authority URL (usually https://login.microsoftonline.com/{tenant_id}) +AZURE_AUTHORITY= + +# Redirect URI (must match what's configured in Azure AD app registration) +AZURE_REDIRECT_URI=http://localhost:5000/auth/callback + +# Scopes to request (openid, profile, email are common) +AZURE_SCOPES=openid profile email + +# Flask Configuration +SECRET_KEY=your-secret-key-here-change-this-in-production \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..62d7cbb --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# Environment variables +.env + +# Python +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python +env/ +venv/ +.venv/ +pip-log.txt +pip-delete-this-directory.txt +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.log +.git +.mypy_cache +.pytest_cache +.hypothesis + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Application specific +results/ +status.txt +jobs.json \ No newline at end of file diff --git a/AZURE_AD_SETUP.md b/AZURE_AD_SETUP.md new file mode 100644 index 0000000..5d25b8f --- /dev/null +++ b/AZURE_AD_SETUP.md @@ -0,0 +1,93 @@ +# Microsoft Entra Authentication Setup Guide + +This application uses Microsoft Entra (Azure Active Directory) for authentication. Follow these steps to configure it: + +## 1. Create an Azure AD App Registration + +1. Go to the [Azure Portal](https://portal.azure.com) +2. Navigate to **Azure Active Directory** > **App registrations** +3. Click **New registration** +4. Enter a name for your app (e.g., "PDF Processor") +5. For **Supported account types**, select "Accounts in this organizational directory only" +6. For **Redirect URI**, select "Web" and enter: `http://localhost:5000/auth/callback` +7. Click **Register** + +## 2. Configure Authentication + +1. In your app registration, go to **Authentication** +2. Under **Redirect URIs**, ensure `http://localhost:5000/auth/callback` is listed +3. Under **Implicit grant and hybrid flows**, check: + - [x] Access tokens + - [x] ID tokens +4. Click **Save** + +## 3. Get Application Credentials + +1. Go to **Overview** in your app registration +2. Copy the **Application (client) ID** - this is your `AZURE_CLIENT_ID` +3. Copy the **Directory (tenant) ID** - this is your `AZURE_TENANT_ID` + +## 4. Create a Client Secret + +1. Go to **Certificates & secrets** +2. Click **New client secret** +3. Enter a description and select an expiration +4. Click **Add** +5. **Important**: Copy the secret value immediately - this is your `AZURE_CLIENT_SECRET` + +## 5. Configure API Permissions + +1. Go to **API permissions** +2. Click **Add a permission** +3. Select **Microsoft Graph** +4. Select **Delegated permissions** +5. Add these permissions: + - `openid` + - `profile` + - `email` +6. Click **Add permissions** +7. Click **Grant admin consent** for your organization + +## 6. Configure Environment Variables + +Create a `.env` file in the project root with your Azure AD settings: + +```env +# Azure AD Configuration +AZURE_TENANT_ID=your-tenant-id-here +AZURE_CLIENT_ID=your-client-id-here +AZURE_CLIENT_SECRET=your-client-secret-here +AZURE_AUTHORITY=https://login.microsoftonline.com/your-tenant-id-here +AZURE_REDIRECT_URI=http://localhost:5000/auth/callback +AZURE_SCOPES=openid profile email +``` + +## 7. Update Redirect URI for Production + +When deploying to production, update the redirect URI in both: +- Azure AD app registration +- Your `.env` file (change localhost to your production domain) + +**Note**: For local development, the app runs on port 5000, so use `http://localhost:5000/auth/callback` as the redirect URI. + +## 8. Test the Authentication + +1. Start the application: `python app.py` +2. Visit `http://localhost:5000` +3. Click "Login with Microsoft" +4. You should be redirected to Microsoft login +5. After authentication, you'll be redirected back to the app + +## Troubleshooting + +- **"invalid_client" error**: Check your client ID and secret +- **"invalid_request" error**: Verify the redirect URI matches exactly +- **Permission denied**: Ensure you've granted admin consent for the API permissions +- **Token errors**: Check that your tenant ID is correct + +## Security Notes + +- Never commit the `.env` file to version control +- Use strong client secrets and rotate them regularly +- In production, use HTTPS for all authentication flows +- Consider implementing token refresh logic for long-running sessions \ No newline at end of file diff --git a/Find_Missing_Addresses.sh b/Find_Missing_Addresses.sh new file mode 100755 index 0000000..f898f90 --- /dev/null +++ b/Find_Missing_Addresses.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +#Get the folder with the pdf documents in it. +#echo "" +#echo "This will sort out the addresses in all the Statement PDF's into 2 folders so that you can then run the Merge file." +#echo "To get started please answer 2 questions." +#while : +#do +# echo -e "Where is the directory with ALL the pdf statments?" +# read PDFDIR +# if [ -d $PDFDIR ] +# then +# break +# else +# echo "Directory not found." +# fi +#done +#Get the folder for the output file. +#echo "" +#while : +#do +# echo -e "What output directory do you want for the sorted pdf's? This will sort into 2 folders, Missing_Addresses and Address" +# read OUTPUTFOLDER +# if [ -d $OUTPUTFOLDER ] +# then +# break +# else +# echo "Directory not found." +# fi +#done + +#Create the csv and headers for the Addresses. It is tab separated. +mkdir $OUTPUTFOLDER/Addresses +touch $OUTPUTFOLDER/Addresses/Addresses.csv +echo -e "File,User,Street,City,State,Zip" > $OUTPUTFOLDER/Addresses/Addresses.csv + +#Create csv with all the files with missing addresses +mkdir $OUTPUTFOLDER/Missing_Addresses +touch $OUTPUTFOLDER/Missing_Addresses/Missing_Addresses.csv + +echo -e "File\tUser" > $OUTPUTFOLDER/Missing_Addresses/Missing_Addresses.csv + +cd $PDFDIR +#FOLDER=$(ls | grep *.pdf) +#echo $FOLDER +#*.pdf represensts All files in the current directory with a pdf extention. +for f in $PDFDIR/* +do + echo "Processing file $f." + #pdftotext converst the file to text and places it in the file name temp + pdftotext "$f" - > TEMP + #assigns the lines with relevant values to their variable + USER=$(sed -n '12p' TEMP) + ADDRESS1=$(sed -n '13p' TEMP) + ADDRESS2=$(sed -n '14p' TEMP) + echo -e "User is $USER" + echo -e "Address is $ADDRESS1 $ADDRESS2" + if [ -z "$ADDRESS1" ] + then + #Move the file with the Missing addresses to the Missing Address folder and make an note in its csv. + echo "Copying file to the folder called Missing_Addresses" + cp "$f" "$OUTPUTFOLDER"/Missing_Addresses/ + echo -e "$f\t$USER" >> "$OUTPUTFOLDER"/Missing_Addresses/Missing_Addresses.csv + #Adds the row of variables to the csv. >> appends to the file. \t represents tabs + else + # Split Addresses + CITY=$(echo $ADDRESS2 | cut -d, -f1) + ADDRESS2=$(echo $ADDRESS2 | sed -e "s/$CITY, //") + + STATE=$(echo $ADDRESS2 | cut -d" " -f1) + ZIP=$(echo $ADDRESS2 | cut -d" " -f2) + + echo -e "$f,$USER,$ADDRESS1,$CITY,$STATE,$ZIP" >> $OUTPUTFOLDER/Addresses/Addresses.csv + cp "$f" $OUTPUTFOLDER/Addresses/ + echo "Copying file to the folder called Addresses" + fi +done +echo "All Done!" You can find all the Files in $OUTPUTFOLDER!" +echo " " +echo " " + +exit diff --git a/README.md b/README.md index fcc5726..426f94a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,209 @@ -# EOY PDF Sortandmerge +# End of Year Statements PDF Processor -Project to sort, filter, and merge pdf files. \ No newline at end of file +A web application for sorting and merging PDF statements by address and page count, optimized for processing large batches of documents. + +## Features + +- Upload individual PDFs, ZIP archives, or entire folders +- Automatic address extraction and sorting +- Parallel processing for maximum speed +- Real-time progress tracking +- Web-based interface with progress bars and status updates +- Asynchronous processing to prevent timeouts +- **Microsoft Entra (Azure AD) authentication for secure access** + +## Authentication + +This application requires Microsoft Entra (Azure Active Directory) authentication. Before running the application, you must: + +1. Set up an Azure AD app registration +2. Configure authentication settings in `.env` +3. Grant necessary API permissions + +See [AZURE_AD_SETUP.md](AZURE_AD_SETUP.md) for detailed setup instructions. + +## Server Recommendations + +### VMware Virtual Machine Specifications + +For optimal performance with large PDF batches (1000+ files), we recommend the following VM configuration: + +#### Minimum Requirements +- **vCPUs**: 4 cores +- **RAM**: 8 GB +- **Storage**: 50 GB SSD +- **OS**: Ubuntu Server 22.04 LTS (or CentOS 8+) + +#### Recommended for Production +- **vCPUs**: 8-16 cores (more cores = faster processing) +- **RAM**: 16-32 GB (PDFs load into memory) +- **Storage**: 100 GB NVMe SSD +- **Network**: 1 Gbps connection + +#### High-Performance Setup (for 5000+ PDFs) +- **vCPUs**: 16-32 cores +- **RAM**: 64 GB +- **Storage**: 200 GB NVMe SSD +- **OS**: Ubuntu Server 22.04 LTS + +### VMware Configuration Tips +1. Enable CPU hot-add and memory hot-plug for future upgrades +2. Use VMXNET3 network adapter for better performance +3. Allocate storage from a high-performance datastore +4. Enable VMware Tools for optimal integration + +## Installation Instructions + +### 1. VM Setup +1. Create a new VM in VMware with the specifications above +2. Install Ubuntu Server 22.04 LTS +3. Update the system: + ```bash + sudo apt update && sudo apt upgrade -y + ``` + +### 2. Install Python and Dependencies +```bash +# Install Python 3.10+ +sudo apt install python3 python3-pip python3-venv -y + +# Create application directory +sudo mkdir -p /opt/pdf-processor +sudo chown $USER:$USER /opt/pdf-processor +cd /opt/pdf-processor + +# Clone or copy the application files +# (Assuming you have the files ready) + +# Create virtual environment +python3 -m venv venv +source venv/bin/activate + +# Install dependencies +pip install -r requirements.txt +``` + +### 3. Install Production Server (Gunicorn) +```bash +pip install gunicorn + +# Create systemd service +sudo nano /etc/systemd/system/pdf-processor.service +``` + +Add the following content: +```ini +[Unit] +Description=PDF Processor Web App +After=network.target + +[Service] +User=your-username +Group=your-group +WorkingDirectory=/opt/pdf-processor +Environment="PATH=/opt/pdf-processor/venv/bin" +ExecStart=/opt/pdf-processor/venv/bin/gunicorn -w 4 -b 0.0.0.0:8000 app:app +Restart=always + +[Install] +WantedBy=multi-user.target +``` + +```bash +# Enable and start service +sudo systemctl enable pdf-processor +sudo systemctl start pdf-processor + +# Check status +sudo systemctl status pdf-processor +``` + +### 4. Configure Nginx (Optional, for Production) +```bash +sudo apt install nginx -y + +# Create nginx config +sudo nano /etc/nginx/sites-available/pdf-processor +``` + +Add: +```nginx +server { + listen 80; + server_name your-server-ip; + + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} +``` + +```bash +sudo ln -s /etc/nginx/sites-available/pdf-processor /etc/nginx/sites-enabled/ +sudo nginx -t +sudo systemctl restart nginx +``` + +### 5. Access the Application +- Open your web browser and navigate to `http://your-vm-ip:8000` (or port 80 if using Nginx) +- Upload your PDF files and monitor the processing progress + +## Performance Tuning + +### Gunicorn Workers +Adjust the number of workers based on your CPU cores: +```bash +gunicorn -w $(nproc) -b 0.0.0.0:8000 app:app +``` + +### Python Optimization +The app automatically uses all available CPU cores for parallel PDF processing. For very large batches, consider: + +1. Increasing system file limits: + ```bash + sudo nano /etc/security/limits.conf + # Add: * soft nofile 65536 + # Add: * hard nofile 65536 + ``` + +2. Tuning kernel parameters for better I/O: + ```bash + sudo nano /etc/sysctl.conf + # Add: vm.dirty_ratio = 10 + # Add: vm.dirty_background_ratio = 5 + sudo sysctl -p + ``` + +## Troubleshooting + +### Common Issues +1. **413 Request Entity Too Large**: The upload limit is set to 1000TB, but check client-side limits +2. **Processing Stuck**: Check server logs with `sudo journalctl -u pdf-processor -f` +3. **Memory Issues**: Monitor with `htop` and increase RAM if needed +4. **Slow Processing**: Ensure all CPU cores are being utilized + +### Monitoring +```bash +# Check CPU usage during processing +htop + +# Monitor application logs +sudo journalctl -u pdf-processor -f + +# Check disk I/O +iotop +``` + +## Security Considerations +- Run behind a reverse proxy (Nginx) in production +- Use HTTPS with SSL certificates +- Restrict access with firewall rules +- Regularly update dependencies +- Monitor for resource usage + +## Support +For issues or questions, check the application logs and ensure your VM meets the minimum requirements. \ No newline at end of file diff --git a/Removing_Files.sh b/Removing_Files.sh new file mode 100755 index 0000000..c1c44bb --- /dev/null +++ b/Removing_Files.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +#Get the folder with the pdf documents in it. +while : +do + echo -e "What is the directory with the pdfs?" + #read PDFDIR + PDFDIR="/home/btmccrea/Downloads/pdfs" + if [ -d $PDFDIR ] + then + break + else + echo -e "Directory not found." + fi +done +#Get the folder for the output file. +while : +do + echo -e "What is the output directory for the pdf's with missing addresses?" + #read OUTPUTFOLDER + OUTPUTFOLDER="/home/btmccrea/Downloads/output/Final" + if [ -d $OUTPUTFOLDER ] + then + break + else + echo -e "Directory not found." + fi +done +#Get the folder with the csv +while : +do + echo -e "What path and file with?" + #read CSVFILE + if [ -f $CSVFILE ] + then + break + else + echo -e "File not found." + fi +done + +#import list from csv +echo "Processing csv." +while IFS=$'\t' read -r FILE +do + echo "FILE $FILE" +# echo "user $USER" +# echo "match $MATCH" +# echo "${array[0]}" +# if [[ "$MATCH" == "Match-Mailable" ]] +# then +# +# echo -e "Processing file $FILE"lY + cp $PDFDIR/"$FILE" $OUTPUTFOLDER +# else +# echo "no match" +# fi +done < /home/btmccrea/Downloads/Smarty_Streets.csv +#FILE=$(cat /home/btmccrea/Downloads/Smarty_Streets.csv) +#echo $FILE +#IFS=$'\t' +#for f in Smarty_Streets.csv +#do +# echo "doing file $f" +# cp $PDFDIR/"$f" $OUTPUTFOLDER +#done + +#cd to and then remove files diff --git a/Run.sh b/Run.sh new file mode 100644 index 0000000..d1133da --- /dev/null +++ b/Run.sh @@ -0,0 +1,94 @@ + +echo "Welcome! It must be the start of a new year! In order to get started with sorting and merging the PDF documents I need to know a little more information." +echo +echo +read -p 'Where is the directory that has the PDF statments with address? ' PDFDIR +#read PDFDIR +#Get the folder for the output file. +echo +read -p 'What output directory do you want for the sorted PDFs? ' OUTPUTFOLDER +#read OUTPUTFOLDER +echo +echo "The PDFs statments are in $PDFDIR?" +echo "The output folder will be $OUTPUTFOLDER." +echo +read -p "Press Enter to continue" $OUTPUTFOLDER/Addresses/Addresses.csv + +#Create csv with all the files with missing addresses +mkdir $OUTPUTFOLDER/Missing_Addresses +touch $OUTPUTFOLDER/Missing_Addresses/Missing_Addresses.csv + +echo -e "File\tUser" > $OUTPUTFOLDER/Missing_Addresses/Missing_Addresses.csv + +cd $PDFDIR +#FOLDER=$(ls | grep *.pdf) +#echo $FOLDER +#*.pdf represensts All files in the current directory with a pdf extention. +for f in $PDFDIR/* +do + echo "Processing file $f." + #pdftotext converst the file to text and places it in the file name temp + pdftotext "$f" - > TEMP + #assigns the lines with relevant values to their variable + USER=$(sed -n '12p' TEMP) + ADDRESS1=$(sed -n '13p' TEMP) + ADDRESS2=$(sed -n '14p' TEMP) + echo -e "User is $USER" + echo -e "Address is $ADDRESS1 $ADDRESS2" + if [ -z "$ADDRESS1" ] + then + #Move the file with the Missing addresses to the Missing Address folder and make an note in its csv. + echo "Copying file to the folder called Missing_Addresses" + cp "$f" "$OUTPUTFOLDER"/Missing_Addresses/ + echo -e "$f\t$USER" >> "$OUTPUTFOLDER"/Missing_Addresses/Missing_Addresses.csv + #Adds the row of variables to the csv. >> appends to the file. \t represents tabs + else + # Split Addresses + CITY=$(echo $ADDRESS2 | cut -d, -f1) + ADDRESS2=$(echo $ADDRESS2 | sed -e "s/$CITY, //") + + STATE=$(echo $ADDRESS2 | cut -d" " -f1) + ZIP=$(echo $ADDRESS2 | cut -d" " -f2) + + echo -e "$f,$USER,$ADDRESS1,$CITY,$STATE,$ZIP" >> $OUTPUTFOLDER/Addresses/Addresses.csv + cp "$f" $OUTPUTFOLDER/Addresses/ + echo "Copying file to the folder called Addresses" + fi +done +echo "All done sorting! You can find all the Files in $OUTPUTFOLDER!" +echo +echo +read -r -p "Do you want to continue? [Y/n]" response +response=${response,,} # tolower +if [[ $response =~ ^(y| ) ]] || [[ -z $response ]]; then + cd $PDFDIR + for FILENAME in $PDFDIR/* + do + echo $FILENAME +#The next Line assigns the number of pages to the value LOCATION to be used for were the file goes + LOCATION=$(pdfinfo "$FILENAME" | grep Pages: | awk '{print $2}') +#Test if the file exists and if it does move it to Temp.pdf to be merged back to its original location. + if test -f $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf + then + mv $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf $OUTPUTFOLDER/temp.pdf + pdfunite "$FILENAME" $OUTPUTFOLDER/temp.pdf $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf + else + #IF the file doesn't exist create the file with the merge. + pdfunite "$FILENAME" $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf + fi +done +echo +echo +echo "All done, you can find the PDF statments at $OUTPUTFOLDER." +echo +echo +exit \ No newline at end of file diff --git a/Sort_and_Merge_Files.sh b/Sort_and_Merge_Files.sh new file mode 100755 index 0000000..52e25e1 --- /dev/null +++ b/Sort_and_Merge_Files.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +#get directory for pdf files +#get output directory for files + + +#Get the folder with the pdf documents in it. +#echo "This will take the PDF statements that have been sorted and will merge them into PDF files based on the numbers of pages in the statement." +#echo "" +#echo "Before we begin we need a little information." +#echo "" +#while : +#do +# echo "Where is the directory that has the PDF statments with address?" +# read PDFDIR +# if [ -d $PDFDIR ] +# then +# break +# else +# echo "Directory not found." +# fi +#done + +#Get the folder for the output file. +#while : +#do +# echo "" +# echo "What output directory do you want for the sorted PDF's?" +# read OUTPUTFOLDER +# #OUTPUTFOLDER="/home/btmccrea/Downloads/output" +# if [ -d $OUTPUTFOLDER ] +# then +# break +# else +# echo "Directory not found." +# fi +#done + +cd $PDFDIR +for FILENAME in $PDFDIR/* +do + echo $FILENAME +#The next Line assigns the number of pages to the value LOCATION to be used for were the file goes + LOCATION=$(pdfinfo "$FILENAME" | grep Pages: | awk '{print $2}') +#Test if the file exists and if it does move it to Temp.pdf to be merged back to its original location. + if test -f $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf + then + mv $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf $OUTPUTFOLDER/temp.pdf + pdfunite "$FILENAME" $OUTPUTFOLDER/temp.pdf $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf + else + #IF the file doesn't exist create the file with the merge. + pdfunite "$FILENAME" $OUTPUTFOLDER/Statement-Pages-$LOCATION.pdf + fi + +done + +echo "" +echo "" +echo "All done, you can find the PDF statments at $OUTPUTFOLDER. +echo "" +echo "" +exit \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..ab66e30 --- /dev/null +++ b/app.py @@ -0,0 +1,935 @@ +from flask import Flask, request, render_template, send_file, flash, redirect, url_for, jsonify, session +import os +import tempfile +import shutil +import fitz +from werkzeug.utils import secure_filename +import zipfile +import time +import threading +from multiprocessing import Pool, cpu_count +import json +from datetime import datetime +from dotenv import load_dotenv +from requests_oauthlib import OAuth2Session +from functools import wraps +import secrets +import base64 +import hashlib + +app = Flask(__name__) +app.config['UPLOAD_FOLDER'] = tempfile.mkdtemp() +app.config['SECRET_KEY'] = os.getenv('SECRET_KEY', 'secret') +app.config['MAX_CONTENT_LENGTH'] = 1000 * 1024 * 1024 * 1024 # 1000TB + +# Load environment variables +load_dotenv() + +# Microsoft Entra (Azure AD) Configuration +AZURE_TENANT_ID = os.getenv('AZURE_TENANT_ID') +AZURE_CLIENT_ID = os.getenv('AZURE_CLIENT_ID') +AZURE_CLIENT_SECRET = os.getenv('AZURE_CLIENT_SECRET') +AZURE_AUTHORITY = os.getenv('AZURE_AUTHORITY', f'https://login.microsoftonline.com/{AZURE_TENANT_ID}') +AZURE_REDIRECT_URI = os.getenv('AZURE_REDIRECT_URI', 'http://localhost:5000/auth/callback') +AZURE_SCOPES = os.getenv('AZURE_SCOPES', 'openid profile email').split() + +# Validate Azure AD configuration +azure_config_valid = all([ + AZURE_TENANT_ID, + AZURE_CLIENT_ID, + AZURE_CLIENT_SECRET +]) + +if not azure_config_valid: + print("Warning: Azure AD configuration is incomplete. Authentication will not work.") + print("Please check your .env file and ensure all AZURE_* variables are set.") + +# OAuth2 configuration +AUTHORIZATION_BASE_URL = f'{AZURE_AUTHORITY}/oauth2/v2.0/authorize' +TOKEN_URL = f'{AZURE_AUTHORITY}/oauth2/v2.0/token' + +# Authentication helper functions +def get_oauth_client(state=None): + """Create OAuth2 client for Azure AD.""" + if not azure_config_valid: + raise ValueError("Azure AD configuration is incomplete. Please check your .env file.") + return OAuth2Session( + client_id=AZURE_CLIENT_ID, + scope=AZURE_SCOPES, + state=state, + redirect_uri=AZURE_REDIRECT_URI + ) + +def generate_pkce(): + """Generate PKCE code verifier and challenge.""" + code_verifier = secrets.token_urlsafe(32) + code_challenge = base64.urlsafe_b64encode( + hashlib.sha256(code_verifier.encode()).digest() + ).decode().rstrip('=') + + return code_verifier, code_challenge + +def login_required(f): + """Decorator to require authentication for routes.""" + @wraps(f) + def decorated_function(*args, **kwargs): + print(f"DEBUG: login_required called for {f.__name__}") + print(f"DEBUG: azure_config_valid = {azure_config_valid}") + print(f"DEBUG: session keys = {list(session.keys())}") + + if not azure_config_valid: + # If Azure AD is not configured, show error + print("DEBUG: Azure AD not configured") + return "Authentication not configured. Please check your .env file.", 500 + if 'oauth_token' not in session: + print("DEBUG: No oauth_token in session, redirecting to login") + return redirect(url_for('login')) + print("DEBUG: Authentication passed") + return f(*args, **kwargs) + return decorated_function + +def get_user_info(): + """Get user information from session.""" + if 'user_info' in session: + return session['user_info'] + return None + +# Authentication routes +@app.route('/login') +def login(): + """Initiate Azure AD login flow.""" + if 'oauth_token' in session: + return redirect(url_for('index')) + + # Generate PKCE manually + code_verifier, code_challenge = generate_pkce() + session['code_verifier'] = code_verifier + + # Build authorization URL manually with PKCE + from urllib.parse import urlencode + auth_params = { + 'client_id': AZURE_CLIENT_ID, + 'response_type': 'code', + 'redirect_uri': AZURE_REDIRECT_URI, + 'scope': ' '.join(AZURE_SCOPES), + 'state': secrets.token_urlsafe(16), # Generate state + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256' + } + + session['oauth_state'] = auth_params['state'] + authorization_url = f"{AUTHORIZATION_BASE_URL}?{urlencode(auth_params)}" + + print(f"DEBUG: Authorization URL: {authorization_url}") + return redirect(authorization_url) + +@app.route('/auth/callback') +def auth_callback(): + """Handle Azure AD OAuth2 callback.""" + print(f"DEBUG: Callback URL: {request.url}") + print(f"DEBUG: Session keys: {list(session.keys())}") + + if 'oauth_state' not in session: + return redirect(url_for('login')) + + # Parse the authorization response + from urllib.parse import parse_qs, urlparse + parsed_url = urlparse(request.url) + query_params = parse_qs(parsed_url.query) + + print(f"DEBUG: Query params: {query_params}") + + if 'error' in query_params: + print(f"DEBUG: Error from Azure AD: {query_params['error'][0]}") + return f"Authentication error: {query_params['error'][0]}", 400 + + if 'code' not in query_params or 'state' not in query_params: + return "Invalid authorization response", 400 + + code = query_params['code'][0] + state = query_params['state'][0] + + if state != session['oauth_state']: + return "State mismatch", 400 + + # Use OAuth2Session for token exchange to maintain PKCE consistency + oauth_session = OAuth2Session( + client_id=AZURE_CLIENT_ID, + scope=AZURE_SCOPES, + redirect_uri=AZURE_REDIRECT_URI + ) + + # Set the same PKCE that was used for authorization + if 'code_verifier' in session and hasattr(oauth_session, '_client'): + oauth_session._client.code_verifier = session['code_verifier'] + + # Set the authorization response for the session + redirect_uri_with_params = f"{AZURE_REDIRECT_URI}?code={code}&state={state}" + print(f"DEBUG: Redirect URI with params: {redirect_uri_with_params}") + + # Exchange code for token using direct HTTP request with PKCE + import requests + + token_data = { + 'client_id': AZURE_CLIENT_ID, + 'client_secret': AZURE_CLIENT_SECRET, # Include client_secret for confidential client + 'code': code, + 'grant_type': 'authorization_code', + 'redirect_uri': AZURE_REDIRECT_URI, + 'scope': ' '.join(AZURE_SCOPES) + } + + # Include code_verifier for PKCE + if 'code_verifier' in session: + token_data['code_verifier'] = session['code_verifier'] + print(f"DEBUG: Including code_verifier in token request: {session['code_verifier'][:10]}...") + + print(f"DEBUG: Token request data keys: {list(token_data.keys())}") + + token_response = requests.post(TOKEN_URL, data=token_data) + print(f"DEBUG: Token response status: {token_response.status_code}") + print(f"DEBUG: Token response: {token_response.text[:200]}...") + + if token_response.status_code != 200: + return f"Token exchange failed: {token_response.text}", 400 + + token = token_response.json() + print(f"DEBUG: Token received successfully") + session['oauth_token'] = token + + # Get user info + user_info = oauth_session.get('https://graph.microsoft.com/v1.0/me').json() + session['user_info'] = user_info + + # Clean up + session.pop('oauth_state', None) + session.pop('code_verifier', None) + + return redirect(url_for('index')) + +@app.route('/logout') +def logout(): + """Logout user and clear session.""" + session.clear() + return redirect(url_for('index')) + +# Job storage +JOBS_FILE = 'jobs.json' +RESULTS_DIR = 'results' +os.makedirs(RESULTS_DIR, exist_ok=True) + +processing_complete = False +current_output_dir = None + +def cleanup_temp_files(): + """Clean up temporary files and directories.""" + import shutil + import glob + + # Clean up any leftover temp directories + temp_base = tempfile.gettempdir() + temp_dirs = glob.glob(os.path.join(temp_base, 'tmp*')) + for temp_dir in temp_dirs: + try: + # Only delete if it's a directory and empty or contains our temp files + if os.path.isdir(temp_dir): + # Check if it contains our processing files + contents = os.listdir(temp_dir) + if any('Addresses' in item or 'Missing_Addresses' in item or 'Originals' in item for item in contents): + shutil.rmtree(temp_dir) + print(f"Cleaned up temp directory: {temp_dir}") + except Exception as e: + print(f"Could not clean up {temp_dir}: {e}") + + # Reset global state + global processing_complete, current_output_dir + processing_complete = False + current_output_dir = None + + # Clear status file + if os.path.exists('status.txt'): + try: + os.remove('status.txt') + except: + pass + +def load_jobs(): + """Load jobs from JSON file.""" + if os.path.exists(JOBS_FILE): + try: + with open(JOBS_FILE, 'r') as f: + jobs = json.load(f) + # Convert timestamp strings back to datetime objects + for job in jobs.values(): + if 'timestamp' in job and isinstance(job['timestamp'], str): + try: + # Handle the format: "2025-11-05 09:10:23.730974" + job['timestamp'] = datetime.strptime(job['timestamp'], '%Y-%m-%d %H:%M:%S.%f') + except (ValueError, TypeError): + # If conversion fails, keep as string + pass + return jobs + except: + return {} + return {} + +def save_jobs(jobs): + """Save jobs to JSON file.""" + with open(JOBS_FILE, 'w') as f: + json.dump(jobs, f, indent=2, default=str) + +@app.route('/', methods=['GET', 'POST']) +@login_required +def index(): + global processing_complete, current_output_dir + if request.method == 'POST': + files = request.files.getlist('files') + if not files or files[0].filename == '': + flash('No files uploaded') + return redirect(request.url) + + # Clean up any previous processing + cleanup_temp_files() + + # Create temp dirs + input_dir = tempfile.mkdtemp() + output_dir = tempfile.mkdtemp() + current_output_dir = output_dir + + for file in files: + if file and file.filename: + filename = secure_filename(file.filename) + if filename.endswith('.zip'): + # Extract zip + zip_temp = tempfile.mkdtemp() + zip_path = os.path.join(zip_temp, filename) + file.save(zip_path) + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(input_dir) + shutil.rmtree(zip_temp) + print(f"Extracted ZIP: {filename}") + elif filename.endswith('.pdf'): + file.save(os.path.join(input_dir, filename)) + print(f"Saved PDF: {filename}") + # For folder upload, files are saved with secure_filename, but since webkitdirectory, filename includes path, but secure_filename flattens it + else: + # Assume folder upload + file.save(os.path.join(input_dir, filename)) + print(f"Saved file from folder: {filename}") + + # Create job immediately + job_start_time = datetime.now() + job_id = job_start_time.strftime('%Y%m%d_%H%M%S') + jobs = load_jobs() + jobs[job_id] = { + 'id': job_id, + 'timestamp': job_start_time, + 'status': 'processing', + 'message': 'Starting processing...', + 'file_count': len([f for f in files if f.filename]), # Estimate, will be updated + 'progress': 0 + } + save_jobs(jobs) + + # Start processing in background + processing_complete = False + thread = threading.Thread(target=process_pdfs_async, args=(input_dir, output_dir, job_id)) + thread.start() + + # Redirect to job status page + return redirect(url_for('job_status', job_id=job_id)) + + return render_template('index.html', user_info=get_user_info()) + +@app.route('/status') +def status(): + print("Status requested") + try: + with open('status.txt', 'r') as f: + content = f.read() + if "Processing complete" in content: + return jsonify({"status": "complete", "message": content}) + elif "Error" in content: + return jsonify({"status": "error", "message": content}) + else: + return jsonify({"status": "processing", "message": content}) + except FileNotFoundError: + return jsonify({"status": "error", "message": "Processing not complete or file not found"}) + +@app.route('/results') +@login_required +def results(): + jobs = load_jobs() + # Sort jobs by timestamp, newest first + sorted_jobs = sorted(jobs.values(), key=lambda x: x['timestamp'], reverse=True) + return render_template('results.html', jobs=sorted_jobs, user_info=get_user_info()) + +@app.route('/jobs') +@login_required +def jobs_page(): + jobs = load_jobs() + # Sort jobs by timestamp, newest first + sorted_jobs = sorted(jobs.values(), key=lambda x: x['timestamp'], reverse=True) + return render_template('jobs.html', jobs=sorted_jobs, user_info=get_user_info()) + +@app.route('/job/') +@login_required +def job_status(job_id): + jobs = load_jobs() + if job_id not in jobs: + flash('Job not found') + return redirect(url_for('index')) + + job = jobs[job_id] + return render_template('job.html', job=job, user_info=get_user_info()) + +@app.route('/api/job//status') +def get_job_status(job_id): + jobs = load_jobs() + if job_id not in jobs: + return jsonify({"error": "Job not found"}), 404 + + job = jobs[job_id] + return jsonify({ + "id": job["id"], + "status": job["status"], + "message": job.get("message", ""), + "progress": job.get("progress", 0), + "file_count": job.get("file_count", 0) + }) + +def process_pdfs(pdfdir, outputfolder): + global processing_complete + status_file = 'status.txt' + with open(status_file, 'w') as status_f: + status_f.write("Step 1/4: Finding PDF files...\n") + print("Step 1/4: Finding PDF files...") + time.sleep(0.1) + + addresses_dir = os.path.join(outputfolder, 'Addresses') + missing_dir = os.path.join(outputfolder, 'Missing_Addresses') + os.makedirs(addresses_dir) + os.makedirs(missing_dir) + + with open(os.path.join(addresses_dir, 'Addresses.csv'), 'w') as addr_csv: + addr_csv.write("File,User,Street,City,State,Zip\n") + + with open(os.path.join(missing_dir, 'Missing_Addresses.csv'), 'w') as miss_csv: + miss_csv.write("File\tUser\n") + + pdf_files = [] + for root, dirs, files in os.walk(pdfdir): + for f in files: + if f.endswith('.pdf'): + pdf_files.append(os.path.join(root, f)) + + msg = f"Step 1/4: Found {len(pdf_files)} PDF files\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.1) + + # Update job status + jobs = load_jobs() + jobs[job_id]['message'] = f"Step 1/4: Found {len(pdf_files)} PDF files" + jobs[job_id]['file_count'] = len(pdf_files) + jobs[job_id]['progress'] = 10 + save_jobs(jobs) + + msg = "Step 2/4: Processing files...\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print("Step 2/4: Processing files...") + + # Update job status + jobs[job_id]['message'] = "Step 2/4: Processing files..." + jobs[job_id]['progress'] = 20 + save_jobs(jobs) + + time.sleep(0.1) + + # Process files in parallel + with Pool(processes=cpu_count()) as pool: + results = pool.map(process_single_pdf, pdf_files) + + for result in results: + f = result['filename'] + if result['type'] == 'missing': + # Missing address + shutil.copy(result['filepath'], os.path.join(missing_dir, f)) + with open(os.path.join(missing_dir, 'Missing_Addresses.csv'), 'a') as miss_csv: + miss_csv.write(f"{f}\t{result['user']}\n") + msg = f" -> Moved to Missing_Addresses\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + elif result['type'] == 'address': + # Valid address + shutil.copy(result['filepath'], os.path.join(addresses_dir, f)) + with open(os.path.join(addresses_dir, 'Addresses.csv'), 'a') as addr_csv: + addr_csv.write(f"{f},{result['user']},{result['address']},{result['city']},{result['state']},{result['zip']}\n") + msg = f" -> Moved to Addresses (User: {result['user']}, Address: {result['address']}, {result['city']})\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + elif result['type'] == 'error': + msg = f" -> Error processing {f}: {result['error']}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.05) + + msg = "Step 2/4: Sorting complete. Starting merging...\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status + jobs[job_id]['message'] = "Step 2/4: Sorting complete. Starting merging..." + jobs[job_id]['progress'] = 80 + save_jobs(jobs) + + time.sleep(0.1) + + # Merge by page count + page_groups = {} + for filepath in pdf_files: + f = os.path.basename(filepath) + doc = fitz.open(filepath) + pages = len(doc) + doc.close() + if pages not in page_groups: + page_groups[pages] = [] + page_groups[pages].append(filepath) + + msg = f"Step 3/4: Found {len(page_groups)} different page counts to merge\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.1) + + for pages, files in page_groups.items(): + msg = f"Merging {len(files)} files with {pages} pages each\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.05) + try: + merger = fitz.open() + for file in files: + doc = fitz.open(file) + merger.insert_pdf(doc) + doc.close() + output_path = os.path.join(outputfolder, f'Statement-Pages-{pages}.pdf') + merger.save(output_path) + merger.close() + msg = f" -> Created {output_path}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + except Exception as e: + msg = f" -> Error merging files with {pages} pages: {str(e)}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.05) + + msg = f"Processing complete! Processed {len(pdf_files)} files total\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status + jobs[job_id]['message'] = f"Processing complete! Processed {len(pdf_files)} files total" + jobs[job_id]['progress'] = 90 + save_jobs(jobs) + + time.sleep(0.1) + + # Step 4/4: Creating ZIP file + msg = "Step 4/4: Creating ZIP file...\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status + jobs[job_id]['message'] = "Step 4/4: Creating ZIP file..." + jobs[job_id]['progress'] = 95 + save_jobs(jobs) + + time.sleep(0.1) + + # Create zip with unique name + zip_filename = f'{job_id}.zip' + zip_path = os.path.join(RESULTS_DIR, zip_filename) + + try: + # Create zip with only merged PDF documents (no CSV files) + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Add merged statement PDFs from root output folder + for file in os.listdir(outputfolder): + if file.endswith('.pdf') and file.startswith('Statement-Pages-'): + file_path = os.path.join(outputfolder, file) + zipf.write(file_path, file) + + # Save job metadata + jobs = load_jobs() + jobs[job_id]['status'] = 'completed' + jobs[job_id]['zip_path'] = zip_path + jobs[job_id]['zip_filename'] = zip_filename + jobs[job_id]['message'] = f'Processing complete! ZIP created: {zip_filename}' + jobs[job_id]['progress'] = 100 + save_jobs(jobs) + + msg = f"ZIP created: {zip_filename}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + except Exception as e: + msg = f"Error creating ZIP: {str(e)}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + # Still save job as failed + jobs = load_jobs() + jobs[job_id]['status'] = 'failed' + jobs[job_id]['error'] = str(e) + jobs[job_id]['message'] = f'Error creating ZIP: {str(e)}' + save_jobs(jobs) + + processing_complete = True + global current_output_dir + current_output_dir = outputfolder + +def process_pdfs_async(pdfdir, outputfolder, job_id): + global processing_complete + try: + jobs = load_jobs() + if job_id not in jobs: + return + + status_file = 'status.txt' + with open(status_file, 'w') as status_f: + status_f.write("Step 1/4: Finding PDF files...\n") + print("Step 1/4: Finding PDF files...") + + # Update job status + jobs[job_id]['message'] = "Step 1/4: Finding PDF files..." + jobs[job_id]['progress'] = 5 + save_jobs(jobs) + + time.sleep(0.1) + + addresses_dir = os.path.join(outputfolder, 'Addresses') + missing_dir = os.path.join(outputfolder, 'Missing_Addresses') + os.makedirs(addresses_dir) + os.makedirs(missing_dir) + + pdf_files = [] + for root, dirs, files in os.walk(pdfdir): + for f in files: + if f.endswith('.pdf'): + pdf_files.append(os.path.join(root, f)) + + msg = f"Step 1/4: Found {len(pdf_files)} PDF files\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status + jobs = load_jobs() + jobs[job_id]['message'] = f"Step 1/4: Found {len(pdf_files)} PDF files" + jobs[job_id]['file_count'] = len(pdf_files) + jobs[job_id]['progress'] = 10 + save_jobs(jobs) + + time.sleep(0.1) + + msg = "Step 2/4: Processing files...\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print("Step 2/4: Processing files...") + + # Update job status + jobs[job_id]['message'] = "Step 2/4: Processing files..." + jobs[job_id]['progress'] = 20 + save_jobs(jobs) + + time.sleep(0.1) + + # Parallel processing + num_processes = min(cpu_count(), len(pdf_files)) + with Pool(processes=num_processes) as pool: + results = pool.map(process_single_pdf, pdf_files) + + # Process results sequentially + with open(os.path.join(addresses_dir, 'Addresses.csv'), 'w') as addr_csv: + addr_csv.write("File,User,Street,City,State,Zip\n") + + with open(os.path.join(missing_dir, 'Missing_Addresses.csv'), 'w') as miss_csv: + miss_csv.write("File\tUser\n") + + page_groups = {} + for i, result in enumerate(results, 1): + progress_percent = 20 + int((i / len(results)) * 60) # 20-80% range for processing + + msg = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status periodically + if i % 100 == 0 or i == len(results): # Update every 100 files or at the end + jobs[job_id]['message'] = f"Processing result {i}/{len(results)}: {result.get('filename', 'unknown')}" + jobs[job_id]['progress'] = progress_percent + save_jobs(jobs) + + if result['type'] == 'address': + with open(os.path.join(addresses_dir, 'Addresses.csv'), 'a') as addr_csv: + addr_csv.write(f"{result['filename']},{result['user']},{result['address']},{result['city']},{result['state']},{result['zip']}\n") + shutil.copy(result['filepath'], os.path.join(addresses_dir, result['filename'])) + msg = f" -> Moved to Addresses\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + elif result['type'] == 'missing': + with open(os.path.join(missing_dir, 'Missing_Addresses.csv'), 'a') as miss_csv: + miss_csv.write(f"{result['filename']}\t{result['user']}\n") + shutil.copy(result['filepath'], os.path.join(missing_dir, result['filename'])) + msg = f" -> Moved to Missing_Addresses\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + elif result['type'] == 'error': + msg = f" -> Error: {result['error']}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Group for merging + if 'pages' in result: + pages = result['pages'] + if pages not in page_groups: + page_groups[pages] = [] + page_groups[pages].append(result['filepath']) + + time.sleep(0.01) # Small delay for status updates + + msg = "Step 2/4: Sorting complete. Starting merging...\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status + jobs[job_id]['message'] = "Step 2/4: Sorting complete. Starting merging..." + jobs[job_id]['progress'] = 80 + save_jobs(jobs) + + time.sleep(0.1) + + # Merge by page count + msg = f"Step 3/4: Found {len(page_groups)} different page counts to merge\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.1) + + for pages, files in page_groups.items(): + msg = f"Merging {len(files)} files with {pages} pages each\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.05) + try: + merger = fitz.open() + for file in files: + doc = fitz.open(file) + merger.insert_pdf(doc) + doc.close() + output_path = os.path.join(outputfolder, f'Statement-Pages-{pages}.pdf') + merger.save(output_path) + merger.close() + msg = f" -> Created {output_path}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + except Exception as e: + msg = f" -> Error merging files with {pages} pages: {str(e)}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + time.sleep(0.05) + + msg = f"Processing complete! Processed {len(pdf_files)} files total\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status + jobs[job_id]['message'] = f"Processing complete! Processed {len(pdf_files)} files total" + jobs[job_id]['progress'] = 90 + save_jobs(jobs) + + time.sleep(0.1) + + # Step 4/4: Creating ZIP file + msg = "Step 4/4: Creating ZIP file...\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + # Update job status + jobs[job_id]['message'] = "Step 4/4: Creating ZIP file..." + jobs[job_id]['progress'] = 95 + save_jobs(jobs) + + time.sleep(0.1) + + # Create zip with unique name + zip_filename = f'{job_id}.zip' + zip_path = os.path.join(RESULTS_DIR, zip_filename) + + try: + # Create zip with only merged PDF documents (no CSV files) + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Add merged statement PDFs from root output folder + for file in os.listdir(outputfolder): + if file.endswith('.pdf') and file.startswith('Statement-Pages-'): + file_path = os.path.join(outputfolder, file) + zipf.write(file_path, file) + + # Save job metadata + jobs = load_jobs() + jobs[job_id]['status'] = 'completed' + jobs[job_id]['zip_path'] = zip_path + jobs[job_id]['zip_filename'] = zip_filename + jobs[job_id]['message'] = f'Processing complete! ZIP created: {zip_filename}' + jobs[job_id]['progress'] = 100 + save_jobs(jobs) + + msg = f"ZIP created: {zip_filename}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + + except Exception as e: + msg = f"Error creating ZIP: {str(e)}\n" + with open(status_file, 'a') as status_f: + status_f.write(msg) + print(msg.strip()) + # Still save job as failed + jobs = load_jobs() + jobs[job_id]['status'] = 'failed' + jobs[job_id]['error'] = str(e) + jobs[job_id]['message'] = f'Error creating ZIP: {str(e)}' + save_jobs(jobs) + + except Exception as e: + msg = f"Processing error: {e}" + with open('status.txt', 'w') as f: + f.write(msg) + # Mark job as failed + jobs = load_jobs() + if job_id in jobs: + jobs[job_id]['status'] = 'failed' + jobs[job_id]['error'] = str(e) + jobs[job_id]['message'] = msg + save_jobs(jobs) + processing_complete = True # Still mark as complete to allow retry + +@app.route('/download/') +@login_required +def download_job(job_id): + jobs = load_jobs() + if job_id in jobs and jobs[job_id]['status'] == 'completed': + zip_path = jobs[job_id]['zip_path'] + if os.path.exists(zip_path): + return send_file(zip_path, as_attachment=True, download_name=jobs[job_id]['zip_filename']) + return jsonify({"error": "Job not found or not completed"}), 404 + +@app.route('/download') +def download(): + # Legacy route - redirect to results page + return redirect(url_for('results')) + +@app.route('/delete/', methods=['POST']) +@login_required +def delete_job(job_id): + jobs = load_jobs() + if job_id in jobs: + job = jobs[job_id] + # Delete ZIP file if it exists + if 'zip_path' in job and os.path.exists(job['zip_path']): + try: + os.remove(job['zip_path']) + except: + pass # Ignore errors + # Remove from jobs + del jobs[job_id] + save_jobs(jobs) + flash(f'Job {job_id} deleted successfully') + else: + flash('Job not found') + return redirect(url_for('results')) + +@app.route('/clear', methods=['POST']) +def clear_processing(): + try: + cleanup_temp_files() + return jsonify({"message": "Temporary files cleared and processing state reset."}) + except Exception as e: + return jsonify({"message": f"Error clearing files: {str(e)}"}), 500 + +@app.route('/clear_and_upload') +def clear_and_upload(): + cleanup_temp_files() + flash('Processing state cleared. Ready to upload new files.') + return redirect(url_for('index')) + +def process_single_pdf(filepath): + """Process a single PDF file and return the result.""" + try: + f = os.path.basename(filepath) + doc = fitz.open(filepath) + text = "" + for page in doc: + text += page.get_text() + pages = len(doc) + doc.close() + + lines = text.split('\n') + if len(lines) < 15: + user = "" + addr1 = "" + addr2 = "" + else: + user = lines[11].strip() + addr1 = lines[12].strip() + addr2 = lines[13].strip() + + if not addr1: + return {'type': 'missing', 'filename': f, 'user': user, 'filepath': filepath, 'pages': pages} + else: + if ',' in addr2: + city, rest = addr2.split(',', 1) + city = city.strip() + rest_parts = rest.strip().split() + state = rest_parts[0] if rest_parts else '' + zip_code = ' '.join(rest_parts[1:]) if len(rest_parts) > 1 else '' + else: + city = addr2 + state = '' + zip_code = '' + return {'type': 'address', 'filename': f, 'user': user, 'address': addr1, 'city': city, 'state': state, 'zip': zip_code, 'filepath': filepath, 'pages': pages} + except Exception as e: + return {'type': 'error', 'filename': os.path.basename(filepath), 'error': str(e), 'filepath': filepath} + +if __name__ == '__main__': + try: + from waitress import serve + print("Starting with Waitress server (supports large uploads)...") + serve(app, host='0.0.0.0', port=5000, max_request_body_size=1000*1024*1024*1024) # 1TB + except ImportError: + print("Waitress not available, using Flask dev server...") + app.run(host='0.0.0.0', port=5000, debug=True) \ No newline at end of file diff --git a/jobs.json b/jobs.json new file mode 100644 index 0000000..af75d2a --- /dev/null +++ b/jobs.json @@ -0,0 +1,30 @@ +{ + "84a67896-ea93-4355-9020-ca26bad91316": { + "id": "84a67896-ea93-4355-9020-ca26bad91316", + "timestamp": "2025-11-05 09:10:23.730974", + "file_count": 5559, + "zip_path": "results\\job_84a67896-ea93-4355-9020-ca26bad91316.zip", + "zip_filename": "job_84a67896-ea93-4355-9020-ca26bad91316.zip", + "status": "completed" + }, + "20251105_093008": { + "id": "20251105_093008", + "timestamp": "2025-11-05 09:30:08.978343", + "status": "completed", + "message": "Processing complete! ZIP created: 20251105_093008.zip", + "file_count": 5559, + "progress": 100, + "zip_path": "results\\20251105_093008.zip", + "zip_filename": "20251105_093008.zip" + }, + "20251105_095440": { + "id": "20251105_095440", + "timestamp": "2025-11-05 09:54:40.248747", + "status": "completed", + "message": "Processing complete! ZIP created: 20251105_095440.zip", + "file_count": 5559, + "progress": 100, + "zip_path": "results\\20251105_095440.zip", + "zip_filename": "20251105_095440.zip" + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..897ba73 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +Flask==3.0.0 +PyMuPDF==1.23.7 +Werkzeug==3.0.1 +waitress==3.0.0 +requests-oauthlib==2.0.0 +python-dotenv==1.2.1 \ No newline at end of file diff --git a/static/trbc-logo-full.png b/static/trbc-logo-full.png new file mode 100644 index 0000000..b242db4 Binary files /dev/null and b/static/trbc-logo-full.png differ diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..b60e260 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,312 @@ + + + + End of Year Statements + + + +
+ +

End of Year Statements Processor

+

Upload PDF files or folders to automatically sort by address and merge by page count

+ + +
+ {% if session.oauth_token %} + + {% else %} + + {% endif %} +
+
+ +
+ View All Jobs & Downloads +
+ +
+
+ + +
+
+
+ +
+ + + +
+
+

Clearing temporary files...

+ +
+
+ +
+ Section 1 + Section 2 + Section 3 +
+ + {% with messages = get_flashed_messages() %} + {% if messages %} +
+ {% for message in messages %} +

{{ message }}

+ {% endfor %} +
+ {% endif %} + {% endwith %} + + + + + + \ No newline at end of file diff --git a/templates/job.html b/templates/job.html new file mode 100644 index 0000000..ddc5aae --- /dev/null +++ b/templates/job.html @@ -0,0 +1,227 @@ + + + + Job Status - End of Year Statements + + + +
+ +

Job Status

+ + +
+ {% if session.oauth_token %} + + {% else %} + + {% endif %} +
+
+ +
+ Upload New Files | + View All Jobs & Downloads +
+ +
+

Job Information

+
+
Job Name: {{ job.id }}
+
Started: {{ job.timestamp.strftime('%Y-%m-%d %H:%M:%S') if job.timestamp else 'Unknown' }}
+
Files: {{ job.file_count if job.file_count else 'Processing...' }}
+
Status: + + {{ job.status.title() }} + +
+
+ + {% if job.status == 'processing' %} +
+

Processing Progress

+

{{ job.message }}

+ +
+ {{ job.progress }}% Complete +
+
+ {% endif %} + + {% if job.status == 'completed' %} +
+

Processing Complete!

+

{{ job.message }}

+ + +
+ {% endif %} + + {% if job.status == 'failed' %} +
+

Processing Failed

+

{{ job.message }}

+ {% if job.error %} +
+ Error: {{ job.error }} +
+ {% endif %} + +
+ {% endif %} +
+ +
+ Back to Upload + View All Jobs +
+ + {% if job.status == 'processing' %} + + {% endif %} + + \ No newline at end of file diff --git a/templates/jobs.html b/templates/jobs.html new file mode 100644 index 0000000..d449eb4 --- /dev/null +++ b/templates/jobs.html @@ -0,0 +1,242 @@ + + + + All Jobs - End of Year Statements + + + +
+ +

All Jobs

+

View all your PDF processing jobs

+ + +
+ {% if session.oauth_token %} + + {% else %} + + {% endif %} +
+
+ +
+ 📤 Upload Files + 📋 View Results + 📊 View All Jobs +
+ + ← Back to Upload + 🔄 Reprocess Files + +
+

All Jobs

+ {% if jobs %} + {% for job in jobs %} +
+
+
+ Job Name: {{ job.id }}
+ Started: {{ job.timestamp.strftime('%Y-%m-%d %H:%M:%S') if job.timestamp else 'Unknown' }}
+ {% if job.file_count %} + Files: {{ job.file_count }}
+ {% endif %} + Status: + {% if job.status == 'completed' %} + Completed + {% elif job.status == 'processing' %} + Processing + {% elif job.status == 'failed' %} + Failed + {% elif job.status == 'queued' %} + Queued + {% else %} + {{ job.status|title }} + {% endif %} + {% if job.get('error') %} +
Error: {{ job.error }} + {% endif %} + {% if job.get('message') %} +
Message: {{ job.message }} + {% endif %} +
+
+ View Details + {% if job.status == 'completed' %} +
+ +
+ {% endif %} +
+ +
+
+
+
+ {% endfor %} + {% else %} +

No jobs yet. Upload some files to get started.

+ {% endif %} +
+ + {% with messages = get_flashed_messages() %} + {% if messages %} +
+ {% for message in messages %} +

{{ message }}

+ {% endfor %} +
+ {% endif %} + {% endwith %} + + \ No newline at end of file diff --git a/templates/results.html b/templates/results.html new file mode 100644 index 0000000..b1413c9 --- /dev/null +++ b/templates/results.html @@ -0,0 +1,219 @@ + + + + Processing Results - End of Year Statements + + + +
+ +

Processing Results

+

View and download your completed PDF processing jobs

+ + +
+ {% if session.oauth_token %} + + {% else %} + + {% endif %} +
+
+ +
+ 📤 Upload Files + 📋 View Results + 📊 View All Jobs +
+ + ← Back to Upload + 🔄 Reprocess Files + +
+

Completed Jobs

+ {% if jobs %} + {% for job in jobs %} +
+
+
+ Job Name: {{ job.id }}
+ Processed: {{ job.timestamp.strftime('%Y-%m-%d %H:%M:%S') if job.timestamp else 'Unknown' }}
+ Files: {{ job.file_count }}
+ Status: + {% if job.status == 'completed' %} + Completed + {% else %} + Failed + {% endif %} + {% if job.get('error') %} +
Error: {{ job.error }} + {% endif %} +
+
+ {% if job.status == 'completed' %} +
+ +
+ {% endif %} +
+ +
+
+
+
+ {% endfor %} + {% else %} +

No completed jobs yet. Upload some files to get started.

+ {% endif %} +
+ + {% with messages = get_flashed_messages() %} + {% if messages %} +
+ {% for message in messages %} +

{{ message }}

+ {% endfor %} +
+ {% endif %} + {% endwith %} + + \ No newline at end of file diff --git a/ubuntu-install.sh b/ubuntu-install.sh new file mode 100644 index 0000000..c922325 --- /dev/null +++ b/ubuntu-install.sh @@ -0,0 +1,237 @@ +#!/bin/bash + +# PDF Sort and Merge Application - Ubuntu Installation Script +# This script installs and configures the Flask web application for PDF processing + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if running as root +if [[ $EUID -eq 0 ]]; then + log_error "This script should not be run as root. Please run as a regular user with sudo privileges." + exit 1 +fi + +# Check Ubuntu version +UBUNTU_VERSION=$(lsb_release -rs) +log_info "Detected Ubuntu version: $UBUNTU_VERSION" + +# Update system +log_info "Updating system packages..." +sudo apt update && sudo apt upgrade -y + +# Install Python 3.9+ and pip +log_info "Installing Python and pip..." +sudo apt install -y python3 python3-pip python3-venv python3-dev + +# Install system dependencies for PDF processing +log_info "Installing system dependencies..." +sudo apt install -y build-essential libpoppler-cpp-dev pkg-config + +# Install Node.js and npm (for potential frontend assets) +log_info "Installing Node.js and npm..." +curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - +sudo apt install -y nodejs + +# Create application directory +APP_DIR="$HOME/pdf-sort-merge" +log_info "Creating application directory: $APP_DIR" +mkdir -p "$APP_DIR" +cd "$APP_DIR" + +# Clone or download the application +# Note: Replace with your actual repository URL +REPO_URL="https://github.com/yourusername/pdf-sort-merge.git" +log_info "Cloning application repository..." +if command -v git &> /dev/null; then + sudo apt install -y git + git clone "$REPO_URL" . +else + log_warning "Git not available. Please manually download and extract the application files to $APP_DIR" + exit 1 +fi + +# Create Python virtual environment +log_info "Creating Python virtual environment..." +python3 -m venv venv +source venv/bin/activate + +# Install Python dependencies +log_info "Installing Python dependencies..." +pip install --upgrade pip +pip install -r requirements.txt + +# Create .env file template +log_info "Creating environment configuration..." +if [ ! -f .env ]; then + cat > .env << 'EOF' +# Microsoft Entra (Azure AD) Configuration +# Get these values from your Azure AD app registration in the Azure portal + +# Azure AD Tenant ID (Directory ID) +AZURE_TENANT_ID=your-tenant-id-here + +# Azure AD Client ID (Application ID) +AZURE_CLIENT_ID=your-client-id-here + +# Azure AD Client Secret +AZURE_CLIENT_SECRET=your-client-secret-here + +# Azure AD Authority URL (usually https://login.microsoftonline.com/{tenant_id}) +AZURE_AUTHORITY=https://login.microsoftonline.com/your-tenant-id-here + +# Redirect URI (must match what's configured in Azure AD app registration) +AZURE_REDIRECT_URI=http://localhost:5000/auth/callback + +# Scopes to request (openid, profile, email are common) +AZURE_SCOPES=openid profile email + +# Flask Configuration +SECRET_KEY=your-secret-key-here-change-this-in-production +EOF + log_warning "Created .env template. You MUST edit it with your Azure AD credentials before running the app." +else + log_info ".env file already exists" +fi + +# Create systemd service file +log_info "Creating systemd service..." +SERVICE_FILE="/etc/systemd/system/pdf-sort-merge.service" +sudo tee "$SERVICE_FILE" > /dev/null << EOF +[Unit] +Description=PDF Sort and Merge Flask Application +After=network.target + +[Service] +Type=simple +User=$USER +WorkingDirectory=$APP_DIR +Environment=PATH=$APP_DIR/venv/bin +ExecStart=$APP_DIR/venv/bin/python $APP_DIR/app.py +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +# Set proper permissions +sudo chmod 644 "$SERVICE_FILE" + +# Create log directory +log_info "Creating log directory..." +sudo mkdir -p /var/log/pdf-sort-merge +sudo chown $USER:$USER /var/log/pdf-sort-merge + +# Create application data directories +log_info "Creating application data directories..." +mkdir -p results static templates + +# Set up firewall (optional) +log_info "Configuring firewall..." +sudo ufw allow 5000/tcp || log_warning "Failed to configure firewall - you may need to do this manually" + +# Enable and start service +log_info "Enabling and starting service..." +sudo systemctl daemon-reload +sudo systemctl enable pdf-sort-merge +sudo systemctl start pdf-sort-merge + +# Check service status +sleep 3 +if sudo systemctl is-active --quiet pdf-sort-merge; then + log_success "Service started successfully!" +else + log_error "Service failed to start. Check logs with: sudo journalctl -u pdf-sort-merge -f" +fi + +# Create management script +log_info "Creating management script..." +cat > manage.sh << 'EOF' +#!/bin/bash + +case "$1" in + start) + sudo systemctl start pdf-sort-merge + echo "Service started" + ;; + stop) + sudo systemctl stop pdf-sort-merge + echo "Service stopped" + ;; + restart) + sudo systemctl restart pdf-sort-merge + echo "Service restarted" + ;; + status) + sudo systemctl status pdf-sort-merge + ;; + logs) + sudo journalctl -u pdf-sort-merge -f + ;; + update) + cd /home/$USER/pdf-sort-merge + source venv/bin/activate + git pull + pip install -r requirements.txt + sudo systemctl restart pdf-sort-merge + echo "Application updated" + ;; + *) + echo "Usage: $0 {start|stop|restart|status|logs|update}" + exit 1 + ;; +esac +EOF + +chmod +x manage.sh + +# Final instructions +log_success "Installation completed!" +echo "" +echo "==================================================================" +echo "NEXT STEPS:" +echo "1. Edit the .env file with your Azure AD credentials:" +echo " nano $APP_DIR/.env" +echo "" +echo "2. Configure your Azure AD app registration with redirect URI:" +echo " http://your-server-ip:5000/auth/callback" +echo "" +echo "3. Access the application at:" +echo " http://localhost:5000" +echo "" +echo "MANAGEMENT COMMANDS:" +echo " Start service: sudo systemctl start pdf-sort-merge" +echo " Stop service: sudo systemctl stop pdf-sort-merge" +echo " Check status: sudo systemctl status pdf-sort-merge" +echo " View logs: sudo journalctl -u pdf-sort-merge -f" +echo " Update app: ./manage.sh update" +echo "==================================================================" + +# Check if .env needs configuration +if grep -q "your-tenant-id-here" .env; then + log_warning "IMPORTANT: Configure your .env file before using the application!" +fi \ No newline at end of file