Add background job processing system with jobs page and download management

README.md

@@ -10,6 +10,8 @@ A Python web application built with Flask that automatically renames PDF files b
 - **Intelligent Renaming**: Renames files to "LastName_FirstName.pdf" format
 - **Duplicate Handling**: Automatically appends counters for duplicate names
 - **ZIP Support**: Upload ZIP files and download processed results as ZIP
+- **Job Management**: Background processing with job status tracking and history
+- **Authentication**: Microsoft Entra ID and local admin login support
 - **Error Handling**: Gracefully skips files that can't be processed
 
 ## Prerequisites
@@ -82,16 +84,19 @@ Users can choose their preferred login method on the login page.
 
 3. **Access the web interface**:
    - Open `http://127.0.0.1:3000/`
-   - You'll be redirected to Microsoft login
-   - After authentication, you'll access the app
+   - Authenticate using Microsoft Entra ID or admin login
 
 4. **Upload files**:
    - Select individual PDF files or ZIP archives containing PDFs
-   - Click "Process PDFs"
+   - Click "Process PDFs" - this creates a background job
 
-5. **Download results**:
-   - Download the `renamed_files.zip` containing all processed PDFs
-   - Review any skipped files listed on the results page
+5. **Monitor jobs**:
+   - Click "View Jobs" to see processing status and history
+   - Jobs show as "Queued", "Processing", or "Completed"
+
+6. **Download results**:
+   - When complete, download the ZIP file from the jobs page
+   - Review any skipped files listed for each job
 
 ## How It Works
 

pdf_renamer.py

@@ -1,8 +1,10 @@
 import os
 import tempfile
 import zipfile
+import threading
+import uuid
 from functools import wraps
-from flask import Flask, request, render_template_string, send_from_directory, redirect, url_for, session
+from flask import Flask, request, render_template_string, redirect, url_for, session, send_from_directory
 from werkzeug.utils import secure_filename
 from PyPDF2 import PdfReader
 from authlib.integrations.flask_client import OAuth
@@ -32,6 +34,11 @@ microsoft = oauth.register(
 
 ALLOWED_EXTENSIONS = {'pdf', 'zip'}
 
+# Global jobs storage (in production, use database)
+jobs = {}
+
+def login_required(f):
+
 def login_required(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
@@ -56,6 +63,68 @@ def extract_names_from_pdf(pdf_path):
     except Exception as e:
         return None
 
+def process_job(job_id, files):
+    jobs[job_id]['status'] = 'processing'
+    processed_files = []
+    temp_extract_dirs = []
+    
+    for file_info in files:
+        filepath = file_info['path']
+        filename = file_info['name']
+        
+        if filename.lower().endswith('.zip'):
+            # Extract ZIP
+            extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], f'extract_{job_id}_{os.path.splitext(filename)[0]}')
+            os.makedirs(extract_dir, exist_ok=True)
+            temp_extract_dirs.append(extract_dir)
+            with zipfile.ZipFile(filepath, 'r') as zip_ref:
+                zip_ref.extractall(extract_dir)
+            # Process PDFs in extract_dir
+            for root, dirs, files_in_zip in os.walk(extract_dir):
+                for f in files_in_zip:
+                    if f.lower().endswith('.pdf'):
+                        pdf_path = os.path.join(root, f)
+                        new_name = extract_names_from_pdf(pdf_path)
+                        if new_name:
+                            new_path = os.path.join(root, new_name)
+                            counter = 1
+                            while os.path.exists(new_path):
+                                base, ext = os.path.splitext(new_name)
+                                new_path = os.path.join(root, f"{base}_{counter}{ext}")
+                                counter += 1
+                            os.rename(pdf_path, new_path)
+                            processed_files.append(('success', os.path.relpath(new_path, extract_dir), new_path))
+                        else:
+                            processed_files.append(('skipped', f, None))
+        else:
+            # Single PDF
+            new_name = extract_names_from_pdf(filepath)
+            if new_name:
+                new_filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{new_name}")
+                counter = 1
+                while os.path.exists(new_filepath):
+                    base, ext = os.path.splitext(new_name)
+                    new_filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{base}_{counter}{ext}")
+                    counter += 1
+                os.rename(filepath, new_filepath)
+                processed_files.append(('success', os.path.basename(new_filepath), new_filepath))
+            else:
+                processed_files.append(('skipped', filename, None))
+    
+    # Create ZIP of all successful renames
+    successful_files = [p for p in processed_files if p[0] == 'success']
+    if successful_files:
+        zip_output = os.path.join(app.config['UPLOAD_FOLDER'], f'renamed_{job_id}.zip')
+        with zipfile.ZipFile(zip_output, 'w') as zip_out:
+            for _, rel_name, full_path in successful_files:
+                zip_out.write(full_path, rel_name)
+        jobs[job_id]['zip_path'] = zip_output
+        jobs[job_id]['zip_name'] = f'renamed_{job_id}.zip'
+    
+    jobs[job_id]['status'] = 'completed'
+    jobs[job_id]['processed_files'] = processed_files
+    jobs[job_id]['skipped'] = [p[1] for p in processed_files if p[0] == 'skipped']
+
 @app.route('/login')
 def login():
     return render_template_string('''
@@ -151,100 +220,36 @@ def logout():
 def upload_files():
     if request.method == 'POST':
         files = request.files.getlist('files')
-        processed_files = []
-        temp_extract_dirs = []
+        if not files or all(not f.filename for f in files):
+            return redirect(url_for('upload_files'))
+        
+        job_id = str(uuid.uuid4())
+        jobs[job_id] = {
+            'status': 'queued',
+            'files': [],
+            'zip_path': None,
+            'zip_name': None,
+            'processed_files': [],
+            'skipped': []
+        }
+        
+        file_list = []
         for file in files:
             if file and allowed_file(file.filename):
                 filename = secure_filename(file.filename)
-                filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
+                filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
                 file.save(filepath)
-                if filename.lower().endswith('.zip'):
-                    # Extract ZIP
-                    extract_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'extract_' + os.path.splitext(filename)[0])
-                    os.makedirs(extract_dir, exist_ok=True)
-                    temp_extract_dirs.append(extract_dir)
-                    with zipfile.ZipFile(filepath, 'r') as zip_ref:
-                        zip_ref.extractall(extract_dir)
-                    # Process PDFs in extract_dir
-                    for root, dirs, files_in_zip in os.walk(extract_dir):
-                        for f in files_in_zip:
-                            if f.lower().endswith('.pdf'):
-                                pdf_path = os.path.join(root, f)
-                                new_name = extract_names_from_pdf(pdf_path)
-                                if new_name:
-                                    new_path = os.path.join(root, new_name)
-                                    counter = 1
-                                    while os.path.exists(new_path):
-                                        base, ext = os.path.splitext(new_name)
-                                        new_path = os.path.join(root, f"{base}_{counter}{ext}")
-                                        counter += 1
-                                    os.rename(pdf_path, new_path)
-                                    processed_files.append(('success', os.path.relpath(new_path, extract_dir), new_path))
-                                else:
-                                    processed_files.append(('skipped', f, None))
-                else:
-                    # Single PDF
-                    new_name = extract_names_from_pdf(filepath)
-                    if new_name:
-                        new_filepath = os.path.join(app.config['UPLOAD_FOLDER'], new_name)
-                        counter = 1
-                        while os.path.exists(new_filepath):
-                            base, ext = os.path.splitext(new_name)
-                            new_filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{base}_{counter}{ext}")
-                            counter += 1
-                        os.rename(filepath, new_filepath)
-                        processed_files.append(('success', os.path.basename(new_filepath), new_filepath))
-                    else:
-                        processed_files.append(('skipped', filename, None))
+                file_list.append({'name': filename, 'path': filepath})
         
-        # Create ZIP of all successful renames
-        successful_files = [p for p in processed_files if p[0] == 'success']
-        if successful_files:
-            zip_output = os.path.join(app.config['UPLOAD_FOLDER'], 'renamed_files.zip')
-            with zipfile.ZipFile(zip_output, 'w') as zip_out:
-                for _, rel_name, full_path in successful_files:
-                    zip_out.write(full_path, rel_name)
-            zip_name = 'renamed_files.zip'
-        else:
-            zip_name = None
+        jobs[job_id]['files'] = file_list
         
-        skipped = [p[1] for p in processed_files if p[0] == 'skipped']
+        # Start processing in background
+        thread = threading.Thread(target=process_job, args=(job_id, file_list))
+        thread.daemon = True
+        thread.start()
         
-        return render_template_string('''
-        <!DOCTYPE html>
-        <html>
-        <head>
-            <title>Results</title>
-            <style>
-                body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
-                img { margin-bottom: 20px; }
-                h1 { color: #333; }
-                p { font-size: 18px; }
-                a { color: #007bff; text-decoration: none; padding: 10px 20px; background-color: #007bff; color: white; border-radius: 5px; }
-                a:hover { background-color: #0056b3; }
-                ul { list-style-type: none; padding: 0; }
-                li { background-color: #fff; margin: 5px 0; padding: 10px; border-radius: 5px; box-shadow: 0 0 5px rgba(0,0,0,0.1); }
-                .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; }
-                .logout:hover { background-color: #c82333; }
-            </style>
-        </head>
-        <body>
-            <a href="/logout" class="logout">Logout</a>
-            <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
-            <h1>Processing Results</h1>
-            {% if zip_name %}
-            <p><a href="/download/{{ zip_name }}">Download Renamed PDFs (ZIP)</a></p>
-            {% endif %}
-            <h2>Skipped Files:</h2>
-            <ul>
-            {% for file in skipped %}
-                <li>{{ file }}</li>
-            {% endfor %}
-            </ul>
-            <a href="/">Upload more files</a>
-        </body>
-        </html>
-        ''', zip_name=zip_name, skipped=skipped)
+        return redirect(url_for('jobs'))
+    
     return render_template_string('''
     <!DOCTYPE html>
     <html>
@@ -260,10 +265,15 @@ def upload_files():
             input[type="submit"]:hover { background-color: #218838; }
             .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
             .logout:hover { background-color: #c82333; }
+            .nav { margin-bottom: 20px; }
+            .nav a { margin: 0 10px; color: #007bff; text-decoration: none; }
         </style>
     </head>
     <body>
         <a href="/logout" class="logout">Logout</a>
+        <div class="nav">
+            <a href="/jobs">View Jobs</a>
+        </div>
         <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
         <h1>Upload PDF Files or ZIP to Rename</h1>
         <form method="post" enctype="multipart/form-data">
@@ -275,10 +285,66 @@ def upload_files():
     </html>
     ''')
 
-@app.route('/download/<filename>')
+@app.route('/jobs')
 @login_required
-def download_file(filename):
-    return send_from_directory(app.config['UPLOAD_FOLDER'], filename, as_attachment=True)
+def jobs():
+    return render_template_string('''
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <title>Jobs</title>
+        <style>
+            body { font-family: Arial, sans-serif; background-color: #f4f4f4; margin: 0; padding: 20px; text-align: center; }
+            img { margin-bottom: 20px; }
+            h1 { color: #333; }
+            .job { background-color: #fff; margin: 10px auto; padding: 15px; border-radius: 10px; box-shadow: 0 0 10px rgba(0,0,0,0.1); max-width: 800px; text-align: left; }
+            .status { padding: 5px 10px; border-radius: 5px; color: white; font-weight: bold; }
+            .status.queued { background-color: #ffc107; }
+            .status.processing { background-color: #17a2b8; }
+            .status.completed { background-color: #28a745; }
+            .download-btn { background-color: #007bff; color: white; text-decoration: none; padding: 8px 16px; border-radius: 5px; margin-left: 10px; }
+            .download-btn:hover { background-color: #0056b3; }
+            .logout { position: absolute; top: 20px; right: 20px; background-color: #dc3545; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; }
+            .logout:hover { background-color: #c82333; }
+            .nav { margin-bottom: 20px; }
+            .nav a { margin: 0 10px; color: #007bff; text-decoration: none; }
+            ul { list-style-type: none; padding: 0; }
+            li { background-color: #f8f9fa; margin: 2px 0; padding: 5px; border-radius: 3px; }
+        </style>
+    </head>
+    <body>
+        <a href="/logout" class="logout">Logout</a>
+        <div class="nav">
+            <a href="/">Upload Files</a>
+        </div>
+        <img src="/static/trbc-logo-full.png" alt="Logo" width="200">
+        <h1>Processing Jobs</h1>
+        {% for job_id, job in jobs.items() %}
+        <div class="job">
+            <h3>Job {{ job_id[:8] }}</h3>
+            <p><strong>Status:</strong> <span class="status {{ job.status }}">{{ job.status.title() }}</span></p>
+            <p><strong>Files:</strong> {{ job.files|length }}</p>
+            {% if job.status == 'completed' %}
+                {% if job.zip_name %}
+                <p><a href="/download/{{ job.zip_name }}" class="download-btn">Download ZIP</a></p>
+                {% endif %}
+                {% if job.skipped %}
+                <p><strong>Skipped Files:</strong></p>
+                <ul>
+                {% for file in job.skipped %}
+                    <li>{{ file }}</li>
+                {% endfor %}
+                </ul>
+                {% endif %}
+            {% endif %}
+        </div>
+        {% endfor %}
+        {% if not jobs %}
+        <p>No jobs yet. <a href="/">Upload some files</a> to get started.</p>
+        {% endif %}
+    </body>
+    </html>
+    ''', jobs=jobs)
 
 if __name__ == '__main__':
     app.run(debug=True, port=3000)