Convert print statements to proper logging for systemd capture
- Convert all print statements in process_pdfs_async to logger.info/logger.error - Update startup diagnostics to use logger.info instead of print - Ensure all diagnostic information is captured by systemd logs on remote server - Add detailed logging for PDF processing steps, file scanning, and error conditions
This commit is contained in:
1 parent
a6b751053a
commit
c9e817547a
3 files changed
+58
-53
No files matched your search
@@ -23,6 +23,11 @@ import secrets
|
|||||||
import base64
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
from flask_session import Session
|
from flask_session import Session
|
||||||
|
import logging
|
||||||
|
|
||||||
|
# Set up logging
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
LOGO_B64 = "iVBORw0KGgoAAAANSUhEUgAAB4AAAALOCAYAAAE/zz7rAAEY70lEQVR42uzdd7wcVf3/8aEXCSIgKBDTG+RuuQkBUkhC6J3QO6GDGAiBkEJCEkok9AC hBBBEBb9fwIICUkVAxfpTREAQpX1VikAsoJT7+5zduWSz2TLlnDNnZl5/PB+Ee+/uzpyZz3zmPW29crnTy7A/2v7MwulfBqzJcvF21aCAQQGnrGgpYFDAFDAFDAo4yeKlgEEBp7h4KWBQwBQwBQwKmAIGKGAKGMhLAb9HAYMCdtd32xTwPRQwKOD07j7fSgGDAk5vAZOBQQFTwBQwKGAKGKCAKWAgCwV8V4Di/YgCBgWc3u57IgUMCpjdZwoYuS7gcwwUFgUMCtiRTpkWZ1DAyGMHzlIRU8CggFPsNQoYeSvgz2asiLsoYOTtKHSWinflFeZv2wmsVKCAE/JcrHkbNZYVCrk4D5yWDnt44HkauwMrEyjgtB1xpvOCK7FWNFj8j8UC/Yu4RHwm9LSOHs9KBArYUCf/q9HP325nViBQwAYL+A2dn9dx7k2sMKCALRbwq9qKd+4SVhZQwJYL+EUtNydQvKCAHS3gUskrD+zvlQcN8EpjtvOKk6Z4hdnXeIUzLmLlAAXs4i40Cx0UcDoKeCkFDAo4Y0/jYKGDAqaAAQqYAgYo4OAFPHgACx0UcFoLmAUOCjhbBdwVUffrP6z7+cQWg93sPRr5MODnx/2cqK+J8hnKvyLOU9RlF/S9Jmt6n0bT1UvDfCgvUMDpKOCg07BeiNeHXYE8S6+pd5vmAo46/3EK2eTnr/B+FLA7BbxRhOm4IcQ8mV7hbBaJF2el11U8DdxvqYC7KGD3CrhL0wqlYwU2XcBxu3mU8XpU03gviDhNugu4K38FPO0S3XlQVwHr7JhBV6KeBla2oF2p1fu+oSm3t5q+Rr97XsMeio4CXjPEe++Xr9NIncPSUsCe5o7WrafhbtHoNYdoOGAVdTkFLeAPQn7W1AB/89m6/980ZgE3HP+sF/BuK7xuUH+TBbyfWKWJZu97lOaDSEGKrEtTAXsxd9WXOlLA7T7r4wgdP+o0rhlmvv5zYQD+5sq4CjZNUoGnBVy5TRxwCfMeP1H48GruLvQ6xs41fa9ut/dTAGb/mbCAf1cLeAbY65QYQvYVjf7tCNHoacYKOA48YgCjvpI2BR34BsiFFScAq7//Y9ijNnKBs+3BnnP1ShgCrhZAY+IsALsrCEDvx+xgKOuZLquzLrdQgGbuPIsztj1poDdLWBdR6FXttANehk+R93sPYLMz+oaI8RnDVy8srPGseEodAYLWDm+wXusHWGFqP3dJA27gHEjwrUWrt82fbFL9+++bbGAG73P1RSwGwVs8kos3cWgI+PHPQag48KYqPM1RNdVVFyJla0CjrIwOw0XcJji2lbjSmriNJKJa6F1XE7KtdAZKmBPw0pusoBtdHLT54GjjvedmnP0F7gbKZsFrFzTYuH92uL9wLpvMzyxxXyNbfSeMZdT/d9spPE65rBjc3LMi2xm8Uid8JdVmjdlgVfuGBr99VMXZu9JE2pMapZT8dgzefoGT+SIdFnlcjrmXq93JZWbKz6ZjmHDg7/2rMu8zv59lq3gBx0f+HXq7zt7bxZ5usvlcuO9makXai/c0EU86yr/GzfKredBxroyDgP7BZ+20y6ovKa0677h5mnGFUHvggu4wV42zmr9oYDbFni/ZZrtevuScJKPeeaFf9WFmbTz2pyt9Ryhd5sBT94+SJWXxcTbZ7L1a+YiTAdcb6SprzVyPbzeOy05YtdfTVO9+8LhUARqbt4l329Trnuu7BW3Eh3zLuh+Rffzbm2biN0fnXs1Vf2NJuXfr1bjIXaI+tYNs8nnd1w3rpRwFG/9Ozsq73C9Mva/133Aq4U7rBg713Zyi6QLts73HRtuXXrFSeMmo1JedTYSK8L1nUvMLqedPbr5XfCReGW78xFldd19u0V7DWjxnmlsduHX49kQ/RJ4Q4dGvr1FPDyxta9/pYI0xNl+u+IMb86pum2EH+r7d7r8pBB7d4/yrhHHZcwr4uyfjV/XeMOG2i+KODljYjxPt1/+7MY059EARubjo7ZK0aI4sEnNHvf/4k49lEK+FVNBazs0uY1B5ucLwq4cQFH6apRCn6VGJ+j62/r/+ZoXe8dsoCjLrMkCjhut/coYHsdOOxgh3mN50AB7xdxzEwU8GYa1wOXCvh0U/NFATcu4Ili35C7Oy+ELLAuDRsLE39bu2dgu4DHWSrglzXuQgeZvhspYHd3obtiLFhPw66V7l3oqBnNxC60+v+5Bqd/nRhjFWec1P9fyS60Gwexmi2gn4bIwC4UsPY9gZAFvJrFg1iexs/6isXPooDb6BDP1Ng8wGvU363V4GfPBHhdlAIO8t5R/jbsmLV975AFXD8dN0WcjjDz/KOQ8x3ns7o/5x6dn0UBw5iIBQwu5KCAKWAKmAIGBUwBU8AUMAVMAVPAFDAoYFDAFDAFDAqYAqaAKWBQwBQwBUwBU8CggCnguAW4Vd3//zhgoTZzbcTLFUEB566AjxTvxSw4W1j4FHAuCzhIcXR30evFX1r83fsGC/RscW+DW7+iFHr3v5eywlDAaSngjVus1C/7v3dhBj4T8u83qNnA1Bf2oiY/f6jBGOzPykMBu1TAs8XqOR+MJ2oKeA//333FpU0Kex//v92d/gBWKAqYo9Dpobr/qPKyZ0o9XlPkGzI+FDAFnO5TXP+siSMUMAVMAYMCBgUMCpgCBihgCti0Xax+3vARfGct+H5gDY5J5MorChgUsJEbHChgUMCO+06T4n2EAgYFTPelgEEBJ3BvMAUMCpgCpoBBAZvwbwoYFHB2H61DAYMCpoApYFDAFDBAAVPAAAVMAYMCTtQSChgUMA93p4BBAVPAFDAoYAoYoIApYFDAFDAFDAqYAgYoYAoYoIANFfDmg1ngoIDTWsCFGVc0GoCuiJq9vtlAB/27INMV9rXt/v6kkK85KMJntJuvX4dccZcGWEZ/0DBdyh4hp03nsm77Pvkp4FPPS0MBd8VdoA3+7sgIK5DOv1deijlPUQq420WGxzvu2HTFmY78FPBpF7hewF1xF2aElahLUwFP17iS6i5gZRuD421iPNuZlrsCLp440+UC7tIwLVFWoi5NBd8VsSs1+t1CAwVcP333axzvJAq4K4dHoYeZyDAmCzhqd41b6KYLuP73R2vowGNCfm7Q6f+Hho1hlz+9cdahNZpNR75OI01Z4GIB686gQYrsgwjFuFfd36xsYJ7D1nAOfq/v8JQ5u8ytIFr+bdZLuCoRRxnQFdpIkzBxd3ItFqJ3o6xsjX6m7grts0C7tvk529o2KhHHZu461tX7gq4WsQXmCrgKDnKZgF3aSxE9bM/1v1sVsRx+lBzAf9Jw3GLuAUcZ1wDT0suC7hNJ85aAXsaCljH9Cd1GslEAU+kgJP+atFiiQKO32FMXLhgooBtj3e31+p+9y4F7HYB614BdL5fqyuVWr3fzjELXPdpl7AFbOJiC0/Txi+3GfitAMX7QkoLWPdR6LgroY5zpfu0mK/rNBTwwBDL5AiN423ivH2YaemR1gJ+PEABn5+SAjZ9HrjR7zzDBTwv5nzFOYhV/16vaB7vlxKIUJ1ZOw98WYACHpjyAraxRQ/btXUcNFIe9n93qZed6DEhx+BwQ8Vn/T3SWsDTAxTweikqYJPXQuss4Cjv8SkL10KPKei7Yi1q4W0QYmzejDkdC/JQwFumrIBN3Y0UtYC315gVpwWcp3UsXUrZ7SmLezq69nIycTfSOQEK+BspLOAk7gcOeuqpYOCCh5bTWTzmDK9w9tW6CljnNd5dCRRwpu4HvlHLV4MGuzbaqNJBx0efDvUg+gw+ZaI8oB+PQgrinOtSW8CP6vp2hlDFM3m+vsLd+9DlpqNj7pLAry0eNy3SCl4uFoNcStpUx/wbG4/joP56CneLzWN9G+Qnt47K+DSdh3lLgl5Su5zOXpvI92QNCzdPZy/2OuZcq22dWe69pHjVPKS1gJ8p2/h+JNnNLg/sVyXfk1RZ6A2KuDx4YOAvRStvNbL5Sjpj0YqvOeuy8F93Ku8TeB5L5eX+v7TzPpXPXGE65t3Q+n3iPHNMCinsPBaPrdmIFQor/G3x6NNXLIC5SxrMf+mTf3dK52+4zKRwg85zedCAqohfTVuU9av2q38KUy9s+V5pLeB3kvyO4MIpcyvZrGHhLlfEy7bY5W1GB3rvzn69/CK8Ivy0DR+hbR5L2+9WnY4zLqrcSx34td2vC9p1x4w3+z3O51xf/ZyB/QO/prOvvwymB1wGas9h1LjQ09bZr/eywpV1Ksr85fdSymDvG+W1Jj4nyvy9EXF6tEyL2t1uWbyNN1DvRBzH+p/Nivi6MPN+R4h149ma160dY75epoDDv2+Q130jZgEr62gu4CdjFKSWjUmrnKppHput6CdZKGCTG99m0/cjCjjcwJnoYM0+J8hrF4f4jDAFfHXI+Q00zSEKuPu9Vqn52W9jdCqTBTzF//ftIQpxq5qf/TXGfFHAhjpwV8hCrP27Pg4UcJQxM1HAutYDkwV8RIhp1j1fFHDA951Y8+9NQiykBw3uipkqYhYQACIAJgAGAAAAABAAEwADAACAABgAAQAAQABMAAwAAAACYAAAAACRW0isL/YS24k9xD5BuJ/wmkG4xzDlBAAgACYABgAAAAEwAAAAgIRYWgwXk4JwX+FulAkAgACYABgAAAAEwAAAAACSOYO4h1hKzEV5AAAIgAmAAQAAQAAMAAAAAAAAEAATAAMAAIAAGAAAAAAAACAABgAAAAiAAQAAALhmTrGA/S/lAQAgACYABgAAQIz+P1GZ7DqsPrIUAAAAAElFTkSuQmCC"
|
LOGO_B64 = "iVBORw0KGgoAAAANSUhEUgAAB4AAAALOCAYAAAE/zz7rAAEY70lEQVR42uzdd7wcVf3/8aEXCSIgKBDTG+RuuQkBUkhC6J3QO6GDGAiBkEJCEkok9AC hBBBEBb9fwIICUkVAxfpTREAQpX1VikAsoJT7+5zduWSz2TLlnDNnZl5/PB+Ee+/uzpyZz3zmPW29crnTy7A/2v7MwulfBqzJcvF21aCAQQGnrGgpYFDAFDAFDAo4yeKlgEEBp7h4KWBQwBQwBQwKmAIGKGAKGMhLAb9HAYMCdtd32xTwPRQwKOD07j7fSgGDAk5vAZOBQQFTwBQwKGAKGKCAKWAgCwV8V4Di/YgCBgWc3u57IgUMCpjdZwoYuS7gcwwUFgUMCtiRTpkWZ1DAyGMHzlIRU8CggFPsNQoYeSvgz2asiLsoYOTtKHSWinflFeZv2wmsVKCAE/JcrHkbNZYVCrk4D5yWDnt44HkauwMrEyjgtB1xpvOCK7FWNFj8j8UC/Yu4RHwm9LSOHs9KBArYUCf/q9HP325nViBQwAYL+A2dn9dx7k2sMKCALRbwq9qKd+4SVhZQwJYL+EUtNydQvKCAHS3gUskrD+zvlQcN8EpjtvOKk6Z4hdnXeIUzLmLlAAXs4i40Cx0UcDoKeCkFDAo4Y0/jYKGDAqaAAQqYAgYo4OAFPHgACx0UcFoLmAUOCjhbBdwVUffrP6z7+cQWg93sPRr5MODnx/2cqK+J8hnKvyLOU9RlF/S9Jmt6n0bT1UvDfCgvUMDpKOCg07BeiNeHXYE8S6+pd5vmAo46/3EK2eTnr/B+FLA7BbxRhOm4IcQ8mV7hbBaJF2el11U8DdxvqYC7KGD3CrhL0wqlYwU2XcBxu3mU8XpU03gviDhNugu4K38FPO0S3XlQVwHr7JhBV6KeBla2oF2p1fu+oSm3t5q+Rr97XsMeio4CXjPEe++Xr9NIncPSUsCe5o7WrafhbtHoNYdoOGAVdTkFLeAPQn7W1AB/89m6/980ZgE3HP+sF/BuK7xuUH+TBbyfWKWJZu97lOaDSEGKrEtTAXsxd9WXOlLA7T7r4wgdP+o0rhlmvv5zYQD+5sq4CjZNUoGnBVy5TRxwCfMeP1H48GruLvQ6xs41fa9ut/dTAGb/mbCAf1cLeAbY65QYQvYVjf7tCNHoacYKOA48YgCjvpI2BR34BsiFFScAq7//Y9ijNnKBs+3BnnP1ShgCrhZAY+IsALsrCEDvx+xgKOuZLquzLrdQgGbuPIsztj1poDdLWBdR6FXttANehk+R93sPYLMz+oaI8RnDVy8srPGseEodAYLWDm+wXusHWGFqP3dJA27gHEjwrUWrt82fbFL9+++bbGAG73P1RSwGwVs8kos3cWgI+PHPQag48KYqPM1RNdVVFyJla0CjrIwOw0XcJji2lbjSmriNJKJa6F1XE7KtdAZKmBPw0pusoBtdHLT54GjjvedmnP0F7gbKZsFrFzTYuH92uL9wLpvMzyxxXyNbfSeMZdT/d9spPE65rBjc3LMi2xm8Uid8JdVmjdlgVfuGBr99VMXZu9JE2pMapZT8dgzefoGT+SIdFnlcjrmXq93JZWbKz6ZjmHDg7/2rMu8zv59lq3gBx0f+HXq7zt7bxZ5usvlcuO9makXai/c0EU86yr/GzfKredBxroyDgP7BZ+20y6ovKa0677h5mnGFUHvggu4wV42zmr9oYDbFni/ZZrtevuScJKPeeaFf9WFmbTz2pyt9Ryhd5sBT94+SJWXxcTbZ7L1a+YiTAdcb6SprzVyPbzeOy05YtdfTVO9+8LhUARqbt4l329Trnuu7BW3Eh3zLuh+Rffzbm2biN0fnXs1Vf2NJuXfr1bjIXaI+tYNs8nnd1w3rpRwFG/9Ozsq73C9Mva/133Aq4U7rBg713Zyi6QLts73HRtuXXrFSeMmo1JedTYSK8L1nUvMLqedPbr5XfCReGW78xFldd19u0V7DWjxnmlsduHX49kQ/RJ4Q4dGvr1FPDyxta9/pYI0xNl+u+IMb86pum2EH+r7d7r8pBB7d4/yrhHHZcwr4uyfjV/XeMOG2i+KODljYjxPt1/+7MY059EARubjo7ZK0aI4sEnNHvf/4k49lEK+FVNBazs0uY1B5ucLwq4cQFH6apRCn6VGJ+j62/r/+ZoXe8dsoCjLrMkCjhut/coYHsdOOxgh3mN50AB7xdxzEwU8GYa1wOXCvh0U/NFATcu4Ili35C7Oy+ELLAuDRsLE39bu2dgu4DHWSrglzXuQgeZvhspYHd3obtiLFhPw66V7l3oqBnNxC60+v+5Bqd/nRhjFWec1P9fyS60Gwexmi2gn4bIwC4UsPY9gZAFvJrFg1iexs/6isXPooDb6BDP1Ng8wGvU363V4GfPBHhdlAIO8t5R/jbsmLV975AFXD8dN0WcjjDz/KOQ8x3ns7o/5x6dn0UBw5iIBQwu5KCAKWAKmAIGBUwBU8AUMAVMAVPAFDAoYFDAFDAFDAqYAqaAKWBQwBQwBUwBU8CggCnguAW4Vd3//zhgoTZzbcTLFUEB566AjxTvxSw4W1j4FHAuCzhIcXR30evFX1r83fsGC/RscW+DW7+iFHr3v5eywlDAaSngjVus1C/7v3dhBj4T8u83qNnA1Bf2oiY/f6jBGOzPykMBu1TAs8XqOR+MJ2oKeA//333FpU0Kex//v92d/gBWKAqYo9Dpobr/qPKyZ0o9XlPkGzI+FDAFnO5TXP+siSMUMAVMAYMCBgUMCpgCBihgCti0Xax+3vARfGct+H5gDY5J5MorChgUsJEbHChgUMCO+06T4n2EAgYFTPelgEEBJ3BvMAUMCpgCpoBBAZvwbwoYFHB2H61DAYMCpoApYFDAFDBAAVPAAAVMAYMCTtQSChgUMA93p4BBAVPAFDAoYAoYoIApYFDAFDAFDAqYAgYoYAoYoIANFfDmg1ngoIDTWsCFGVc0GoCuiJq9vtlAB/27INMV9rXt/v6kkK85KMJntJuvX4dccZcGWEZ/0DBdyh4hp03nsm77Pvkp4FPPS0MBd8VdoA3+7sgIK5DOv1deijlPUQq420WGxzvu2HTFmY78FPBpF7hewF1xF2aElahLUwFP17iS6i5gZRuD421iPNuZlrsCLp440+UC7tIwLVFWoi5NBd8VsSs1+t1CAwVcP333axzvJAq4K4dHoYeZyDAmCzhqd41b6KYLuP73R2vowGNCfm7Q6f+Hho1hlz+9cdahNZpNR75OI01Z4GIB686gQYrsgwjFuFfd36xsYJ7D1nAOfq/v8JQ5u8ytIFr+bdZLuCoRRxnQFdpIkzBxd3ItFqJ3o6xsjX6m7grts0C7tvk529o2KhHHZu461tX7gq4WsQXmCrgKDnKZgF3aSxE9bM/1v1sVsRx+lBzAf9Jw3GLuAUcZ1wDT0suC7hNJ85aAXsaCljH9Cd1GslEAU+kgJP+atFiiQKO32FMXLhgooBtj3e31+p+9y4F7HYB614BdL5fqyuVWr3fzjELXPdpl7AFbOJiC0/Txi+3GfitAMX7QkoLWPdR6LgroY5zpfu0mK/rNBTwwBDL5AiN423ivH2YaemR1gJ+PEABn5+SAjZ9HrjR7zzDBTwv5nzFOYhV/16vaB7vlxKIUJ1ZOw98WYACHpjyAraxRQ/btXUcNFIe9n93qZed6DEhx+BwQ8Vn/T3SWsDTAxTweikqYJPXQuss4Cjv8SkL10KPKei7Yi1q4W0QYmzejDkdC/JQwFumrIBN3Y0UtYC315gVpwWcp3UsXUrZ7SmLezq69nIycTfSOQEK+BspLOAk7gcOeuqpYOCCh5bTWTzmDK9w9tW6CljnNd5dCRRwpu4HvlHLV4MGuzbaqNJBx0efDvUg+gw+ZaI8oB+PQgrinOtSW8CP6vp2hlDFM3m+vsLd+9DlpqNj7pLAry0eNy3SCl4uFoNcStpUx/wbG4/joP56CneLzWN9G+Qnt47K+DSdh3lLgl5Su5zOXpvI92QNCzdPZy/2OuZcq22dWe69pHjVPKS1gJ8p2/h+JNnNLg/sVyXfk1RZ6A2KuDx4YOAvRStvNbL5Sjpj0YqvOeuy8F93Ku8TeB5L5eX+v7TzPpXPXGE65t3Q+n3iPHNMCinsPBaPrdmIFQor/G3x6NNXLIC5SxrMf+mTf3dK52+4zKRwg85zedCAqohfTVuU9av2q38KUy9s+V5pLeB3kvyO4MIpcyvZrGHhLlfEy7bY5W1GB3rvzn69/CK8Ivy0DR+hbR5L2+9WnY4zLqrcSx34td2vC9p1x4w3+z3O51xf/ZyB/QO/prOvvwymB1wGas9h1LjQ09bZr/eywpV1Ksr85fdSymDvG+W1Jj4nyvy9EXF6tEyL2t1uWbyNN1DvRBzH+p/Nivi6MPN+R4h149ma160dY75epoDDv2+Q130jZgEr62gu4CdjFKSWjUmrnKppHput6CdZKGCTG99m0/cjCjjcwJnoYM0+J8hrF4f4jDAFfHXI+Q00zSEKuPu9Vqn52W9jdCqTBTzF//ftIQpxq5qf/TXGfFHAhjpwV8hCrP27Pg4UcJQxM1HAutYDkwV8RIhp1j1fFHDA951Y8+9NQiykBw3uipkqYhYQACIAJgAGAAAAABAAEwADAACAABgAAQAAQABMAAwAAAACYAAAAACRW0isL/YS24k9xD5BuJ/wmkG4xzDlBAAgACYABgAAAAEwAAAAgIRYWgwXk4JwX+FulAkAgACYABgAAAAEwAAAAACSOYO4h1hKzEV5AAAIgAmAAQAAQAAMAAAAAAAAEAATAAMAAIAAGAAAAAAAACAABgAAAAiAAQAAALhmTrGA/S/lAQAgACYABgAAQIz+P1GZ7DqsPrIUAAAAAElFTkSuQmCC"
|
||||||
|
|
||||||
@@ -54,13 +59,13 @@ except Exception as e:
|
|||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
# Startup diagnostics
|
# Startup diagnostics
|
||||||
print("=" * 60)
|
logger.info("=" * 60)
|
||||||
print("APPLICATION STARTUP DIAGNOSTICS")
|
logger.info("APPLICATION STARTUP DIAGNOSTICS")
|
||||||
print("=" * 60)
|
logger.info("=" * 60)
|
||||||
print(f"Working directory: {os.getcwd()}")
|
logger.info(f"Working directory: {os.getcwd()}")
|
||||||
print(f"App directory: {os.path.dirname(os.path.abspath(__file__))}")
|
logger.info(f"App directory: {os.path.dirname(os.path.abspath(__file__))}")
|
||||||
print(f"PDF Support (PyMuPDF): {PDF_SUPPORT}")
|
logger.info(f"PDF Support (PyMuPDF): {PDF_SUPPORT}")
|
||||||
print("=" * 60)
|
logger.info("=" * 60)
|
||||||
|
|
||||||
@app.context_processor
|
@app.context_processor
|
||||||
def inject_logo():
|
def inject_logo():
|
||||||
@@ -735,27 +740,27 @@ def process_pdfs(pdfdir, outputfolder, job_id=None):
|
|||||||
current_output_dir = outputfolder
|
current_output_dir = outputfolder
|
||||||
|
|
||||||
def process_pdfs_async(pdfdir, outputfolder, job_id):
|
def process_pdfs_async(pdfdir, outputfolder, job_id):
|
||||||
print("=" * 60)
|
logger.info("=" * 60)
|
||||||
print(f"PROCESS_PDFS_ASYNC CALLED")
|
logger.info(f"PROCESS_PDFS_ASYNC CALLED")
|
||||||
print(f" Job ID: {job_id}")
|
logger.info(f" Job ID: {job_id}")
|
||||||
print(f" PDF Dir: {pdfdir}")
|
logger.info(f" PDF Dir: {pdfdir}")
|
||||||
print(f" Output Folder: {outputfolder}")
|
logger.info(f" Output Folder: {outputfolder}")
|
||||||
print(f" PDF Dir Exists: {os.path.exists(pdfdir)}")
|
logger.info(f" PDF Dir Exists: {os.path.exists(pdfdir)}")
|
||||||
if os.path.exists(pdfdir):
|
if os.path.exists(pdfdir):
|
||||||
print(f" PDF Dir Writable: {os.access(pdfdir, os.W_OK)}")
|
logger.info(f" PDF Dir Writable: {os.access(pdfdir, os.W_OK)}")
|
||||||
print(f" PDF Dir Contents: {os.listdir(pdfdir)}")
|
logger.info(f" PDF Dir Contents: {os.listdir(pdfdir)}")
|
||||||
print(f" Output Dir Exists: {os.path.exists(outputfolder)}")
|
logger.info(f" Output Dir Exists: {os.path.exists(outputfolder)}")
|
||||||
if os.path.exists(outputfolder):
|
if os.path.exists(outputfolder):
|
||||||
print(f" Output Dir Writable: {os.access(outputfolder, os.W_OK)}")
|
logger.info(f" Output Dir Writable: {os.access(outputfolder, os.W_OK)}")
|
||||||
print(f" PDF Support Available: {PDF_SUPPORT}")
|
logger.info(f" PDF Support Available: {PDF_SUPPORT}")
|
||||||
print(f" Timestamp: {datetime.now()}")
|
logger.info(f" Timestamp: {datetime.now()}")
|
||||||
print("=" * 60)
|
logger.info("=" * 60)
|
||||||
|
|
||||||
global processing_complete
|
global processing_complete
|
||||||
|
|
||||||
if not PDF_SUPPORT:
|
if not PDF_SUPPORT:
|
||||||
error_msg = "PyMuPDF (fitz) is not installed - PDF processing cannot continue"
|
error_msg = "PyMuPDF (fitz) is not installed - PDF processing cannot continue"
|
||||||
print(f"ERROR: {error_msg}")
|
logger.error(f"ERROR: {error_msg}")
|
||||||
jobs = load_jobs()
|
jobs = load_jobs()
|
||||||
if job_id in jobs:
|
if job_id in jobs:
|
||||||
jobs[job_id]['status'] = 'failed'
|
jobs[job_id]['status'] = 'failed'
|
||||||
@@ -766,16 +771,16 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
jobs = load_jobs()
|
jobs = load_jobs()
|
||||||
print(f"Jobs loaded: {len(jobs)} total, job {job_id} exists: {job_id in jobs}")
|
logger.info(f"Jobs loaded: {len(jobs)} total, job {job_id} exists: {job_id in jobs}")
|
||||||
|
|
||||||
if job_id not in jobs:
|
if job_id not in jobs:
|
||||||
print(f"ERROR: Job {job_id} not found in jobs")
|
logger.error(f"ERROR: Job {job_id} not found in jobs")
|
||||||
return
|
return
|
||||||
|
|
||||||
status_file = 'status.txt'
|
status_file = 'status.txt'
|
||||||
with open(status_file, 'w') as status_f:
|
with open(status_file, 'w') as status_f:
|
||||||
status_f.write("Step 1/4: Finding PDF files...\n")
|
status_f.write("Step 1/4: Finding PDF files...\n")
|
||||||
print("Step 1/4: Finding PDF files...")
|
logger.info("Step 1/4: Finding PDF files...")
|
||||||
|
|
||||||
# Update job status
|
# Update job status
|
||||||
jobs[job_id]['message'] = "Step 1/4: Finding PDF files..."
|
jobs[job_id]['message'] = "Step 1/4: Finding PDF files..."
|
||||||
@@ -788,24 +793,24 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
missing_dir = os.path.join(outputfolder, 'Missing_Addresses')
|
missing_dir = os.path.join(outputfolder, 'Missing_Addresses')
|
||||||
os.makedirs(addresses_dir, exist_ok=True)
|
os.makedirs(addresses_dir, exist_ok=True)
|
||||||
os.makedirs(missing_dir, exist_ok=True)
|
os.makedirs(missing_dir, exist_ok=True)
|
||||||
print(f"Created output directories: {addresses_dir}, {missing_dir}")
|
logger.info(f"Created output directories: {addresses_dir}, {missing_dir}")
|
||||||
|
|
||||||
# Scan for PDF files
|
# Scan for PDF files
|
||||||
pdf_files = []
|
pdf_files = []
|
||||||
print(f"Scanning directory: {pdfdir}")
|
logger.info(f"Scanning directory: {pdfdir}")
|
||||||
for root, dirs, files in os.walk(pdfdir):
|
for root, dirs, files in os.walk(pdfdir):
|
||||||
print(f" Checking directory: {root} with {len(files)} files")
|
logger.info(f" Checking directory: {root} with {len(files)} files")
|
||||||
for f in files:
|
for f in files:
|
||||||
if f.endswith('.pdf'):
|
if f.endswith('.pdf'):
|
||||||
pdf_path = os.path.join(root, f)
|
pdf_path = os.path.join(root, f)
|
||||||
pdf_files.append(pdf_path)
|
pdf_files.append(pdf_path)
|
||||||
print(f" Found PDF: {f}")
|
logger.info(f" Found PDF: {f}")
|
||||||
|
|
||||||
print(f"Total PDFs found: {len(pdf_files)}")
|
logger.info(f"Total PDFs found: {len(pdf_files)}")
|
||||||
|
|
||||||
if len(pdf_files) == 0:
|
if len(pdf_files) == 0:
|
||||||
error_msg = f"No PDF files found in {pdfdir}"
|
error_msg = f"No PDF files found in {pdfdir}"
|
||||||
print(f"ERROR: {error_msg}")
|
logger.error(f"ERROR: {error_msg}")
|
||||||
jobs = load_jobs()
|
jobs = load_jobs()
|
||||||
jobs[job_id]['status'] = 'failed'
|
jobs[job_id]['status'] = 'failed'
|
||||||
jobs[job_id]['error'] = error_msg
|
jobs[job_id]['error'] = error_msg
|
||||||
@@ -816,7 +821,7 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
msg = f"Step 1/4: Found {len(pdf_files)} PDF files\n"
|
msg = f"Step 1/4: Found {len(pdf_files)} PDF files\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
|
|
||||||
# Update job status
|
# Update job status
|
||||||
jobs = load_jobs()
|
jobs = load_jobs()
|
||||||
@@ -830,7 +835,7 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
msg = "Step 2/4: Processing files...\n"
|
msg = "Step 2/4: Processing files...\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print("Step 2/4: Processing files...")
|
logger.info("Step 2/4: Processing files...")
|
||||||
|
|
||||||
# Update job status
|
# Update job status
|
||||||
jobs[job_id]['message'] = "Step 2/4: Processing files..."
|
jobs[job_id]['message'] = "Step 2/4: Processing files..."
|
||||||
@@ -840,16 +845,16 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
|
|
||||||
# Parallel processing
|
# Parallel processing
|
||||||
print(f"Processing {len(pdf_files)} files sequentially (avoiding multiprocessing issues)")
|
logger.info(f"Processing {len(pdf_files)} files sequentially (avoiding multiprocessing issues)")
|
||||||
results = []
|
results = []
|
||||||
for i, pdf_file in enumerate(pdf_files, 1):
|
for i, pdf_file in enumerate(pdf_files, 1):
|
||||||
print(f"Processing file {i}/{len(pdf_files)}: {os.path.basename(pdf_file)}")
|
logger.info(f"Processing file {i}/{len(pdf_files)}: {os.path.basename(pdf_file)}")
|
||||||
try:
|
try:
|
||||||
result = process_single_pdf(pdf_file)
|
result = process_single_pdf(pdf_file)
|
||||||
results.append(result)
|
results.append(result)
|
||||||
print(f" Result type: {result['type']}")
|
logger.info(f" Result type: {result['type']}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f" ERROR processing {pdf_file}: {e}")
|
logger.error(f" ERROR processing {pdf_file}: {e}")
|
||||||
results.append({
|
results.append({
|
||||||
'type': 'error',
|
'type': 'error',
|
||||||
'filename': os.path.basename(pdf_file),
|
'filename': os.path.basename(pdf_file),
|
||||||
@@ -864,10 +869,10 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
jobs[job_id]['message'] = f"Processing file {i}/{len(pdf_files)}"
|
jobs[job_id]['message'] = f"Processing file {i}/{len(pdf_files)}"
|
||||||
jobs[job_id]['progress'] = progress_percent
|
jobs[job_id]['progress'] = progress_percent
|
||||||
save_jobs(jobs)
|
save_jobs(jobs)
|
||||||
print(f" Progress: {progress_percent}%")
|
logger.info(f" Progress: {progress_percent}%")
|
||||||
|
|
||||||
# Process results sequentially
|
# Process results sequentially
|
||||||
print("Processing results and sorting files...")
|
logger.info("Processing results and sorting files...")
|
||||||
with open(os.path.join(addresses_dir, 'Addresses.csv'), 'w') as addr_csv:
|
with open(os.path.join(addresses_dir, 'Addresses.csv'), 'w') as addr_csv:
|
||||||
addr_csv.write("File,User,Street,City,State,Zip\n")
|
addr_csv.write("File,User,Street,City,State,Zip\n")
|
||||||
|
|
||||||
@@ -890,14 +895,14 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
miss_csv.write(f"{result['filename']}\t{result['user']}\n")
|
miss_csv.write(f"{result['filename']}\t{result['user']}\n")
|
||||||
shutil.copy(result['filepath'], os.path.join(missing_dir, result['filename']))
|
shutil.copy(result['filepath'], os.path.join(missing_dir, result['filename']))
|
||||||
elif result['type'] == 'error':
|
elif result['type'] == 'error':
|
||||||
print(f" Error with {result['filename']}: {result.get('error', 'Unknown error')}")
|
logger.error(f" Error with {result['filename']}: {result.get('error', 'Unknown error')}")
|
||||||
|
|
||||||
print(f"Results sorted. Found {len(page_groups)} page count groups for merging")
|
logger.info(f"Results sorted. Found {len(page_groups)} page count groups for merging")
|
||||||
|
|
||||||
msg = "Step 2/4: Sorting complete. Starting merging...\n"
|
msg = "Step 2/4: Sorting complete. Starting merging...\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
|
|
||||||
# Update job status
|
# Update job status
|
||||||
jobs[job_id]['message'] = "Step 2/4: Sorting complete. Starting merging..."
|
jobs[job_id]['message'] = "Step 2/4: Sorting complete. Starting merging..."
|
||||||
@@ -911,14 +916,14 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
msg = f"Step 3/4: Found {len(page_groups)} different page counts to merge\n"
|
msg = f"Step 3/4: Found {len(page_groups)} different page counts to merge\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
|
|
||||||
for pages, files in page_groups.items():
|
for pages, files in page_groups.items():
|
||||||
msg = f"Merging {len(files)} files with {pages} pages each\n"
|
msg = f"Merging {len(files)} files with {pages} pages each\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
time.sleep(0.05)
|
time.sleep(0.05)
|
||||||
try:
|
try:
|
||||||
merger = fitz.open()
|
merger = fitz.open()
|
||||||
@@ -932,12 +937,12 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
msg = f" -> Created {output_path}\n"
|
msg = f" -> Created {output_path}\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
msg = f" -> Error merging files with {pages} pages: {str(e)}\n"
|
msg = f" -> Error merging files with {pages} pages: {str(e)}\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.error(msg.strip())
|
||||||
time.sleep(0.05)
|
time.sleep(0.05)
|
||||||
else:
|
else:
|
||||||
msg = "Step 3/4: PDF merging skipped (PyMuPDF not available)\n"
|
msg = "Step 3/4: PDF merging skipped (PyMuPDF not available)\n"
|
||||||
@@ -949,7 +954,7 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
msg = f"Processing complete! Processed {len(pdf_files)} files total\n"
|
msg = f"Processing complete! Processed {len(pdf_files)} files total\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
|
|
||||||
# Update job status
|
# Update job status
|
||||||
jobs[job_id]['message'] = f"Processing complete! Processed {len(pdf_files)} files total"
|
jobs[job_id]['message'] = f"Processing complete! Processed {len(pdf_files)} files total"
|
||||||
@@ -962,7 +967,7 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
msg = "Step 4/4: Creating ZIP file...\n"
|
msg = "Step 4/4: Creating ZIP file...\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
|
|
||||||
# Update job status
|
# Update job status
|
||||||
jobs[job_id]['message'] = "Step 4/4: Creating ZIP file..."
|
jobs[job_id]['message'] = "Step 4/4: Creating ZIP file..."
|
||||||
@@ -992,18 +997,18 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
jobs[job_id]['message'] = f'Processing complete! ZIP created: {zip_filename}'
|
jobs[job_id]['message'] = f'Processing complete! ZIP created: {zip_filename}'
|
||||||
jobs[job_id]['progress'] = 100
|
jobs[job_id]['progress'] = 100
|
||||||
save_jobs(jobs)
|
save_jobs(jobs)
|
||||||
print(f"Job {job_id} completed successfully")
|
logger.info(f"Job {job_id} completed successfully")
|
||||||
|
|
||||||
msg = f"ZIP created: {zip_filename}\n"
|
msg = f"ZIP created: {zip_filename}\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.info(msg.strip())
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
msg = f"Error creating ZIP: {str(e)}\n"
|
msg = f"Error creating ZIP: {str(e)}\n"
|
||||||
with open(status_file, 'a') as status_f:
|
with open(status_file, 'a') as status_f:
|
||||||
status_f.write(msg)
|
status_f.write(msg)
|
||||||
print(msg.strip())
|
logger.error(msg.strip())
|
||||||
# Still save job as failed
|
# Still save job as failed
|
||||||
jobs = load_jobs()
|
jobs = load_jobs()
|
||||||
jobs[job_id]['status'] = 'failed'
|
jobs[job_id]['status'] = 'failed'
|
||||||
@@ -1012,7 +1017,7 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
save_jobs(jobs)
|
save_jobs(jobs)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Processing failed for job {job_id}: {e}")
|
logger.error(f"Processing failed for job {job_id}: {e}")
|
||||||
msg = f"Processing error: {e}"
|
msg = f"Processing error: {e}"
|
||||||
with open('status.txt', 'w') as f:
|
with open('status.txt', 'w') as f:
|
||||||
f.write(msg)
|
f.write(msg)
|
||||||
@@ -1024,7 +1029,7 @@ def process_pdfs_async(pdfdir, outputfolder, job_id):
|
|||||||
jobs[job_id]['message'] = msg
|
jobs[job_id]['message'] = msg
|
||||||
save_jobs(jobs)
|
save_jobs(jobs)
|
||||||
processing_complete = True # Still mark as complete to allow retry
|
processing_complete = True # Still mark as complete to allow retry
|
||||||
print(f"Processing finished for job {job_id} at {datetime.now()}")
|
logger.info(f"Processing finished for job {job_id} at {datetime.now()}")
|
||||||
|
|
||||||
@app.route('/download/<job_id>')
|
@app.route('/download/<job_id>')
|
||||||
@login_required
|
@login_required
|
||||||
|
|||||||
+1
-1
@@ -4,4 +4,4 @@ Werkzeug==3.0.1
|
|||||||
waitress==3.0.0
|
waitress==3.0.0
|
||||||
requests-oauthlib==2.0.0
|
requests-oauthlib==2.0.0
|
||||||
python-dotenv==1.2.1
|
python-dotenv==1.2.1
|
||||||
# PyMuPDF==1.23.7 # Optional - will be installed separately if possible
|
PyMuPDF==1.23.7 # Optional - will be installed separately if possible
|
||||||
Binary file not shown.
Reference in new issue
Block a user