Update docker-compose.yml

.dockerignore

@@ -0,0 +1,16 @@
+# Exclude everything
+*
+
+# Include specific files and directories
+!start.sh
+!requirements.txt
+!app.py
+!celery_worker.py
+!models.py
+!extensions.py
+!utils/
+!routes/
+!templates/
+!static/
+!migrations/
+!uploads/ 

Dockerfile

@@ -5,6 +5,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     libpq-dev \
     curl \
+    wget \
     netcat-traditional \
     && rm -rf /var/lib/apt/lists/*
 
@@ -14,42 +15,22 @@ RUN useradd -m -u 1000 celery
 # Set working directory
 WORKDIR /app
 
-# Copy requirements first to leverage Docker cache
+# Copy the entire application
-COPY requirements.txt .
+COPY . /app/
+
+# Set up start.sh
+RUN chmod +x /app/start.sh && \
+    chown celery:celery /app/start.sh
+
+# Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy application code
-COPY . .
-
 # Create necessary directories and set permissions
-RUN mkdir -p /app/uploads /app/static/uploads && \
+RUN mkdir -p /data/rooms && \
+    chown -R celery:celery /data && \
+    chmod -R 755 /data && \
     chown -R celery:celery /app
 
-# Create and set up startup script
-RUN echo '#!/bin/bash\n\
-echo "Waiting for database..."\n\
-while ! nc -z db 5432; do\n\
-  sleep 0.1\n\
-done\n\
-echo "Database is ready!"\n\
-\n\
-echo "Waiting for Redis..."\n\
-while ! nc -z redis 6379; do\n\
-  sleep 0.1\n\
-done\n\
-echo "Redis is ready!"\n\
-\n\
-echo "Running database migrations..."\n\
-flask db upgrade\n\
-\n\
-echo "Creating admin user..."\n\
-flask create-admin\n\
-\n\
-echo "Starting application..."\n\
-exec "$@"' > /app/start.sh && \
-    chmod +x /app/start.sh && \
-    chown celery:celery /app/start.sh
-
 # Switch to non-root user
 USER celery
 

app.py

@@ -13,6 +13,7 @@ import click
 from utils import timeago
 from extensions import db, login_manager, csrf
 from utils.email_templates import create_default_templates
+from celery_worker import init_celery, celery
 
 # Load environment variables
 load_dotenv()
@@ -34,6 +35,9 @@ def create_app():
     login_manager.login_view = 'auth.login'
     csrf.init_app(app)
 
+    # Initialize Celery
+    init_celery(app)
+
     @app.context_processor
     def inject_csrf_token():
         return dict(csrf_token=generate_csrf())
@@ -58,9 +62,12 @@ def create_app():
         try:
             # Check database connection
             db.session.execute('SELECT 1')
+            # Check Redis connection
+            celery.control.inspect().ping()
             return jsonify({
                 'status': 'healthy',
-                'database': 'connected'
+                'database': 'connected',
+                'redis': 'connected'
             }), 200
         except Exception as e:
             return jsonify({

celery_worker.py

@@ -0,0 +1,51 @@
+from celery import Celery
+from flask import current_app
+import os
+import logging
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Get Redis URL from environment variable or use default
+REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
+
+# Configure Celery
+celery = Celery(
+    'docupulse',
+    backend=REDIS_URL,
+    broker=REDIS_URL,
+    # Add some default configuration
+    task_serializer='json',
+    accept_content=['json'],
+    result_serializer='json',
+    timezone='UTC',
+    enable_utc=True,
+    # Add retry configuration
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_default_retry_delay=300,  # 5 minutes
+    task_max_retries=3
+)
+
+def init_celery(app):
+    """Initialize Celery with Flask app context"""
+    celery.conf.update(app.config)
+
+    class ContextTask(celery.Task):
+        """Celery task that runs within Flask app context"""
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return self.run(*args, **kwargs)
+
+        def on_failure(self, exc, task_id, args, kwargs, einfo):
+            """Handle task failure"""
+            logger.error(f'Task {task_id} failed: {exc}')
+            super().on_failure(exc, task_id, args, kwargs, einfo)
+
+        def on_retry(self, exc, task_id, args, kwargs, einfo):
+            """Handle task retry"""
+            logger.warning(f'Task {task_id} is being retried: {exc}')
+            super().on_retry(exc, task_id, args, kwargs, einfo)
+
+    celery.Task = ContextTask
+    return celery 

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   web:
     build: .
@@ -13,13 +11,17 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
       - POSTGRES_DB=docupulse
+      - REDIS_URL=redis://redis:6379/0
     volumes:
-      - docupulse_uploads:/app/uploads
+      - uploads:/data
     depends_on:
-      - db
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
     restart: unless-stopped
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5000/health"]
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:5000/health"]
       interval: 30s
       timeout: 10s
       retries: 3
@@ -36,7 +38,7 @@ services:
       - POSTGRES_PASSWORD=postgres
       - POSTGRES_DB=docupulse
     volumes:
-      - docupulse_postgres_data:/var/lib/postgresql/data
+      - postgres_data:/var/lib/postgresql/data
     restart: unless-stopped
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -44,18 +46,46 @@ services:
       timeout: 10s
       retries: 3
 
+  redis:
+    image: redis:7
+    ports:
+      - "26379:6379"
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  celery_worker:
+    build:
+      context: https://git.kobeamerijckx.com/Kobe/docupulse.git
+      dockerfile: Dockerfile
+    command: celery -A celery_worker.celery worker --loglevel=info
     volumes:
-  docupulse_postgres_data:
+      - .:/app
-    name: docupulse_${COMPOSE_PROJECT_NAME:-default}_postgres_data
+    environment:
-    driver: local
+      - FLASK_APP=app.py
-    driver_opts:
+      - FLASK_ENV=development
-      type: none
+      - DATABASE_URL=postgresql://postgres:postgres@db:5432/docupulse
-      device: /var/lib/docupulse/postgres/${COMPOSE_PROJECT_NAME:-default}
+      - REDIS_URL=redis://redis:6379/0
-      o: bind
+    depends_on:
-  docupulse_uploads:
+      db:
-    name: docupulse_${COMPOSE_PROJECT_NAME:-default}_uploads
+        condition: service_healthy
-    driver: local
+      redis:
-    driver_opts:
+        condition: service_healthy
-      type: none
+    restart: unless-stopped
-      device: /var/lib/docupulse/uploads/${COMPOSE_PROJECT_NAME:-default}
+    healthcheck:
-      o: bind
+      test: ["CMD", "celery", "-A", "celery_worker.celery", "inspect", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M
+
+volumes:
+  postgres_data:
+  uploads:

entrypoint.sh

@@ -6,6 +6,7 @@ echo "POSTGRES_USER: $POSTGRES_USER"
 echo "POSTGRES_PASSWORD: $POSTGRES_PASSWORD"
 echo "POSTGRES_DB: $POSTGRES_DB"
 echo "DATABASE_URL: $DATABASE_URL"
+echo "REDIS_URL: $REDIS_URL"
 
 # Wait for the database to be ready
 echo "Waiting for database to be ready..."
@@ -14,6 +15,13 @@ while ! nc -z db 5432; do
 done
 echo "Database is ready!"
 
+# Wait for Redis to be ready
+echo "Waiting for Redis to be ready..."
+while ! nc -z redis 6379; do
+  sleep 0.1
+done
+echo "Redis is ready!"
+
 # Wait for PostgreSQL to be ready to accept connections
 echo "Waiting for PostgreSQL to accept connections..."
 until PGPASSWORD=$POSTGRES_PASSWORD psql -h db -U $POSTGRES_USER -d $POSTGRES_DB -c '\q'; do

requirements.txt

@@ -10,5 +10,8 @@ python-dotenv>=0.19.0
 psycopg2-binary==2.9.9
 gunicorn==21.2.0
 email_validator==2.1.0.post1
+celery>=5.3.0
+redis>=4.5.0
 alembic>=1.7.0
+flower>=2.0.0
 prometheus-client>=0.16.0 

utils/notification.py

@@ -10,6 +10,7 @@ from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 from email.utils import formatdate
 import json
+from celery_worker import celery
 
 logger = logging.getLogger(__name__)
 
@@ -30,9 +31,16 @@ def get_smtp_settings() -> Optional[Dict[str, Any]]:
         logger.error(f"Error retrieving SMTP settings: {str(e)}")
         return None
 
-def send_email_via_smtp(mail: Mail) -> bool:
+@celery.task
-    """Send an email synchronously"""
+def send_email_task(mail_id: int):
+    """Celery task to send an email asynchronously"""
     try:
+        # Get the mail record
+        mail = Mail.query.get(mail_id)
+        if not mail:
+            logger.error(f"Mail record not found for ID: {mail_id}")
+            return False
+
         # Get SMTP settings
         smtp_settings = get_smtp_settings()
         if not smtp_settings:
@@ -68,6 +76,20 @@ def send_email_via_smtp(mail: Mail) -> bool:
 
     except Exception as e:
         logger.error(f"Error sending email: {str(e)}")
+        if mail:
+            mail.status = 'failed'
+            mail.error_message = str(e)
+            db.session.commit()
+        return False
+
+def send_email_via_smtp(mail: Mail) -> bool:
+    """Queue an email to be sent asynchronously"""
+    try:
+        # Queue the email sending task
+        send_email_task.delay(mail.id)
+        return True
+    except Exception as e:
+        logger.error(f"Error queueing email: {str(e)}")
         mail.status = 'failed'
         mail.error_message = str(e)
         db.session.commit()