如何在 devcontainer 中设置 celery、celerybeat、flower 和 Fastapi，以便 celery 容器仅在某些操作上启动

问题描述 投票：0回答：1

这可能是一个很长的上下文，所以请耐心等待。我有一个 fastapi 应用程序，可以安排在后台执行的任务。后台任务应该每 5 分钟运行一次。问题是，在我开发的大部分时间里，我不希望后台任务运行（又名 celery 处于活动状态）。我需要一种方法来根据需要启动和停止它。我正在使用 Visual Studio 代码。这是我到目前为止所尝试过的

原生开发容器

我的

.devcontainer/docker-compose.yaml
看起来像这样

services:
  trader:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
      - ~/.ssh:/home/monkey/.ssh:ro  # Mount SSH keys
      - ~/.gitconfig:/home/monkey/.gitconfig:cached
      - ruff_cache:/app/.ruff_cache
    command: sleep infinity
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    depends_on:
      - redis
      - postgres

  celery:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
    command: celery -A src.celery worker --loglevel=debug
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    depends_on:
      - redis
      - postgres

  celerybeat:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
    command: celery -A src.celery beat --loglevel=debug
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    depends_on:
      - redis
      - postgres

  redis:
    image: redis:latest
    networks:
      - trader-network

  postgres:
    build:
      context: .
      dockerfile: Dockerfile.postgres
      args:
        REBUILD_DATE: ${REBUILD_DATE:-1}
    environment:
      - POSTGRES_USER=myuser
      - POSTGRES_PASSWORD=mypassword
      - POSTGRES_MULTIPLE_DATABASES=db
    volumes:
      - postgres_data_trader:/var/lib/postgresql/data
      - ./scripts/init_multiple_db.sh:/docker-entrypoint-initdb.d/init_multiple_db.sh
      - ./scripts/pg_custom_entrypoint.sh:/usr/local/bin/custom-entrypoint.sh
    ports:
      - "6543:5432" # Expose the port to the host machine for debugging
    networks:
      - trader-network
    entrypoint: ["/usr/local/bin/custom-entrypoint.sh"]
    command: ["postgres"]

  flower:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
    ports:
      - "5555:5555"  # Expose Flower's web interface
    command: celery -A src.celery flower --port=5555 --broker=redis://redis:6379/0
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    depends_on:
      - redis
      - postgres

networks:
  trader-network:
    driver: bridge

volumes:
  ruff_cache:
    driver: local
  postgres_data_trader:
    driver: local

我的

devcontainer.json
看起来像这样

{
  "name": "Python 3.12.3",
  "dockerComposeFile": ["./docker-compose.yml"],
  "service": "trader",
  "workspaceFolder": "/app/",
  "customizations": {
      "vscode": {
          "settings": {
              "python.defaultInterpreterPath": "/usr/local/bin/python",
              "python.linting.enabled": true,
              "python.linting.pylintEnabled": true,
              "python.formatting.autopep8Path": "/usr/local/bin/autopep8",
              "python.formatting.blackPath": "/usr/local/bin/black",
              "python.formatting.yapfPath": "/usr/local/bin/yapf",
              "python.linting.banditPath": "/usr/local/bin/bandit",
              "python.linting.flake8Path": "/usr/local/bin/flake8",
              "python.linting.mypyPath": "/usr/local/bin/mypy",
              "python.linting.pycodestylePath": "/usr/local/bin/pycodestyle",
              "python.linting.pydocstylePath": "/usr/local/bin/pydocstyle"
          },
          "extensions": [
            "ms-python.python",
            "ms-python.vscode-pylance",
            "ms-toolsai.jupyter",
            "ms-azuretools.vscode-docker",
            "github.vscode-github-actions",
            "vscodevim.vim"
          ]
      }
  },
  "forwardPorts": [8000, 5432, 5555],
  "remoteUser": "monkey",
  "overrideCommand": false,
  "runServices": ["trader", "redis", "postgres"]
}

当我在 devcontainer 中时，这将立即启动所有服务，即 trader、postgres、redis、celery、celerybeat、flower 等。问题是我正在使用一些付费 API，在开发时我不需要总是使用我的 celery。

带有配置文件的开发容器

在此方法中，我创建了一个配置文件，默认情况下只有 Trader、Redis 和 Postgres 会打开。然后我设置了 docker 套接字，以便当我从我的交易者 devcontainer 中运行命令时，它会在主机中旋转芹菜和花等容器。

services:
  trader:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
      - ~/.ssh:/home/monkey/.ssh:ro  # Mount SSH keys
      - ~/.gitconfig:/home/monkey/.gitconfig:cached
      - ruff_cache:/app/.ruff_cache
    command: sleep infinity
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    depends_on:
      - redis
      - postgres
    profiles: ["default"]

  celery:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
    command: celery -A src.celery worker --loglevel=debug
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    profiles: ["optional"]

  celerybeat:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
    command: celery -A src.celery beat --loglevel=debug
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    profiles: ["optional"]

  redis:
    image: redis:latest
    networks:
      - trader-network
    profiles: ["default"]

  postgres:
    build:
      context: .
      dockerfile: Dockerfile.postgres
      args:
        REBUILD_DATE: ${REBUILD_DATE:-1}
    environment:
      - POSTGRES_USER=myuser
      - POSTGRES_PASSWORD=mypassword
      - POSTGRES_MULTIPLE_DATABASES=db
    volumes:
      - postgres_data_trader:/var/lib/postgresql/data
      - ./scripts/init_multiple_db.sh:/docker-entrypoint-initdb.d/init_multiple_db.sh
      - ./scripts/pg_custom_entrypoint.sh:/usr/local/bin/custom-entrypoint.sh
    ports:
      - "6543:5432" # Expose the port to the host machine for debugging
    networks:
      - trader-network
    entrypoint: ["/usr/local/bin/custom-entrypoint.sh"]
    command: ["postgres"]
    profiles: ["default"]

  flower:
    build:
      context: ..
      dockerfile: .devcontainer/Dockerfile
    volumes:
      - ../:/app:cached
    ports:
      - "5555:5555"  # Expose Flower's web interface
    command: celery -A src.celery flower --port=5555 --broker=redis://redis:6379/0
    environment:
      - AGENT_CACHE_REDIS_HOST=redis
      - DB_URL=postgresql://myuser:mypassword@postgres:5432/db
    env_file:
      - .env.local
    networks:
      - trader-network
    profiles: ["optional"]

networks:
  trader-network:
    driver: bridge

volumes:
  ruff_cache:
    driver: local
  postgres_data_trader:
    driver: local

还有我的 devcontainer.json 

{
  "name": "Python 3.12.3",
  "dockerComposeFile": ["./docker-compose.yml"],
  "service": "trader",
  "workspaceFolder": "/app/",
  "customizations": {
      "vscode": {
          "settings": {
              "python.defaultInterpreterPath": "/usr/local/bin/python",
              "python.linting.enabled": true,
              "python.linting.pylintEnabled": true,
              "python.formatting.autopep8Path": "/usr/local/bin/autopep8",
              "python.formatting.blackPath": "/usr/local/bin/black",
              "python.formatting.yapfPath": "/usr/local/bin/yapf",
              "python.linting.banditPath": "/usr/local/bin/bandit",
              "python.linting.flake8Path": "/usr/local/bin/flake8",
              "python.linting.mypyPath": "/usr/local/bin/mypy",
              "python.linting.pycodestylePath": "/usr/local/bin/pycodestyle",
              "python.linting.pydocstylePath": "/usr/local/bin/pydocstyle"
          },
          "extensions": [
            "ms-python.python",
            "ms-python.vscode-pylance",
            "ms-toolsai.jupyter",
            "ms-azuretools.vscode-docker",
            "github.vscode-github-actions",
            "vscodevim.vim"
          ]
      }
  },
  "mounts": [
    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
  ],
  "forwardPorts": [8000, 5432, 5555],
  "remoteUser": "monkey",
  "overrideCommand": false,
  "runServices": ["trader", "redis", "postgres"]
}

问题是，由于某种原因，当我启动 celery 容器时，我不断收到错误

[+] Running 3/0
 ✔ Container devcontainer-celerybeat-1  Created                                                                                           0.0s 
 ✔ Container devcontainer-flower-1      Created                                                                                           0.0s 
 ✔ Container devcontainer-celery-1      Created                                                                                           0.0s 
Attaching to celery-1, celerybeat-1, flower-1
celery-1      | Usage: celery [OPTIONS] COMMAND [ARGS]...
celery-1      | Try 'celery --help' for help.
celery-1      | 
celery-1      | Error: Invalid value for '-A' / '--app': 
celery-1      | Unable to load celery application.
celery-1      | The module src.celery was not found.
flower-1      | Usage: celery [OPTIONS] COMMAND [ARGS]...
flower-1      | Try 'celery --help' for help.
flower-1      | 
flower-1      | Error: Invalid value for '-A' / '--app': 
flower-1      | Unable to load celery application.
flower-1      | The module src.celery was not found.
celerybeat-1  | Usage: celery [OPTIONS] COMMAND [ARGS]...
celerybeat-1  | Try 'celery --help' for help.
celerybeat-1  | 
celerybeat-1  | Error: Invalid value for '-A' / '--app': 
celerybeat-1  | Unable to load celery application.
celerybeat-1  | The module src.celery was not found.
celery-1 exited with code 2
celerybeat-1 exited with code 2
flower-1 exited with code 2

所以我很困惑（也很沮丧）。当然，我已经尝试了所有 GPT 和 Claudes，但我认为这是需要人脑来解决的问题之一。所以请帮助我。

docker docker-compose celery vscode-devcontainer devcontainer
1个回答
0
投票

首先，我很抱歉我将其作为答案，因为我无法发表评论。

正如 Slava 评论的那样，很高兴看到你的 .devcontainer/Dockerfile

我假设运行 docker-compose 文件没有问题，直到您尝试有条件地运行与 celery 相关的容器。所以我认为了解您使用的命令以及启动应用程序的顺序会很有帮助。

而且，只要使用付费 API 的 celery 工作人员不执行任何任务（即使用付费 API 执行某些操作），我怀疑您将只为 celery 容器启动而付费，因为它会处于空闲状态状态。

希望这有帮助。

最新问题
© www.soinside.com 2019 - 2024. All rights reserved.