Add Dockerfile and .dockerignore for CrawlGPT project setup

.dockerignore

@@ -0,0 +1,173 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+secrets.toml
+.streamlit/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc

.gitignore

@@ -135,6 +135,8 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+secrets.toml
+.streamlit/
 
 # Spyder project settings
 .spyderproject

Dockerfile

@@ -0,0 +1,77 @@
+# This Dockerfile is used to build a Docker image for the CrawlGPT project using Streamlit as the front-end
+# Specifically for huggingface spaces
+
+# Use Python 3.12 as base image
+FROM python:3.12-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies including Chrome/Playwright dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    software-properties-common \
+    sudo \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user and set permissions
+RUN useradd -m -s /bin/bash appuser && \
+    echo "appuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+RUN mkdir -p /app/.crawl4ai && chown -R appuser:appuser /app/.crawl4ai
+RUN mkdir -p /app/exports && chown -R appuser:appuser /app/exports
+
+# Copy project files
+COPY pyproject.toml setup_env.py ./ 
+COPY src/ ./src/
+COPY tests/ ./tests/
+
+# Gotta tweak some things in our main core code (LLMBasedCrowler.py) Comment out the following line:
+# from dotenv import load_dotenv # line 11 It is not needed in the docker container 
+# Because it's trying to load the API credentials from .env file which we don't have in the container
+
+
+# Accept the secret token as a build argument
+ARG GROQ_API_KEY
+ARG OLLAMA_API_TOKEN
+
+# Docs: https://huggingface.co/docs/hub/en/spaces-sdks-docker#secrets-and-variables-management
+
+# Expose the secret GROQ_API_KEY and OLLAMA_API_TOKEN at build time and set them as environment variables
+RUN --mount=type=secret,id=GROQ_API_KEY,mode=0444,required=true \
+    export GROQ_API_KEY=$(cat /run/secrets/GROQ_API_KEY) && \
+    echo "GROQ_API_KEY is set."
+    
+RUN --mount=type=secret,id=OLLAMA_API_TOKEN,mode=0444,required=true \
+    export OLLAMA_API_TOKEN=$(cat /run/secrets/OLLAMA_API_TOKEN) && \
+    echo "OLLAMA_API_TOKEN is set."
+
+# Set environment variables using the build arguments
+ENV OLLAMA_API_TOKEN=${OLLAMA_API_TOKEN}
+ENV GROQ_API_KEY=${GROQ_API_KEY}
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -e .
+RUN pip install --no-cache-dir pytest pytest-mockito black isort flake8
+
+# Set environment variables
+ENV PYTHONPATH=/app
+ENV PATH="/app/src:${PATH}"
+
+# Switch to non-root user
+USER appuser
+
+# Allow appuser to install Python packages locally (user-level installations)
+ENV PATH="/home/appuser/.local/bin:${PATH}"
+RUN mkdir -p /home/appuser/.local && chown -R appuser:appuser /home/appuser
+
+# Install Playwright and dependencies
+RUN playwright install 
+RUN playwright install-deps
+
+# Expose Streamlit port
+EXPOSE 7860
+
+# Set default command to run the Streamlit app
+CMD ["python", "-m", "streamlit", "run", "src/crawlgpt/ui/chat_app.py", "--server.port=7860", "--server.address=0.0.0.0"]