From b2f4d37f194fb7fe86d9ccf0b36f8b8a94f6d75d Mon Sep 17 00:00:00 2001 From: KansaiGaijin <83641841+KansaiGaijin@users.noreply.github.com> Date: Thu, 14 May 2026 22:58:11 +1200 Subject: [PATCH] Split monolithic compose into 5 independent stacks - docker-compose.infra.yml: core infrastructure (portainer, npm, homepage, wud, etc.) - docker-compose.media.yml: media stack (arrs, jellyfin, qbittorrent, scrobbling) - docker-compose.documents.yml: paperless-ngx, onlyoffice, stirling, open-webui - docker-compose.photo-roms.yml: immich, syncthing, retrom - docker-compose.utils.yml: gitea, tandoor, speedtest, linkwarden, rustdesk, etc. Each stack has its own project name (docker-infra, docker-media, etc.) to prevent orphan warnings. Networks defined in infra.yml, referenced as external by others. Original preserved as docker-compose.full.yaml.bak. Updated .gitignore, README, AGENTS.md, and RESTORE.md to reflect new structure. --- .gitignore | 7 +- AGENTS.md | 209 +++++----- README.md | 371 +++++++++--------- RESTORE.md | 325 +++++++-------- docker-compose.documents.yml | 210 ++++++++++ ...mpose.yaml => docker-compose.full.yaml.bak | 242 ++++++------ docker-compose.infra.yml | 207 ++++++++++ docker-compose.media.yml | 275 +++++++++++++ docker-compose.photo-roms.yml | 178 +++++++++ docker-compose.utils.yml | 252 ++++++++++++ 10 files changed, 1675 insertions(+), 601 deletions(-) create mode 100644 docker-compose.documents.yml rename docker-compose.yaml => docker-compose.full.yaml.bak (85%) create mode 100644 docker-compose.infra.yml create mode 100644 docker-compose.media.yml create mode 100644 docker-compose.photo-roms.yml create mode 100644 docker-compose.utils.yml diff --git a/.gitignore b/.gitignore index d1844ae..5506502 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,12 @@ /* # Only track these files -!docker-compose.yaml +!docker-compose.infra.yml +!docker-compose.media.yml +!docker-compose.documents.yml +!docker-compose.photo-roms.yml +!docker-compose.utils.yml +!docker-compose.full.yaml.bak !.env.example !.gitignore !README.md diff --git a/AGENTS.md b/AGENTS.md index 0debe38..b06e984 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,20 +1,34 @@ # AGENTS - Docker Infrastructure Documentation ## Overview -This document provides a comprehensive overview of the Docker infrastructure managed in `/docker/`. This self-hosted ecosystem contains 30+ services organized into functional categories, all integrated through a reverse proxy and secured with proper network segmentation. +This document provides a comprehensive overview of the Docker infrastructure managed in `/docker/`. This self-hosted ecosystem contains 50+ services organized into 5 compose stacks, all integrated through a reverse proxy and secured with proper network segmentation. -## Core Infrastructure Agents +## Compose Stack Organization + +The infrastructure is split into 5 independent compose files sharing a common `.env`: + +| Stack | File | Services | +|-------|------|----------| +| **Infrastructure** | `docker-compose.infra.yml` | Core: portainer, npm, dockerproxy, homepage, wud, ntopng, newt | +| **Media** | `docker-compose.media.yml` | Media: arr stack, jellyfin, qbittorrent, slskd, metube, maloja, scrobbler | +| **Documents** | `docker-compose.documents.yml` | Paperless-ngx + AI, onlyoffice, stirling-pdf, open-webui | +| **Photo & ROM Library** | `docker-compose.photo-roms.yml` | Immich, syncthing, retrom | +| **Utilities** | `docker-compose.utils.yml` | Gitea, tandoor, speedtest-tracker, rustdesk, redbot, linkwarden, neolink, iperf3 | + +## Core Infrastructure Agents (infra.yml) ### Management & Monitoring - **Portainer** - Container management UI (port 9443) - **Homepage** - Service dashboard (port 7575) - **WUD (WhatsUpDocker)** - Docker image management utility +- **ntopng** - Network traffic monitoring (port 3939) ### Reverse Proxy & Security - **Nginx Proxy Manager** - SSL termination and reverse proxy (ports 80, 443, 81) - **Docker Socket Proxy** - Secure Docker API access (port 2375) +- **Newt** - Pangolin tunnel client -## Media & Entertainment Stack +## Media & Entertainment Stack (media.yml) ### Media Server - **Jellyfin** - Media server with hardware transcoding (port 8096) @@ -25,6 +39,7 @@ This document provides a comprehensive overview of the Docker infrastructure man - **qBittorrent** - Torrent client (ports 56881, 7070) - **Prowlarr** - Indexer manager (port 9696) - **FlareSolverr** - Cloudflare bypass service (port 8191) +- **Metube** - YouTube video downloader (port 8081) ### Content Management - **Radarr** - Movie management (port 7878) @@ -36,7 +51,12 @@ This document provides a comprehensive overview of the Docker infrastructure man ### File Sharing - **Slskd** - Soulseek file sharing client (ports 5030, 5031, 50300) -## Document & Data Management +### Music & Scrobbling +- **Maloja** - Music scrobbling service (port 42010) +- **Multi-Scrobbler** - Cross-platform scrobbling (port 9078) + - Integrates with Jellyfin and Last.fm + +## Document & AI Suite (documents.yml) ### Document Processing - **Paperless-ngx** - Document management (port 8100) @@ -45,9 +65,14 @@ This document provides a comprehensive overview of the Docker infrastructure man - **Stirling-PDF** - PDF manipulation tools (port 8090) - **OnlyOffice** - Document collaboration (port 8091) +### AI Interface +- **Open WebUI** - LLM interface (port 3000) + +## Photo Management & Library (photo-roms.yml) + ### Photo Management - **Immich** - AI-powered photo management (port 2283) - - PostgreSQL with vector search, Redis cache + - PostgreSQL with vector search, Valkey/Redis cache - Local SSD storage for config and thumbs ### File Synchronization @@ -55,58 +80,6 @@ This document provides a comprehensive overview of the Docker infrastructure man - Obsidian vault synchronization - Multiple shared folders -## AI & Development Services - -### AI Infrastructure -- **Open WebUI** - LLM interface (port 3000) -- **LiteLLM** - AI gateway (port 4000) - - PostgreSQL database for configuration - - Supports multiple AI providers - -### Development Tools -- **Gitea** - Git service (ports 222, 8418) - - MySQL database -- **Newt** - AI service integration - -## Genealogy & Personal Tools - -### Family History -- **GrampsWeb Jamie** - Genealogy UI (port 5511) -- **GrampsWeb Helen** - Genealogy UI (port 5512) - - Shared Redis broker - - Separate trees for different family branches - -### Finance & Tracking -- **Speedtest Tracker** - Network monitoring (port 8180) - - MariaDB database, automated testing - -### Personal Tools -- **Tandoor Recipes** - Self-hosted recipe management -- **Surmai** - Personal flight tracking tool - -### Music & Scrobbling -- **Maloja** - Music scrobbling service (port 42010) -- **Multi-Scrobbler** - Cross-platform scrobbling (port 9078) - - Integrates with Jellyfin and Last.fm - -## Utilities & External Services - -### Remote Access -- **RustDesk** - Remote desktop (host mode) - -### Network Tools -- **iperf3-server** - Network performance testing (port 5201) -- **ntopng** - Network traffic monitoring tool (port 3939) - -### Browser Workspace -- **Kasm** - Browser isolation workspace (containerized browsing) - -### Game Servers -- **Foundry Watcher** - Foundry VTT player monitoring service - - SSH log tailing from Foundry server - - REST API for player status (port 30001) - - MQTT integration for connection events - ### ROM Management - **Retrom** - ROM library management service - **retrom**: Main ROM service container (port 5111) @@ -114,29 +87,50 @@ This document provides a comprehensive overview of the Docker infrastructure man - **retrom-adminer**: Adminer interface for database management (port 8080) - **retrom-jaeger**: Distributed tracing for performance monitoring +## Utilities Stack (utils.yml) + +### Development Tools +- **Gitea** - Git service (ports 222, 8418) + - MySQL database + +### Personal Tools +- **Tandoor Recipes** - Self-hosted recipe management (port 8450) + - PostgreSQL database + +### Finance & Tracking +- **Speedtest Tracker** - Network monitoring (port 8180) + - MariaDB database, automated testing + +### Bookmark Management +- **Linkwarden** - Collaborative bookmark manager (port 3400) + - PostgreSQL database + - Meilisearch for full-text search + +### Remote Access +- **RustDesk** - Remote desktop (host mode) + ### Home Automation - **Neolink** - Reolink camera bridge for Frigate/Home Assistant integration -### Discord Bot (OpenCode) -- **Service**: Discord Agent Bot -- **Role**: AI-powered Discord bot using Ollama LLM -- **Container**: `discord-agent` -- **Network**: `internal_net`, `db_net` -- **Configuration**: `/docker/discord-agent/config/agent-config.yaml` -- **Data**: `/docker/discord-agent/data/` -- **LLM**: Ollama (ministral-3:8b) at `http://192.168.0.31:11434` -- **Features**: Discord commands, service integrations, AI chat capabilities -- **Database**: MySQL for conversation persistence -- **Cogs**: Modular architecture with base_cog and integration_cog +### Discord Bots +- **RedBot** - Discord bot with custom cogs (internal_net) -### RedBot -- **Service**: RedBot Discord Bot -- **Role**: Alternative Discord bot with custom cogs -- **Container**: `redbot` -- **Network**: `web_net` +### Network Tools +- **iperf3-server** - Network performance testing (port 5201) -### Other Tools -- **Newt** - AI service integration +## Independent Stacks (separate compose files) + +These services are deployed independently and are not part of the main 5-stack split: + +| Service | Directory | Role | +|---------|-----------|------| +| **Discord Agent Bot** | `discord-agent/` | AI-powered Discord bot using Ollama LLM | +| **Foundry Watcher** | `foundry-watcher/` | Foundry VTT player monitoring | +| **GrampsWeb Jamie** | `gramps-jamie/` | Genealogy UI (port 5511) | +| **GrampsWeb Helen** | `gramps-helen/` | Genealogy UI (port 5512) | +| **Kasm** | `kasm/` | Browser isolation workspace | +| **LiteLLM** | `litellm/` | AI gateway (port 4000) | +| **MBI Poller** | `mbi-poller/` | MBI data polling service | ## Network Architecture @@ -146,17 +140,18 @@ This document provides a comprehensive overview of the Docker infrastructure man - **web_net** - Web-accessible services - **internal_net** - Internal service communication +Networks are defined in `docker-compose.infra.yml` and referenced as `external: true` by all other stacks. + ### Storage Structure ``` /docker/ ├── Arrs/ (Media stack: Prowlarr, Radarr, Sonarr, Lidarr, Bazarr, Jellyfin, Jellyseerr) ├── immich/ (Photo management) ├── paperless/ (Document management) -├── litellm/ (AI gateway) -├── discord-agent/ (Discord bot with Ollama LLM integration) +├── discord-agent/ (Independent — Discord bot with Ollama) ├── gitea/ (Git service) -├── gramps-jamie/ (Genealogy) -├── gramps-helen/ (Genealogy) +├── gramps-jamie/ (Independent — Genealogy) +├── gramps-helen/ (Independent — Genealogy) ├── npm/ (Nginx Proxy Manager) ├── qBittorrent/ (Download client) ├── slskd/ (Soulseek client) @@ -168,19 +163,43 @@ This document provides a comprehensive overview of the Docker infrastructure man ├── retrom/ (ROM library management) ├── wud/ (Docker image management) ├── ntopng/ (Network traffic monitoring) -├── kasm/ (Browser workspace) +├── kasm/ (Independent — Browser workspace) +├── litellm/ (Independent — AI gateway) ├── neolink/ (Reolink camera bridge) +├── linkwarden/ (Bookmark manager) ├── surmai/ (Flight tracking) ├── tandoor/ (Recipe management) -├── foundry-watcher/ (Foundry VTT monitoring) +├── foundry-watcher/ (Independent — Foundry VTT) +├── rustdesk/ (Remote desktop) +├── redbot/ (Discord bot) +├── stirling/ (PDF tools) └── Various other service configs ``` +## Deployment + +### Deploy all main stacks +```bash +for f in docker-compose.infra.yml docker-compose.media.yml docker-compose.documents.yml docker-compose.photo-roms.yml docker-compose.utils.yml; do + docker compose -f "$f" up -d +done +``` + +### Deploy a specific stack +```bash +docker compose -f docker-compose.media.yml up -d +``` + +### Stop a specific stack +```bash +docker compose -f docker-compose.media.yml down +``` + ## Key Features & Configuration ### Security - Container security with `no-new-privileges:true` -- Network segmentation (databases internal-only) +- Network segmentation (databases internal-only via `db_net`) - Reverse proxy with SSL termination ### Performance @@ -189,7 +208,7 @@ This document provides a comprehensive overview of the Docker infrastructure man - Resource limits (Portainer: 512MB RAM) ### Data Management -- Comprehensive backup script included +- Comprehensive backup script included (`backup.sh`) - Separate volumes for critical data - NAS storage integration for media files @@ -197,25 +216,30 @@ This document provides a comprehensive overview of the Docker infrastructure man - System configured for Pacific/Auckland timezone - PUID/PGID for proper file permissions - Extensive environment variable configuration +- Shared `.env` file across all stacks ## Service Dependencies ### Database Services -- MariaDB instances for media stack, tracking, and document services -- PostgreSQL for AI services, photo management, and finance -- Redis for caching and message brokering +- MariaDB: npm, paperless, speedtest-tracker +- PostgreSQL: immich, retrom, tandoor, linkwarden +- MySQL: gitea +- Redis/Valkey: paperless-broker, immich-redis +- Meilisearch: linkwarden ### Network Dependencies +- `infra.yml` must be deployed first (creates shared networks) - All web services route through Nginx Proxy Manager -- Internal services communicate via internal_net -- Media services isolated on media_net -- Database services on internal-only db_net +- Internal services communicate via `internal_net` +- Media services isolated on `media_net` +- Database services on internal-only `db_net` -## Maintenance Agents +## Maintenance ### Automated Tasks -- **Backup Script** - Regular data backups +- **Backup Script** (`backup.sh`) - Regular data backups - **Speedtest Tracker** - Automated network testing +- **WUD** - Automatic Docker image update monitoring ### Manual Tasks - Service monitoring via Portainer @@ -232,5 +256,6 @@ This document provides a comprehensive overview of the Docker infrastructure man ### Troubleshooting - Container logs accessible via Portainer - Network diagnostics via iperf3-server +- Per-stack logs: `docker compose -f docker-compose..yml logs -f ` -This Docker infrastructure represents a comprehensive self-hosted ecosystem covering media management, document processing, AI services, development tools, and personal productivity applications, all integrated through a reverse proxy and organized with proper network segmentation. \ No newline at end of file +This Docker infrastructure represents a comprehensive self-hosted ecosystem covering media management, document processing, AI services, development tools, and personal productivity applications, all integrated through a reverse proxy and organized into independent compose stacks with proper network segmentation. diff --git a/README.md b/README.md index 51fa8a4..7dc1d15 100644 --- a/README.md +++ b/README.md @@ -2,101 +2,62 @@ ## Overview -This infrastructure manages a comprehensive self-hosted environment including media management (Jellyfin, *arr stack), document management (Paperless-ngx), AI services (Open WebUI, LiteLLM), photo management (Immich), genealogy tools (GrampsWeb), and various utilities. +This infrastructure manages a comprehensive self-hosted environment including media management (Jellyfin, *arr stack), document management (Paperless-ngx), AI services (Open WebUI), photo management (Immich), and various utilities. + +The deployment is split into **5 independent compose stacks** sharing a common `.env` file and Docker networks. + +## Compose Stacks + +| Stack | File | Services | +|-------|------|----------| +| **Infrastructure** | `docker-compose.infra.yml` | portainer, ntopng, wud, homepage, dockerproxy, newt, npm, npm-db | +| **Media** | `docker-compose.media.yml` | prowlarr, flaresolverr, qbittorrent, radarr, sonarr, lidarr, bazarr, seerr, jellyfin, slskd, metube, maloja, multi-scrobbler | +| **Documents** | `docker-compose.documents.yml` | onlyoffice, paperless-ngx (db, broker, tika, gotenberg, webserver, ai), stirling-pdf, open-webui | +| **Photo & ROM Library** | `docker-compose.photo-roms.yml` | immich (server, postgres, ml, redis), syncthing, retrom (db, adminer, service, jaeger) | +| **Utilities** | `docker-compose.utils.yml` | gitea (server + db), tandoor (db + web), speedtest-tracker (app + db), rustdesk (hbbs + hbbr), redbot, iperf3-server, neolink, linkwarden (app + db + meilisearch) | ## Architecture ### Internal Docker Services -| Service Categories | Service | Role / Component | Web UI Port | -|---|---|---|---| -| Media & Arr Stack | Jellyfin | Media server with hardware transcoding | 8096 | -| Media & Arr Stack | Jellyseerr | Media request management | 5055 | -| Media & Arr Stack | Radarr | Movie management | 7878 | -| Media & Arr Stack | Sonarr | TV show management | 8989 | -| Media & Arr Stack | Lidarr | Music management | 8686 | -| Media & Arr Stack | Bazarr | Subtitle management | 6767 | -| Media & Arr Stack | Prowlarr | Indexer manager | 9696 | -| Media & Arr Stack | FlareSolverr | Cloudflare bypass service | 8191 | -| Media & Arr Stack | qBittorrent | Download client | 7070 | -| Data Management | Paperless | Document management interface | 8100 | -| Data Management | Paperless-AI | AI-powered document processing | - | -| Data Management | Stirling-PDF | PDF manipulation tools | 8090 | -| Data Management | OnlyOffice | Document collaboration server | 8091 | -| Local AI | Open WebUI | LLM interface | 3000 | -| Local AI | LiteLLM | AI gateway | 4000 | -| Data Management | Immich | Photo management | 2283 | -| Data Management | Syncthing | File synchronization | 8384 | -| Management & Infrastructure | Gitea | Git service | 8418 | -| Data Management | GrampsWeb Jamie | Genealogy UI | 5511 | -| Data Management | GrampsWeb Helen | Genealogy UI | 5512 | -| Data Management | Speedtest Tracker | Network performance | 8180 | -| Management & Infrastructure | NPM Admin | NPM admin UI | 81 | -| Data Management | Maloja | Music scrobbling | 42010 | -| Data Management | Multi-Scrobbler | Cross-platform scrobbler | 9078 | -| Management & Infrastructure | Portainer | Container management UI | 9443 | -| Management & Infrastructure | Homepage | Service dashboard | 7575 | -| Management & Infrastructure | WUD | Docker image management | 3000 | -| Management & Infrastructure | ntopng | Network traffic monitoring | 3939 | -| Management & Infrastructure | Retrom | ROM library management | 5111 | -| Management & Infrastructure | iperf3-server | Network performance testing | 5201 | -| Management & Infrastructure | Foundry Watcher | Foundry VTT player monitoring | 30001 | -| Discord Bots | Discord Agent Bot | AI-powered Discord bot with Ollama | - | -| Discord Bots | RedBot | Alternative Discord bot | - | - -### Discord Agent Integration - -### Discord Bot Service -- **Service**: Discord Agent Bot -- **Role**: AI-powered Discord bot using Ollama LLM -- **Container**: `discord-agent` -- **Network**: `internal_net`, `db_net` -- **Web UI**: Discord (no direct web interface) -- **Configuration**: `/docker/discord-agent/config/agent-config.yaml` -- **Data**: `/docker/discord-agent/data/` - -### Features -- **AI Integration**: Uses Ollama LLM (ministral-3:8b) at `http://192.168.0.31:11434` -- **Modular Architecture**: Cogs for service integrations -- **Database Support**: MySQL for conversation persistence -- **Caching**: Redis for improved performance (configurable) -- **Security**: Discord role-based permissions -- **Logging**: Configurable logging levels (default: INFO) -- **Rate Limiting**: Built-in rate limiting for API calls - -### Available Commands -- `!agent` - Chat with the AI assistant -- `!status` - Check bot status and latency -- `!help` - Show available commands -- Service integration commands (configurable) - -### Configuration -- **Main Config**: `config/agent-config.yaml` (YAML-based with env var substitution) -- **Discord Token**: Sourced from `DISCORD_BOT_TOKEN` environment variable -- **Ollama Config**: Model, endpoint, temperature, top_p, top_k parameters -- **Integrations**: Configurable service integrations in YAML - -### Environment Variables -``` -DISCORD_BOT_TOKEN=your_discord_bot_token_here -OLLAMA_ENDPOINT=http://192.168.0.31:11434 -OLLAMA_MODEL=ministral-3:8b -TZ=Pacific/Auckland -``` - -### Bot Architecture -- **Main Bot**: `discord_agent.py` - Core Discord bot implementation -- **Base Cog**: `cogs/base_cog.py` - Base functionality and commands -- **Integration Cog**: `cogs/integration_cog.py` - Service integrations -- **Config**: YAML-based configuration with environment variable substitution -- **Database**: MySQL with asyncpg driver for conversation history - -### Security -- Runs on internal networks only -- Discord role-based access control -- Configurable rate limiting to prevent abuse -- No direct database access from external networks - +| Stack | Service | Role | Port | +|-------|---------|------|------| +| Infrastructure | Nginx Proxy Manager | SSL termination & reverse proxy | 80, 443, 81 | +| Infrastructure | Portainer | Container management UI | 9443 | +| Infrastructure | Homepage | Service dashboard | 7575 | +| Infrastructure | WUD | Docker image update monitoring | 3666 | +| Infrastructure | ntopng | Network traffic monitoring | 3939 | +| Infrastructure | Docker Socket Proxy | Secure Docker API access | 2375 | +| Infrastructure | Newt | Pangolin tunnel client | - | +| Media | Jellyfin | Media server (HW transcoding) | 8096 | +| Media | Jellyseerr (seerr) | Media request management | 5055 | +| Media | Radarr | Movie management | 7878 | +| Media | Sonarr | TV show management | 8989 | +| Media | Lidarr | Music management | 8686 | +| Media | Bazarr | Subtitle management | 6767 | +| Media | Prowlarr | Indexer manager | 9696 | +| Media | FlareSolverr | Cloudflare bypass | 8191 | +| Media | qBittorrent | Download client | 7070 | +| Media | Slskd | Soulseek file sharing | 5030, 5031 | +| Media | Metube | YouTube downloader | 8081 | +| Media | Maloja | Music scrobbling | 42010 | +| Media | Multi-Scrobbler | Cross-platform scrobbler | 9078 | +| Documents | Paperless-ngx | Document management | 8100 | +| Documents | Paperless-AI | AI document processing | 3040 | +| Documents | Stirling-PDF | PDF manipulation | 8090 | +| Documents | OnlyOffice | Document collaboration | 8091 | +| Documents | Open WebUI | LLM interface | 3000 | +| Photo & ROM | Immich | Photo management | 2283 | +| Photo & ROM | Syncthing | File synchronization | 8384 | +| Photo & ROM | Retrom | ROM library management | 5111 | +| Utilities | Gitea | Git service | 8418 | +| Utilities | Tandoor Recipes | Recipe management | 8450 | +| Utilities | Speedtest Tracker | Network monitoring | 8180 | +| Utilities | RustDesk | Remote desktop (host mode) | - | +| Utilities | RedBot | Discord bot | - | +| Utilities | iperf3-server | Network performance test | 5201 | +| Utilities | Neolink | Reolink camera bridge | 8554 | +| Utilities | Linkwarden | Bookmark manager | 3400 | ### Network Segmentation @@ -105,47 +66,95 @@ TZ=Pacific/Auckland - **web_net**: Web-accessible services - **internal_net**: Internal service communication +Networks are created by `infra.yml` and referenced as external by all other stacks. + ## Storage Structure ### Local Docker Configs (`/docker/`) ``` /docker/ -├── Homepage/ ├── Arrs/ (Prowlarr, Radarr, Sonarr, Lidarr, Bazarr, Jellyfin, Jellyseerr) -├── qBittorrent/ -├── discord-agent/ (Discord bot with Ollama LLM) -├── paperless/ -├── stirling/ +├── Homepage/ ├── immich/ -├── syncthing/ +├── paperless/ +├── discord-agent/ (Separate stack) ├── gitea/ -├── gramps-jamie/ -├── gramps-helen/ -├── speedtest-tracker/ -├── rustdesk/ -├── redbot/ +├── gramps-jamie/ (Separate stack) +├── gramps-helen/ (Separate stack) +├── kasm/ (Separate stack) +├── foundry-watcher/ (Separate stack) +├── litellm/ (Separate stack) +├── linkwarden/ ├── maloja/ -├── scrobble/ -├── litellm/ -├── retrom/ -├── wud/ +├── neolink/ +├── npm/ ├── ntopng/ +├── qBittorrent/ +├── redbot/ +├── retrom/ +├── rustdesk/ +├── scrobble/ (Multi-scrobbler) ├── slskd/ -└── npm/ +├── speedtest-tracker/ +├── stirling/ +├── surmai/ +├── syncthing/ +├── tandoor/ +├── wud/ +└── .env (shared environment variables) ``` -### NAS Storage (`/mnt/Nas-Storage/data/`) +### NAS Storage (`/mnt/nas-storage/data/`) ``` -/mnt/Nas-Storage/data/ +/mnt/nas-storage/data/ ├── media/ │ ├── movies/ │ ├── tv/ │ ├── music/ -│ └── images/ +│ └── romms/ └── torrents/ + ├── metube/ └── soulsync/ ``` +## Deployment + +### First-time deploy (or after network cleanup) + +```bash +# Deploy infrastructure first (creates shared networks) +docker compose -f docker-compose.infra.yml up -d + +# Then deploy remaining stacks in any order +docker compose -f docker-compose.media.yml up -d +docker compose -f docker-compose.documents.yml up -d +docker compose -f docker-compose.photo-roms.yml up -d +docker compose -f docker-compose.utils.yml up -d +``` + +### Deploy all stacks + +```bash +for f in docker-compose.*.yml; do + if [ "$f" != "docker-compose.full.yaml.bak" ]; then + docker compose -f "$f" up -d + fi +done +``` + +### Stop/start individual stacks + +```bash +docker compose -f docker-compose.media.yml down +docker compose -f docker-compose.media.yml up -d +``` + +### View logs + +```bash +docker compose -f docker-compose.media.yml logs -f jellyfin +``` + ## Prerequisites ### Required Environment Variables (.env file) @@ -159,7 +168,6 @@ TZ=Pacific/Auckland # URLs JELLYFIN_URL=https://your-jellyfin-domain.com PAPERLESS_URL=https://your-paperless-domain.com -WYGIWYH_URL=https://your-wygiwyh-domain.com HOMEPAGE_ALLOWED_HOSTS=your-homepage-domain.com # Nginx Proxy Manager @@ -175,10 +183,6 @@ PAPERLESS_SECRET_KEY=your_secret_key # Immich IMMICH_POSTGRES_PASSWORD=your_secure_password -# LiteLLM -GROQ_API_KEY=your_groq_api_key -LITELLM_MASTER_KEY=your_master_key - # Slskd SLSKD_USERNAME=your_username SLSKD_PASSWORD=your_secure_password @@ -189,15 +193,35 @@ SPEEDTEST_DB_USER=speedtest SPEEDTEST_DB_PASSWORD=your_secure_password SPEEDTEST_APP_KEY=base64:your_generated_key -# WYGIWYH -WYGIWYH_DB_DATABASE=wygiwyh -WYGIWYH_DB_USER=wygiwyh -WYGIWYH_DB_PASSWORD=your_secure_password -WYGIWYH_SECRET_KEY=your_secret_key -WYGIWYH_ALLOWED_HOSTS=localhost,127.0.0.1,your-domain.com - # RedBot REDBOT_TOKEN=your_discord_bot_token + +# Tandoor +TANDOOR_SECRET_KEY=your_secret_key +TANDOOR_POSTGRES_DB=tandoor +TANDOOR_POSTGRES_USER=tandoor +TANDOOR_POSTGRES_PASSWORD=your_secure_password + +# Linkwarden +LINKWARDEN_DB_PASSWORD=your_secure_password +NEXTAUTH_SECRET=your_secret +NEXTAUTH_URL=https://your-linkwarden-domain.com + +# WUD +WUD_REGISTRY_HUB_0_LOGIN=your_dockerhub_username +WUD_REGISTRY_HUB_0_TOKEN=your_dockerhub_token + +# OnlyOffice +OO_JWT_SECRET=your_secret + +# Maloja +MALOJA_FORCE_PASSWORD=your_password +MALOJA_URL=http://maloja:42010 +MALOJA_API_KEY=your_key + +# Jellyfin +JELLYFIN_APIKEY_MS=your_api_key_for_multiscrobbler +JELLYFIN_USER=your_username ``` ### System Requirements @@ -209,27 +233,15 @@ REDBOT_TOKEN=your_discord_bot_token ## Installation -1. Clone/Download this repository -2. Create required directories +1. Clone this repository +2. Create `.env` from `.env.example` and fill in values +3. Run infrastructure: ```bash - # See directory_structure.sh for automated setup - chmod +x directory_structure.sh - ./directory_structure.sh + docker compose -f docker-compose.infra.yml up -d ``` -3. Create .env file +4. Deploy desired stacks: ```bash - cp .env.example .env - # Edit .env with your values - nano .env - ``` -4. Set correct permissions - ```bash - sudo chown -R $PUID:$PGID /docker - sudo chown -R $PUID:$PGID /mnt/Nas-Storage/data - ``` -5. Start services - ```bash - docker compose up -d + docker compose -f docker-compose.media.yml up -d ``` ## Maintenance @@ -237,34 +249,26 @@ REDBOT_TOKEN=your_discord_bot_token ### Backup Strategy **Critical Data to Backup:** -- `/docker/` - All service configurations -- Database volumes (see docker-compose.yaml) -- `.env` file (store securely, contains secrets) +- `/docker/` — all service configurations +- Named volumes: `portainer_data`, `docker_dbdata`, `open-webui` +- `.env` file (contains secrets) **Optional (can be regenerated):** - Media files on NAS ### Updates -To manually update containers: - -``` -docker compose pull - -docker compose up -d -``` - -### Logs - -View logs for any service: -``` -docker compose logs -f [service_name] +```bash +for f in docker-compose.infra.yml docker-compose.media.yml docker-compose.documents.yml docker-compose.photo-roms.yml docker-compose.utils.yml; do + docker compose -f "$f" pull + docker compose -f "$f" up -d +done ``` ### Resource Limits - Portainer: 512MB RAM limit -- Log rotation: 10MB max, 3 files +- Log rotation: 10MB max, 3 files (on services with logging config) ## Hardware Acceleration @@ -273,71 +277,68 @@ Jellyfin is configured for Intel GPU transcoding: - Group: `104` (render group) Verify GPU access: -``` +```bash ls -l /dev/dri/renderD128 ``` ## Security Considerations 1. Secrets Management: Store `.env` securely, never commit to version control -2. Network Segmentation: Database network is internal-only +2. Network Segmentation: Database network is internal-only (`db_net`) 3. Container Security: `no-new-privileges:true` on supported services 4. Reverse Proxy: Use NPM for SSL termination and authentication -5. Updates: Regularly update containers with `docker compose pull && docker compose up -d` +5. Updates: Regularly update containers as shown above ## Troubleshooting ### Service won't start -``` -docker compose logs [service_name] -docker compose restart [service_name] +```bash +docker compose -f docker-compose.media.yml logs service_name +docker compose -f docker-compose.media.yml restart service_name ``` ### Database connection issues -``` -# Check database is healthy -docker compose ps -# Verify network connectivity -docker compose exec [service] ping [db_service] +```bash +docker compose -f docker-compose.documents.yml ps +docker compose -f docker-compose.documents.yml exec paperless-webserver ping paperless-db ``` ### Permission errors -``` -# Verify ownership -ls -la /docker/[service]/ -# Fix if needed -sudo chown -R $PUID:$PGID /docker/[service]/ +```bash +ls -la /docker/service_name/ +sudo chown -R $PUID:$PGID /docker/service_name/ ``` ### Storage full -``` -# Check Docker disk usage +```bash docker system df -# Clean up unused resources docker system prune -a ``` ## Contributing Services To add new services: -1. Add service definition to appropriate section +1. Add service definition to the appropriate compose file 2. Assign to correct network(s) 3. Add volume mounts under `/docker/[service]/` 4. Update this README with service description 5. Add required environment variables to `.env.example` +## Independent Stacks + +The following services have their own compose files (not part of the main split): +- **discord-agent/** — Discord Agent Bot +- **foundry-watcher/** — Foundry VTT monitoring +- **gramps-jamie/** — Genealogy (Jamie) +- **gramps-helen/** — Genealogy (Helen) +- **kasm/** — Browser workspace +- **litellm/** — AI gateway +- **mbi-poller/** — MBI poller service + ## License This is a personal infrastructure setup. Adapt as needed for your use case. -## Support - -For issues with specific services, consult their official documentation: -- Jellyfin: https://jellyfin.org/docs/ -- Paperless-ngx: https://docs.paperless-ngx.com/ -- Immich: https://immich.app/docs/ -- And so on... - --- -**Last Updated**: January 2026 +**Last Updated**: May 2026 diff --git a/RESTORE.md b/RESTORE.md index 515d6a2..588018f 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -4,274 +4,187 @@ This guide walks you through restoring your Docker infrastructure from a backup. ## Architecture Alignment -The architecture inventory is documented in README.md as two tables under Architecture: -- Internal Docker Services (Service | Category | Web UI Port) -- External Non-Docker Services (Service | Category | IP | Web UI Port) +The infrastructure is split into 5 compose files (stacks) sharing a common `.env`: -If you are restoring, follow the standard restoration steps below, and refer to README.md for the exact service inventory and ports as implemented in your environment. +| Stack | File | Purpose | +|-------|------|---------| +| Infrastructure | `docker-compose.infra.yml` | Portainer, NPM, homepage, WUD, ntopng, newt, dockerproxy | +| Media | `docker-compose.media.yml` | arr stack, Jellyfin, qBittorrent, slskd, metube, scrobbling | +| Documents | `docker-compose.documents.yml` | Paperless-ngx, AI, OnlyOffice, Stirling-PDF, Open WebUI | +| Photo & ROM | `docker-compose.photo-roms.yml` | Immich, Syncthing, Retrom | +| Utilities | `docker-compose.utils.yml` | Gitea, Tandoor, Speedtest, RustDesk, RedBot, Linkwarden, Neolink, iperf3 | + +Refer to README.md for the exact service inventory and ports. ## Prerequisites -- Fresh system with Docker and Docker Compose installed -- Backup archive (`docker-backup-YYYYMMDD_HHMMSS.tar.gz`) +- Fresh system with Docker and Docker Compose V2 installed +- Backup archive containing `/docker/` configs - Access to NAS storage (if applicable) - Root or sudo access -## Discord Agent Service - -### Step 1: Create Discord Agent Directory -```bash -mkdir -p /docker/discord-agent/config /docker/discord-agent/data/logs /docker/discord-agent/data/database /docker/discord-agent/data/cache /docker/discord-agent/cogs /docker/discord-agent/scripts -``` - -### Step 2: Copy Configuration Files -```bash -# Copy agent configuration -cp agent-config.yaml /docker/discord-agent/config/ -cp permissions.json /docker/discord-agent/config/ - -# Copy scripts -cp startup.sh /docker/discord-agent/scripts/ -cp health_check.sh /docker/discord-agent/scripts/ -``` - -### Step 3: Copy Python Files -```bash -cp discord_agent.py /docker/discord-agent/ -cp base_cog.py /docker/discord-agent/cogs/ -cp integration_cog.py /docker/discord-agent/cogs/ -``` - -### Step 4: Copy Requirements -```bash -cp requirements.txt /docker/discord-agent/ -cp Dockerfile /docker/discord-agent/ -``` - -### Step 5: Update Environment Variables -Add these to your `.env` file: -```bash -# Discord Agent -DISCORD_BOT_TOKEN=your_discord_bot_token_here -OLLAMA_ENDPOINT=http://192.168.0.31:11434 -OLLAMA_MODEL=ministral-3:8b -TZ=Pacific/Auckland -``` - -### Step 6: Update Configuration -Edit `/docker/discord-agent/config/agent-config.yaml` with your specific settings: -```yaml -discord: - token: ${DISCORD_BOT_TOKEN} - prefix: "!" - status: "AI Assistant | !help" - -ollama: - endpoint: "${OLLAMA_ENDPOINT:http://192.168.0.31:11434}" - model: "${OLLAMA_MODEL:ministral-3:8b}" - parameters: - temperature: 0.7 - top_p: 0.9 - top_k: 40 - timeout: 60 -``` - -### Step 7: Build and Start Service -```bash -# Build the Discord agent image -docker compose build discord-agent - -# Start the service -docker compose up -d discord-agent - -# Verify the service is running -docker compose logs discord-agent -``` - -### Step 8: Verify Integration -```bash -# Check Discord bot connection -docker exec discord-agent python3 -c "import discord; print('Discord library available')" - -# Check Ollama connection -curl http://192.168.0.31:11434/api/tags - -# Check MySQL database -docker compose exec agent-db mysql -u agent -pagent -e "SHOW DATABASES;" -``` - -### Step 9: Test Bot Functionality -- The bot should appear in your Discord server -- Test with `!help` command to verify functionality -- Test with `!agent ` to verify Ollama integration -- Test with `!status` to check bot status - -### Troubleshooting -- If bot doesn't start, check logs: `docker compose logs discord-agent` -- Verify DISCORD_BOT_TOKEN is set correctly in `.env` file -- Ensure Ollama is running and accessible at the configured endpoint -- Verify internal_net and db_net networks are available -- Check that MySQL database agent-db is running and healthy - -## Step 2: Restore Directory Structure +## Step 1: Restore Directory Structure ```bash -# Copy the directory structure script -cp directory_structure.sh /opt/docker-compose/ -cd /opt/docker-compose/ +# Create the root docker directory +mkdir -p /docker +cd /docker -# Make it executable and run -chmod +x directory_structure.sh -./directory_structure.sh -``` - -## Step 3: Restore Docker Compose Configuration - -```bash -# Copy main configuration files -cp /path/to/backup/docker-compose.yaml . +# Copy configuration files from backup cp /path/to/backup/.env . cp /path/to/backup/.gitignore . cp /path/to/backup/README.md . +cp /path/to/backup/AGENTS.md . +cp /path/to/backup/RESTORE.md . +cp /path/to/backup/backup.sh . + +# Copy all compose files +cp /path/to/backup/docker-compose.infra.yml . +cp /path/to/backup/docker-compose.media.yml . +cp /path/to/backup/docker-compose.documents.yml . +cp /path/to/backup/docker-compose.photo-roms.yml . +cp /path/to/backup/docker-compose.utils.yml . + +# Copy the original full backup (for reference) +cp /path/to/backup/docker-compose.full.yaml.bak . # IMPORTANT: Edit .env file with new system-specific values nano .env - -# Verify the configuration -docker compose config ``` -## Step 4: Restore Service Configurations +## Step 2: Restore Service Configuration Directories ```bash # Copy service configurations back to /docker rsync -av /path/to/backup/docker/ /docker/ +# Create required subdirectories that may be missing +mkdir -p /docker/Arrs/{Prowlarr,Radarr,Sonarr,Lidarr,Bazarr,Jellyfin,Seerr}/config +mkdir -p /docker/{immich,qBittorrent,paperless,stirling,syncthing,gitea} +mkdir -p /docker/{speedtest-tracker,rustdesk,redbot,maloja,scrobble} +mkdir -p /docker/{retrom/config,retrom/data} +mkdir -p /docker/{wud,ntopng,slskd,npm/data,npm/letsencrypt,npm/mysql} +mkdir -p /docker/{Homepage/config,neolink,tandoor,linkwarden} + # Set correct permissions PUID=$(id -u) PGID=$(id -g) sudo chown -R $PUID:$PGID /docker ``` -## Step 5: Restore Docker Volumes +## Step 3: Restore Docker Volumes -For each volume backup in `/path/to/backup/volumes/`: +For each volume backup: ```bash # Create the volume if it doesn't exist -docker volume create VOLUME_NAME - -# Restore the volume data -docker run --rm \ - -v VOLUME_NAME:/volume \ - -v /path/to/backup/volumes:/backup \ - alpine \ - tar xzf /backup/VOLUME_NAME.tar.gz -C /volume -``` - -Example for specific volumes: - -```bash -# Restore Portainer data docker volume create portainer_data -docker run --rm \ - -v portainer_data:/volume \ - -v /path/to/backup/volumes:/backup \ - alpine \ - tar xzf /backup/portainer_data.tar.gz -C /volume - -# Restore Open WebUI data docker volume create open-webui -docker run --rm \ - -v open-webui:/volume \ - -v /path/to/backup/volumes:/backup \ - alpine \ - tar xzf /backup/open-webui.tar.gz -C /volume +docker volume create docker_dbdata +docker volume create docker_aidata +docker volume create docker_onlyoffice +docker volume create docker_redisdata -# Repeat for other volumes... +# Restore volume data +for volume in portainer_data open-webui docker_dbdata docker_aidata docker_onlyoffice docker_redisdata; do + if [ -f "/path/to/backup/volumes/${volume}.tar.gz" ]; then + docker run --rm \ + -v ${volume}:/volume \ + -v /path/to/backup/volumes:/backup \ + alpine \ + tar xzf /backup/${volume}.tar.gz -C /volume + fi +done ``` -## Step 6: Start Database Services First +## Step 4: Start Infrastructure First ```bash -# Start only database services -docker compose up -d paperless-db immich-postgres litellm-postgres gitea-db speedtest-db npm-db +# Start infrastructure (creates shared networks: media_net, db_net, web_net, internal_net) +docker compose -f docker-compose.infra.yml up -d + +# Verify networks were created +docker network ls | grep -E "media_net|db_net|web_net|internal_net" +``` + +## Step 5: Start Database Services + +```bash +# Start database-dependent stacks +docker compose -f docker-compose.documents.yml up -d paperless-db paperless-broker +docker compose -f docker-compose.photo-roms.yml up -d immich-postgres immich-redis retrom-db +docker compose -f docker-compose.utils.yml up -d gitea-db speedtest-db tandoor_db linkwarden-db meilisearch # Wait for databases to be healthy -docker compose ps - -# Check logs for any errors -docker compose logs -f paperless-db -# Press Ctrl+C to exit logs +docker compose -f docker-compose.documents.yml ps +docker compose -f docker-compose.utils.yml ps ``` -## Step 7: Restore Database Dumps +## Step 6: Restore Database Dumps (if available) ### Paperless MariaDB -``` -# Copy SQL file into container -docker cp /path/to/backup/database-dumps/paperless.sql paperless-db:/tmp/ - -# Import the database +```bash docker exec -i paperless-db mysql -u root -p"${PAPERLESS_DB_ROOT_PASSWORD}" paperless < /path/to/backup/database-dumps/paperless.sql ``` ### Immich PostgreSQL -``` +```bash docker exec -i immich_postgres psql -U postgres immich < /path/to/backup/database-dumps/immich.sql ``` -### LiteLLM PostgreSQL -``` -docker exec -i litellm-postgres psql -U litellm litellm_db < /path/to/backup/database-dumps/litellm.sql -``` - ### Gitea MySQL -``` +```bash docker exec -i gitea-db mysql -u root -pgitea gitea < /path/to/backup/database-dumps/gitea.sql ``` ### Speedtest Tracker MariaDB -``` +```bash docker exec -i speedtest-db mysql -u root -p"${SPEEDTEST_DB_PASSWORD}" speedtest < /path/to/backup/database-dumps/speedtest.sql ``` -## Step 8: Mount NAS Storage (if applicable) +### Tandoor PostgreSQL +```bash +docker exec -i tandoor_DB psql -U "${TANDOOR_POSTGRES_USER}" "${TANDOOR_POSTGRES_DB}" < /path/to/backup/database-dumps/tandoor.sql +``` + +## Step 7: Mount NAS Storage (if applicable) ```bash # Create mount point -sudo mkdir -p /mnt/Nas-Storage +sudo mkdir -p /mnt/nas-storage # Add to /etc/fstab for permanent mounting # Example for NFS: -# nas-server:/volume1/data /mnt/Nas-Storage nfs defaults 0 0 - -# Example for CIFS/SMB: -# //nas-server/data /mnt/Nas-Storage cifs credentials=/root/.smbcredentials,uid=1000,gid=1000 0 0 +# nas-server:/volume1/data /mnt/nas-storage nfs defaults 0 0 # Mount immediately sudo mount -a # Verify mount -df -h /mnt/Nas-Storage +df -h /mnt/nas-storage ``` -## Step 9: Start All Services +## Step 8: Start All Services ```bash -# Start all services -docker compose up -d +# Deploy all stacks +for f in docker-compose.infra.yml docker-compose.media.yml docker-compose.documents.yml docker-compose.photo-roms.yml docker-compose.utils.yml; do + docker compose -f "$f" up -d +done -# Watch the startup process -docker compose logs -f +# Watch the startup process for a specific stack +docker compose -f docker-compose.media.yml logs -f -# Check service health -docker compose ps +# Check service health across all stacks +for f in docker-compose.*.yml; do + if [ "$f" != "docker-compose.full.yaml.bak" ]; then + echo "=== $f ===" + docker compose -f "$f" ps + fi +done ``` -## Step 10: Verify Services - -Go through each service and verify it's working correctly: +## Step 9: Verify Services ### Check Web Interfaces - Homepage: http://your-server:7575 @@ -279,9 +192,11 @@ Go through each service and verify it's working correctly: - Jellyfin: http://your-server:8096 - Paperless: http://your-server:8100 - Immich: http://your-server:2283 +- Gitea: http://your-server:8418 +- Speedtest: http://your-server:8180 ### Verify Databases -``` +```bash # Paperless docker exec paperless-db mysql -u root -p"${PAPERLESS_DB_ROOT_PASSWORD}" -e "SELECT COUNT(*) FROM paperless.documents_document;" @@ -289,9 +204,35 @@ docker exec paperless-db mysql -u root -p"${PAPERLESS_DB_ROOT_PASSWORD}" -e "SEL docker exec immich_postgres psql -U postgres -d immich -c "SELECT COUNT(*) FROM assets;" ``` -... (rest of file continues) +## Step 10: Restore Independent Stacks (if applicable) + +These services have their own compose files and are restored separately: + +```bash +# Discord Agent Bot +cd /docker/discord-agent +docker compose up -d + +# LiteLLM +cd /docker/litellm +docker compose up -d + +# Gramps genealogy +cd /docker/gramps-jamie +docker compose up -d +cd /docker/gramps-helen +docker compose up -d + +# Other independent stacks +cd /docker/kasm && docker compose up -d +cd /docker/foundry-watcher && docker compose up -d +``` ## Architecture Alignment -- This section connects to the Architecture two-table layout described in README.md. +- This section connects to the Architecture table described in README.md. +- Each compose file corresponds to a functional domain in the architecture. +- Networks are created by `infra.yml` and shared across all other stacks. -**Last Updated**: December 2025 +--- + +**Last Updated**: May 2026 diff --git a/docker-compose.documents.yml b/docker-compose.documents.yml new file mode 100644 index 0000000..3fe8f3d --- /dev/null +++ b/docker-compose.documents.yml @@ -0,0 +1,210 @@ +# ============================================================================= +# DOCUMENTS STACK - Document management, PDF tools, and AI services +# ============================================================================= +# DEPLOYMENT INSTRUCTIONS +# ============================================================================= +# This is one of multiple compose files in the /docker/ directory. +# +# Deploy ALL stacks (from /docker/ directory): +# Get-ChildItem docker-compose.*.yml | ForEach-Object { docker compose -f $_ up -d } +# +# Deploy this stack only: +# docker compose -f docker-compose.documents.yml up -d +# +# Stop this stack: +# docker compose -f docker-compose.documents.yml down +# +# View logs for this stack: +# docker compose -f docker-compose.documents.yml logs -f +# +# IMPORTANT: Requires infra stack to be deployed first (shared networks). +# ============================================================================= + +name: docker-documents + +# Common configurations for re-use +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +x-security: &default-security + security_opt: + - no-new-privileges:true + +services: + onlyoffice: + image: onlyoffice/documentserver:latest + container_name: onlyoffice-docs + restart: always + ports: + - "8091:80" + environment: + - JWT_ENABLED=true + - JWT_SECRET=${OO_JWT_SECRET} + - JWT_HEADER=Authorization + - ALLOW_PRIVATE_IP_ADDRESS=true + - USE_UNAUTHORIZED_STORAGE=true + volumes: + - docker_onlyoffice:/var/lib/onlyoffice + - docker_onlyoffice:/var/www/onlyoffice/Data + - docker_onlyoffice:/var/log/onlyoffice + + paperless-db: + image: mariadb:11 + container_name: paperless-db + restart: unless-stopped + networks: + - db_net + environment: + - MARIADB_ROOT_PASSWORD=${PAPERLESS_DB_ROOT_PASSWORD} + - MARIADB_DATABASE=${PAPERLESS_DB_NAME} + - MARIADB_USER=${PAPERLESS_DB_USER} + - MARIADB_PASSWORD=${PAPERLESS_DB_PASSWORD} + volumes: + - docker_dbdata:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 3 + logging: *default-logging + + paperless-broker: + image: redis:8 + container_name: paperless-broker + restart: unless-stopped + networks: + - internal_net + volumes: + - docker_redisdata:/data + logging: *default-logging + + paperless-tika: + image: apache/tika:latest + container_name: paperless-tika + restart: unless-stopped + networks: + - internal_net + logging: *default-logging + + paperless-gotenberg: + image: gotenberg/gotenberg:8.20 + container_name: paperless-gotenberg + restart: unless-stopped + networks: + - internal_net + command: ["gotenberg", "--chromium-disable-javascript=true", "--chromium-allow-list=file:///tmp/.*"] + logging: *default-logging + + paperless-webserver: + image: ghcr.io/paperless-ngx/paperless-ngx:latest + container_name: paperless-webserver + restart: unless-stopped + networks: + - web_net + - db_net + - internal_net + ports: + - "8100:8000" + environment: + - PAPERLESS_DBENGINE=mariadb + - PAPERLESS_DBHOST=paperless-db + - PAPERLESS_DBPASS=${PAPERLESS_DB_PASSWORD} + - PAPERLESS_DBUSER=${PAPERLESS_DB_USER} + - PAPERLESS_DBPORT=3306 + - PAPERLESS_REDIS=redis://paperless-broker:6379 + - PAPERLESS_TIKA_ENDPOINT=http://paperless-tika:9998 + - PAPERLESS_TIKA_GOTENBERG_ENDPOINT=http://paperless-gotenberg:3000 + - PAPERLESS_SECRET_KEY=${PAPERLESS_SECRET_KEY} + - PAPERLESS_URL=${PAPERLESS_URL} + - PAPERLESS_TIME_ZONE=${TZ} + - PAPERLESS_OCR_LANGUAGE=eng + volumes: + - /docker/paperless/consume:/usr/src/paperless/consume + - /docker/paperless/data:/usr/src/paperless/data + - /docker/paperless/export:/usr/src/paperless/export + - /docker/paperless/media:/usr/src/paperless/media + depends_on: + paperless-db: + condition: service_healthy + paperless-broker: + condition: service_started + logging: *default-logging + + paperless-ai: + image: clusterzx/paperless-ai + container_name: paperless-ai + restart: unless-stopped + networks: + - internal_net + ports: + - 3040:3000 + environment: + - PUID=${PUID} + - PGID=${PGID} + - PAPERLESS_USERNAME=jamie + - PAPERLESS_API_URL=http://paperless-webserver:8000/api + - PAPERLESS_API_TOKEN=${PAPERLESS_SECRET_KEY} + - RAG_SERVICE_URL=http://localhost:8000 + - RAG_SERVICE_ENABLED=true + volumes: + - docker_aidata:/app/data + depends_on: + - paperless-webserver + logging: *default-logging + + stirling-pdf: + image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest + container_name: stirling-PDF + restart: unless-stopped + networks: + - web_net + ports: + - "8090:8080" + environment: + - UI_APPNAME=Stirling-PDF + - SHOW_SURVEY=false + - SECURITY_ENABLELOGIN=false + - SYSTEM_MAXFILESIZE=100 + - PUID=${PUID} + - PGID=${PGID} + volumes: + - /docker/stirling/config:/configs + - /docker/stirling/data:/usr/share/tessdata + - /docker/stirling/logs:/logs + logging: *default-logging + + open-webui: + image: ghcr.io/open-webui/open-webui:main + container_name: open-webui + restart: unless-stopped + networks: + - web_net + - internal_net + ports: + - "3000:8080" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - open-webui:/app/backend/data + logging: *default-logging + +networks: + db_net: + name: db_net + external: true + internal_net: + name: internal_net + external: true + web_net: + name: web_net + external: true + +volumes: + docker_aidata: + docker_dbdata: + docker_onlyoffice: + docker_redisdata: + open-webui: diff --git a/docker-compose.yaml b/docker-compose.full.yaml.bak similarity index 85% rename from docker-compose.yaml rename to docker-compose.full.yaml.bak index 829627b..59f971e 100644 --- a/docker-compose.yaml +++ b/docker-compose.full.yaml.bak @@ -10,24 +10,7 @@ x-security: &default-security - no-new-privileges:true services: - # --- MANAGEMENT & INFRASTRUCTURE --- - glances: - image: nicolargo/glances:ubuntu-latest-full - container_name: glances - restart: unless-stopped - network_mode: host - environment: - - GLANCES_OPT=-w - devices: - - /dev/dri:/dev/dri - volumes: - - /data:/data:ro - - /docker:/docker:ro - - /docker:/docker-local:ro - - /run/user/1000/podman/podman.sock:/run/user/1000/podman/podman.sock - - /var/run/docker.sock:/var/run/docker.sock - logging: *default-logging - + # --- MANAGEMENT & INFRASTRUCTURE --- ntopng: image: ntop/ntopng:latest container_name: ntopng @@ -71,14 +54,27 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ./wud/store:/store environment: - - WUD_SERVER_ENABLED=false - - WUD_REGISTRY_CUSTOM_LSCR_URL=https://lscr.io + - WUD_SERVER_ENABLED=true + - WUD_REGISTRY_HUB_0_LOGIN=${WUD_REGISTRY_HUB_0_LOGIN} + - WUD_REGISTRY_HUB_0_TOKEN=${WUD_REGISTRY_HUB_0_TOKEN} + - WUD_REGISTRY_HUB_0_AUTH=true + - WUD_REGISTRY_GHCR_0_USERNAME=${WUD_REGISTRY_GHCR_0_USERNAME} + - WUD_REGISTRY_GHCR_0_TOKEN=${WUD_REGISTRY_GHCR_0_TOKEN} + - WUD_REGISTRY_LSCR_0_USERNAME=${WUD_REGISTRY_GHCR_0_USERNAME} + - WUD_REGISTRY_LSCR_0_TOKEN=${WUD_REGISTRY_GHCR_0_TOKEN} + - WUD_TRIGGER_MQTT_MOSQUITTO_URL=${MQTT_MOSQUITTO_URL} + - WUD_TRIGGER_MQTT_MOSQUITTO_USER=${MQTT_MOSQUITTO_USER} + - WUD_TRIGGER_MQTT_MOSQUITTO_PASSWORD=${MQTT_MOSQUITTO_PASSWORD} + - WUD_TRIGGER_MQTT_MOSQUITTO_HASS_ENABLED=true + - WUD_TRIGGER_MQTT_MOSQUITTO_HASS_PREFIX=homeassistant + ports: + - 3666:3000 healthcheck: test: curl --fail http://localhost:${WUD_SERVER_PORT:-3000}/health || exit 1 interval: 10s timeout: 10s retries: 3 - start_period: 10s + start_period: 10s labels: - 'wud.tag.include=^\d+\.\d+\.\d+$$' - 'wud.link.template=https://github.com/getwud/wud/releases/tag/$${major}.$${minor}.$${patch}' @@ -167,7 +163,7 @@ services: db_net: volumes: - ./npm/mysql:/var/lib/mysql - + # --- MEDIA & ARRS (Static IPs 172.20.0.x) --- prowlarr: image: lscr.io/linuxserver/prowlarr:latest @@ -356,7 +352,6 @@ services: image: postgres container_name: retrom-db hostname: retrom-db - env_file: ./.env restart: always # set shared memory limit when using docker-compose shm_size: 128mb @@ -422,7 +417,7 @@ services: - docker_onlyoffice:/var/lib/onlyoffice - docker_onlyoffice:/var/www/onlyoffice/Data - docker_onlyoffice:/var/log/onlyoffice - + paperless-db: image: mariadb:11 container_name: paperless-db @@ -512,10 +507,13 @@ services: networks: - internal_net ports: - - "127.0.0.1:3040:3000" + - 3040:3000 environment: - PUID=${PUID} - PGID=${PGID} + - PAPERLESS_USERNAME=jamie + - PAPERLESS_API_URL=http://paperless-webserver:8000/api + - PAPERLESS_API_TOKEN=${PAPERLESS_SECRET_KEY} - RAG_SERVICE_URL=http://localhost:8000 - RAG_SERVICE_ENABLED=true volumes: @@ -560,43 +558,6 @@ services: - open-webui:/app/backend/data logging: *default-logging - litellm: - image: docker.litellm.ai/berriai/litellm:main-latest - container_name: litellm - restart: unless-stopped - networks: - - internal_net - - db_net - ports: - - "127.0.0.1:4000:4000" - environment: - - GROQ_API_KEY=${GROQ_API_KEY} - - DATABASE_URL=postgresql://litellm:litellm_pass@litellm-postgres:5432/litellm_db - - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} - volumes: - - /docker/litellm/config.yaml:/app/config.yaml - depends_on: - litellm-postgres: - condition: service_healthy - logging: *default-logging - - litellm-postgres: - image: postgres:15 - container_name: litellm-postgres - restart: unless-stopped - networks: - - db_net - environment: - - POSTGRES_USER=litellm - - POSTGRES_PASSWORD=litellm_pass - - POSTGRES_DB=litellm_db - volumes: - - /docker/litellm/postgres-data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U litellm -d litellm_db"] - interval: 10s - logging: *default-logging - # --- PHOTO & DATA MGMT --- immich-server: image: ghcr.io/immich-app/immich-server:release @@ -645,6 +606,8 @@ services: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s logging: *default-logging + labels: + - wud.watch=false immich-machine-learning: image: ghcr.io/immich-app/immich-machine-learning:release @@ -680,7 +643,7 @@ services: - /docker/syncthing:/var/syncthing logging: *default-logging - # --- DEVELOPMENT & APPS --- + # --- GIT --- gitea: image: docker.gitea.com/gitea:1.25.3 container_name: gitea @@ -722,56 +685,45 @@ services: interval: 10s logging: *default-logging - # --- GENEALOGY (Gramps) --- - grampsweb-redis: - image: redis:7.2.4-alpine - container_name: grampsweb_redis + # --- UTILITIES --- + tandoor_db: restart: unless-stopped - networks: - - internal_net - logging: *default-logging - - grampsweb-jamie: - image: ghcr.io/gramps-project/grampsweb:latest - container_name: grampsweb-jamie - restart: unless-stopped - networks: - - web_net - - internal_net - ports: - - "5511:5000" + image: postgres:16-alpine + container_name: tandoor_DB environment: - - GRAMPSWEB_TREE=Miller Tree - - GRAMPSWEB_CELERY_CONFIG__broker_url=redis://grampsweb_redis:6379/0 + - POSTGRES_DB=${TANDOOR_POSTGRES_DB} + - POSTGRES_USER=${TANDOOR_POSTGRES_USER} + - POSTGRES_PASSWORD=${TANDOOR_POSTGRES_PASSWORD} volumes: - - /docker/gramps-jamie/cache:/app/cache - - /docker/gramps-jamie/db:/app/data/.gramps/grampsdb - - /docker/gramps-jamie/media:/app/media - depends_on: - - grampsweb-redis - logging: *default-logging - - grampsweb-helen: - image: ghcr.io/gramps-project/grampsweb:latest - container_name: grampsweb-helen - restart: unless-stopped - networks: - - web_net - - internal_net - ports: - - "5512:5000" - environment: - - GRAMPSWEB_TREE=Helen Tree - - GRAMPSWEB_CELERY_CONFIG__broker_url=redis://grampsweb_redis:6379/0 - volumes: - - /docker/gramps-helen/cache:/app/cache - - /docker/gramps-helen/db:/app/data/.gramps/grampsdb - - /docker/gramps-helen/media:/app/media - depends_on: - - grampsweb-redis - logging: *default-logging + - ./tandoor/postgresql:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${TANDOOR_POSTGRES_USER} -d ${TANDOOR_POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 - # --- UTILITIES --- + tandoor_web: + restart: unless-stopped + image: vabene1111/recipes + container_name: tandoor_web + environment: + - ALLOWED_HOSTS=recipes.kansaigaijin.com + - SECRET_KEY=${TANDOOR_SECRET_KEY} + - DB_ENGINE=${TANDOOR_DB_ENGINE} + - POSTGRES_HOST=${TANDOOR_POSTGRES_HOST} + - POSTGRES_DB=${TANDOOR_POSTGRES_DB} + - POSTGRES_PORT=${TANDOOR_POSTGRES_PORT} + - POSTGRES_USER=${TANDOOR_POSTGRES_USER} + - POSTGRES_PASSWORD=${TANDOOR_POSTGRES_PASSWORD} + ports: + - 8450:80 + volumes: + - ./tandoor/staticfiles:/opt/recipes/staticfiles + - ./tandoor/mediafiles:/opt/recipes/mediafiles + depends_on: + tandoor_db: + condition: service_healthy + speedtest-tracker: image: lscr.io/linuxserver/speedtest-tracker:latest container_name: speedtest-tracker @@ -847,25 +799,6 @@ services: - /docker/redbot:/data logging: *default-logging - discord-agent: - build: ./discord-agent - container_name: discord-agent - restart: unless-stopped - networks: - - internal_net - volumes: - - ./discord-agent/data:/app/data - - /docker:/docker - env_file: - - .env - environment: - - DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN} - - OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT:-http://ollama:11434} - - OLLAMA_MODEL=${OLLAMA_MODEL:-llama3.2} - - OPENCODE_ENDPOINT=${OPENCODE_ENDPOINT:-http://192.168.0.10:4096} - - OPENCODE_API_KEY=${OPENCODE_API_KEY} - logging: *default-logging - iperf3-server: image: networkstatic/iperf3 container_name: iperf3-server @@ -876,9 +809,48 @@ services: - "5201:5201" command: -s logging: *default-logging + + neolink: + image: quantumentangledandy/neolink + container_name: neolink + ports: + - 8554:8554 + volumes: + - ./neolink/neolink.toml:/etc/neolink.toml + restart: unless-stopped - # --- FINANCE --- - + linkwarden-db: + container_name: linkwarden-db + image: postgres:16-alpine + restart: always + environment: + - POSTGRES_PASSWORD=${LINKWARDEN_DB_PASSWORD} + volumes: + - ./linkwarden/pgdata:/var/lib/postgresql/data + + linkwarden: + container_name: linkwarden + environment: + - DATABASE_URL=postgresql://postgres:${LINKWARDEN_DB_PASSWORD}@linkwarden-db:5432/postgres + - NEXTAUTH_SECRET=${NEXTAUTH_SECRET} + - NEXTAUTH_URL=${NEXTAUTH_URL} + restart: always + image: ghcr.io/linkwarden/linkwarden:latest + ports: + - 3400:3000 + volumes: + - ./linkwarden/data:/data/data + depends_on: + - linkwarden-db + - meilisearch + + meilisearch: + container_name: meilisearch + image: getmeili/meilisearch:v1.12.8 + restart: always + volumes: + - ./linkwarden/meili_data:/meili_data + # --- MUSIC & SCROBBLING --- maloja: image: krateng/maloja:latest @@ -917,6 +889,15 @@ services: - /docker/scrobble/config:/config logging: *default-logging + metube: + image: ghcr.io/alexta69/metube + container_name: metube + restart: unless-stopped + ports: + - "8081:8081" + volumes: + - /mnt/nas-storage/data/torrents/metube:/downloads + networks: media_net: name: media_net @@ -940,4 +921,3 @@ volumes: docker_soulsync: open-webui: portainer_data: - discord_agent_data: diff --git a/docker-compose.infra.yml b/docker-compose.infra.yml new file mode 100644 index 0000000..263e0c7 --- /dev/null +++ b/docker-compose.infra.yml @@ -0,0 +1,207 @@ +# ============================================================================= +# INFRASTRUCTURE STACK - Core services & shared networks +# ============================================================================= +# DEPLOYMENT INSTRUCTIONS +# ============================================================================= +# This is one of multiple compose files in the /docker/ directory. +# +# Deploy ALL stacks (from /docker/ directory): +# Get-ChildItem docker-compose.*.yml | ForEach-Object { docker compose -f $_ up -d } +# +# Deploy this stack only: +# docker compose -f docker-compose.infra.yml up -d +# +# Stop this stack: +# docker compose -f docker-compose.infra.yml down +# +# View logs for this stack: +# docker compose -f docker-compose.infra.yml logs -f +# +# IMPORTANT: Deploy this stack FIRST - it defines the shared networks +# (media_net, db_net, web_net, internal_net) that all other stacks require. +# ============================================================================= + +name: docker-infra + +# Common configurations for re-use +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +x-security: &default-security + security_opt: + - no-new-privileges:true + +services: + portainer: + image: portainer/portainer-ce:2.21.5 + container_name: portainer + restart: unless-stopped + networks: + - web_net + ports: + - "8000:8000" + - "9443:9443" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - portainer_data:/data + deploy: + resources: + limits: + memory: 512M + logging: *default-logging + + ntopng: + image: ntop/ntopng:latest + container_name: ntopng + restart: unless-stopped + network_mode: host + cap_add: + - NET_ADMIN + - NET_RAW + command: > + --interface=eth0 + --http-port=3939 + --disable-login=1 + --community + volumes: + - /docker-local/ntopng/data:/var/lib/ntopng + - /docker-local/ntopng/redis:/var/lib/redis + logging: *default-logging + + whatsupdocker: + image: getwud/wud:latest + container_name: wud + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./wud/store:/store + environment: + - WUD_SERVER_ENABLED=true + - WUD_REGISTRY_HUB_0_LOGIN=${WUD_REGISTRY_HUB_0_LOGIN} + - WUD_REGISTRY_HUB_0_TOKEN=${WUD_REGISTRY_HUB_0_TOKEN} + - WUD_REGISTRY_HUB_0_AUTH=true + - WUD_REGISTRY_GHCR_0_USERNAME=${WUD_REGISTRY_GHCR_0_USERNAME} + - WUD_REGISTRY_GHCR_0_TOKEN=${WUD_REGISTRY_GHCR_0_TOKEN} + - WUD_REGISTRY_LSCR_0_USERNAME=${WUD_REGISTRY_GHCR_0_USERNAME} + - WUD_REGISTRY_LSCR_0_TOKEN=${WUD_REGISTRY_GHCR_0_TOKEN} + - WUD_TRIGGER_MQTT_MOSQUITTO_URL=${MQTT_MOSQUITTO_URL} + - WUD_TRIGGER_MQTT_MOSQUITTO_USER=${MQTT_MOSQUITTO_USER} + - WUD_TRIGGER_MQTT_MOSQUITTO_PASSWORD=${MQTT_MOSQUITTO_PASSWORD} + - WUD_TRIGGER_MQTT_MOSQUITTO_HASS_ENABLED=true + - WUD_TRIGGER_MQTT_MOSQUITTO_HASS_PREFIX=homeassistant + ports: + - 3666:3000 + healthcheck: + test: curl --fail http://localhost:${WUD_SERVER_PORT:-3000}/health || exit 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 10s + labels: + - 'wud.tag.include=^\d+\.\d+\.\d+$$' + - 'wud.link.template=https://github.com/getwud/wud/releases/tag/$${major}.$${minor}.$${patch}' + + homepage: + image: ghcr.io/gethomepage/homepage:latest + container_name: homepage + restart: unless-stopped + networks: + - web_net + ports: + - "7575:3000" + environment: + - PUID=${PUID} + - PGID=${PGID} + - TZ=${TZ} + - HOMEPAGE_ALLOWED_HOSTS=${HOMEPAGE_ALLOWED_HOSTS} + volumes: + - ./Homepage/config/icons:/app/public/icons + - ./Homepage/config/images:/app/public/images + - ./Homepage/config:/app/config + logging: *default-logging + + dockerproxy: + image: ghcr.io/tecnativa/docker-socket-proxy:latest + container_name: dockerproxy + restart: unless-stopped + networks: + - internal_net + ports: + - "127.0.0.1:2375:2375" + environment: + - CONTAINERS=1 + - SERVICES=1 + - TASKS=1 + - EVENTS=1 + - PING=1 + - VERSION=1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + logging: *default-logging + + newt: + image: fosrl/newt:latest + container_name: newt + restart: unless-stopped + environment: + - PANGOLIN_ENDPOINT=https://png.kansaigaijin.com + - NEWT_ID=cuvfw5hnsszh0gc + - NEWT_SECRET=iitbnuk2cevm40lt1xtrgmnehce4f2bdk4rnllj6ebeznf6h + - LOG_LEVEL=DEBUG + + npm: + image: 'jc21/nginx-proxy-manager:latest' + container_name: npm + restart: unless-stopped + ports: + - '80:80' + - '443:443' + - '81:81' + environment: + DB_MYSQL_HOST: "npm-db" + DB_MYSQL_PORT: 3306 + DB_MYSQL_USER: "npm" + DB_MYSQL_PASSWORD: ${NPM_PASSWORD} + DB_MYSQL_NAME: "npm" + volumes: + - ./npm/data:/data + - ./npm/letsencrypt:/etc/letsencrypt + networks: + web_net: + db_net: + depends_on: + - npm-db + + npm-db: + image: 'jc21/mariadb-aria:latest' + container_name: npm-db + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${NPM_PASSWORD} + MYSQL_DATABASE: 'npm' + MYSQL_USER: 'npm' + MYSQL_PASSWORD: ${NPM_PASSWORD} + networks: + db_net: + volumes: + - ./npm/mysql:/var/lib/mysql + +networks: + media_net: + name: media_net + driver: bridge + db_net: + name: db_net + internal: true + web_net: + name: web_net + driver: bridge + internal_net: + name: internal_net + driver: bridge + +volumes: + portainer_data: diff --git a/docker-compose.media.yml b/docker-compose.media.yml new file mode 100644 index 0000000..bb2e70d --- /dev/null +++ b/docker-compose.media.yml @@ -0,0 +1,275 @@ +# ============================================================================= +# MEDIA STACK - Media management, downloading, and playback services +# ============================================================================= +# DEPLOYMENT INSTRUCTIONS +# ============================================================================= +# This is one of multiple compose files in the /docker/ directory. +# +# Deploy ALL stacks (from /docker/ directory): +# Get-ChildItem docker-compose.*.yml | ForEach-Object { docker compose -f $_ up -d } +# +# Deploy this stack only: +# docker compose -f docker-compose.media.yml up -d +# +# Stop this stack: +# docker compose -f docker-compose.media.yml down +# +# View logs for this stack: +# docker compose -f docker-compose.media.yml logs -f +# +# IMPORTANT: Requires infra stack to be deployed first (shared networks). +# ============================================================================= + +name: docker-media + +# Common configurations for re-use +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +x-security: &default-security + security_opt: + - no-new-privileges:true + +services: + prowlarr: + image: lscr.io/linuxserver/prowlarr:latest + container_name: prowlarr + restart: unless-stopped + networks: + media_net: + ports: + - "9696:9696" + environment: + - PUID=${PUID} + - PGID=${PGID} + - TZ=${TZ} + volumes: + - /docker/Arrs/Prowlarr/config:/config + logging: *default-logging + + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest + container_name: flaresolverr + restart: unless-stopped + networks: + media_net: + ports: + - "127.0.0.1:8191:8191" + environment: + - TZ=${TZ} + logging: *default-logging + + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:latest + container_name: qbittorrent + restart: unless-stopped + networks: + media_net: + ports: + - "56881:6881" + - "56881:6881/udp" + - "7070:8080" + environment: + - TZ=${TZ} + - WEBUI_PORT=8080 + - PUID=${PUID} + - PGID=${PGID} + volumes: + - /mnt/nas-storage/data:/data + - /docker/qBittorrent/config:/config + logging: *default-logging + + radarr: + image: lscr.io/linuxserver/radarr:latest + container_name: radarr + restart: unless-stopped + networks: + media_net: + ports: + - "7878:7878" + environment: + - PUID=${PUID} + - PGID=${PGID} + - TZ=${TZ} + volumes: + - /mnt/nas-storage/data/:/data + - /docker/Arrs/Radarr/config:/config + logging: *default-logging + + sonarr: + image: lscr.io/linuxserver/sonarr:latest + container_name: sonarr + restart: unless-stopped + networks: + media_net: + ports: + - "8989:8989" + environment: + - PUID=${PUID} + - PGID=${PGID} + - TZ=${TZ} + volumes: + - /mnt/nas-storage/data:/data + - /docker/Arrs/Sonarr/config:/config + logging: *default-logging + + lidarr: + image: ghcr.io/hotio/lidarr:nightly + container_name: lidarr + restart: unless-stopped + networks: + media_net: + ports: + - "8686:8686" + environment: + - TZ=${TZ} + - PUID=${PUID} + - PGID=${PGID} + volumes: + - /mnt/nas-storage/data:/data + - /docker/Arrs/Lidarr/config:/config + logging: *default-logging + + bazarr: + image: lscr.io/linuxserver/bazarr:latest + container_name: bazarr + restart: unless-stopped + networks: + media_net: + ports: + - "6767:6767" + environment: + - PUID=${PUID} + - PGID=${PGID} + - TZ=${TZ} + volumes: + - /mnt/nas-storage/data:/data + - /docker/Arrs/Bazarr/config:/config + logging: *default-logging + + seerr: + image: ghcr.io/seerr-team/seerr:latest + init: true + container_name: seerr + environment: + - LOG_LEVEL=debug + - TZ=${TZ} + networks: + media_net: + ports: + - 5055:5055 + volumes: + - /docker/Arrs/Seerr/config:/app/config + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:5055/api/v1/status || exit 1 + start_period: 20s + timeout: 3s + interval: 15s + retries: 3 + restart: unless-stopped + + jellyfin: + image: jellyfin/jellyfin:latest + container_name: jellyfin + restart: unless-stopped + networks: + - media_net + - internal_net + ports: + - "8096:8096" + environment: + - JELLYFIN_PublishedServerUrl=${JELLYFIN_URL} + - TZ=${TZ} + - PUID=${PUID} + - PGID=${PGID} + group_add: + - "104" + devices: + - /dev/dri/renderD128:/dev/dri/renderD128 + volumes: + - /mnt/nas-storage/data:/data + - /docker/Arrs/Jellyfin/cache:/cache + - /docker/Arrs/Jellyfin/config:/config + logging: *default-logging + + slskd: + image: slskd/slskd:latest + container_name: slskd + restart: unless-stopped + networks: + media_net: + ports: + - "5030:5030" + - "50300:50300" + - "5031:5031" + hostname: slskd + environment: + - TZ=${TZ} + - SLSKD_REMOTE_CONFIGURATION=true + - SLSKD_USERNAME=${SLSKD_USERNAME} + - SLSKD_PASSWORD=${SLSKD_PASSWORD} + volumes: + - /mnt/nas-storage/data/torrents/soulsync/complete:/data/torrents/soulsync/complete + - /mnt/nas-storage/data/torrents/soulsync/incomplete:/data/torrents/soulsync/incomplete + - /docker/slskd:/app + logging: *default-logging + + metube: + image: ghcr.io/alexta69/metube + container_name: metube + restart: unless-stopped + ports: + - "8081:8081" + volumes: + - /mnt/nas-storage/data/torrents/metube:/downloads + + maloja: + image: krateng/maloja:latest + container_name: maloja + restart: unless-stopped + environment: + - MALOJA_FORCE_PASSWORD=${MALOJA_FORCE_PASSWORD} + networks: + - internal_net + - web_net + ports: + - "42010:42010" + volumes: + - /docker/maloja/config:/etc/maloja + - /docker/maloja/data:/var/lib/maloja + logging: *default-logging + + multi-scrobbler: + image: foxxmd/multi-scrobbler:latest + container_name: multi-scrobbler + restart: unless-stopped + networks: + - internal_net + ports: + - "9078:9078" + environment: + - MALOJA_URL=${MALOJA_URL} + - MALOJA_API_KEY=${MALOJA_API_KEY} + - JELLYFIN_URL=${JELLYFIN_URL} + - JELLYFIN_APIKEY=${JELLYFIN_APIKEY_MS} + - JELLYFIN_USER=${JELLYFIN_USER} + - JELLYFIN_TRANSFORMS=musicbrainz + - MB_CONTACT=${MB_CONTACT} + - MB_PRESETS=default,sensible,native + volumes: + - /docker/scrobble/config:/config + logging: *default-logging + +networks: + media_net: + name: media_net + external: true + internal_net: + name: internal_net + external: true + web_net: + name: web_net + external: true diff --git a/docker-compose.photo-roms.yml b/docker-compose.photo-roms.yml new file mode 100644 index 0000000..e3b2c62 --- /dev/null +++ b/docker-compose.photo-roms.yml @@ -0,0 +1,178 @@ +# ============================================================================= +# PHOTO & ROM LIBRARY STACK - Photo management, file sync, and ROM library +# ============================================================================= +# DEPLOYMENT INSTRUCTIONS +# ============================================================================= +# This is one of multiple compose files in the /docker/ directory. +# +# Deploy ALL stacks (from /docker/ directory): +# Get-ChildItem docker-compose.*.yml | ForEach-Object { docker compose -f $_ up -d } +# +# Deploy this stack only: +# docker compose -f docker-compose.photo-roms.yml up -d +# +# Stop this stack: +# docker compose -f docker-compose.photo-roms.yml down +# +# View logs for this stack: +# docker compose -f docker-compose.photo-roms.yml logs -f +# +# IMPORTANT: Requires infra stack to be deployed first (shared networks). +# ============================================================================= + +name: docker-photo-roms + +# Common configurations for re-use +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +x-security: &default-security + security_opt: + - no-new-privileges:true + +services: + immich-server: + image: ghcr.io/immich-app/immich-server:release + container_name: immich_server + restart: unless-stopped + networks: + - web_net + - db_net + - internal_net + ports: + - "2283:2283" + environment: + - TZ=${TZ} + - DB_USERNAME=postgres + - DB_PASSWORD=${IMMICH_POSTGRES_PASSWORD} + - DB_DATABASE_NAME=immich + - DB_HOSTNAME=immich-postgres + - REDIS_HOSTNAME=immich-redis + - UPLOAD_LOCATION=/data + volumes: + # LOCAL (SSD) - Config, Thumbs, Profile, and Backups + - /docker/immich:/usr/src/app/upload/library + - /docker/immich/thumbs:/usr/src/app/upload/thumbs + - /docker/immich/profile:/usr/src/app/upload/profile + - /docker/immich/backups:/usr/src/app/upload/backups + - /docker/immich/encoded-video:/usr/src/app/upload/encoded-video + - /docker/immich/upload:/usr/src/app/upload/upload + depends_on: + immich-postgres: + condition: service_healthy + logging: *default-logging + + immich-postgres: + image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 + container_name: immich_postgres + restart: unless-stopped + networks: + - db_net + environment: + - POSTGRES_USER=postgres + - POSTGRES_DB=immich + - POSTGRES_PASSWORD=${IMMICH_POSTGRES_PASSWORD} + volumes: + - /docker/immich/postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + logging: *default-logging + labels: + - wud.watch=false + + immich-machine-learning: + image: ghcr.io/immich-app/immich-machine-learning:release + container_name: immich-machine-learning + restart: unless-stopped + networks: + - internal_net + volumes: + - /docker/immich/model-cache:/cache + logging: *default-logging + + immich-redis: + image: valkey/valkey:8-bookworm + container_name: immich_redis + restart: unless-stopped + networks: + - internal_net + logging: *default-logging + + syncthing: + image: syncthing/syncthing + container_name: syncthing + restart: unless-stopped + networks: + - web_net + - internal_net + ports: + - "21027:21027/udp" + - "22000:22000" + - "8384:8384" + volumes: + - /docker/obsidian/vaults:/var/syncthing/obsidian + - /docker/syncthing:/var/syncthing + logging: *default-logging + + retrom-db: + image: postgres + container_name: retrom-db + hostname: retrom-db + restart: always + shm_size: 128mb + ports: + - 5432:5432 + environment: + TZ: "America/Los_Angeles" + PGTZ: "America/Los_Angeles" + POSTGRES_PASSWORD: ${DB_PASS:-password} + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_DB: ${DB_NAME:-retrom-dev} + + retrom-adminer: + container_name: retrom-adminer + image: adminer + restart: always + ports: + - 8080:8080 + + retrom: + container_name: retrom + hostname: retrom + image: ghcr.io/jmberesford/retrom-service:latest + ulimits: + nofile: + hard: 65536 + soft: 65536 + ports: + - 5111:5101 + volumes: + - /mnt/nas-storage/data/media/romms:/app/library + - ./retrom/config:/app/config/ + - ./retrom/data:/app/data/ + depends_on: + - retrom-db + + retrom-jaeger: + image: jaegertracing/jaeger:2.2.0 + ports: + - 16686:16686 + - 4317:4317 + - 4318:4318 + - 5778:5778 + - 9411:9411 + +networks: + web_net: + name: web_net + external: true + db_net: + name: db_net + external: true + internal_net: + name: internal_net + external: true diff --git a/docker-compose.utils.yml b/docker-compose.utils.yml new file mode 100644 index 0000000..601c9ae --- /dev/null +++ b/docker-compose.utils.yml @@ -0,0 +1,252 @@ +# ============================================================================= +# UTILITIES STACK - Development tools, tracking, remote access, and misc services +# ============================================================================= +# DEPLOYMENT INSTRUCTIONS +# ============================================================================= +# This is one of multiple compose files in the /docker/ directory. +# +# Deploy ALL stacks (from /docker/ directory): +# Get-ChildItem docker-compose.*.yml | ForEach-Object { docker compose -f $_ up -d } +# +# Deploy this stack only: +# docker compose -f docker-compose.utils.yml up -d +# +# Stop this stack: +# docker compose -f docker-compose.utils.yml down +# +# View logs for this stack: +# docker compose -f docker-compose.utils.yml logs -f +# +# IMPORTANT: Requires infra stack to be deployed first (shared networks). +# ============================================================================= + +name: docker-utils + +# Common configurations for re-use +x-logging: &default-logging + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + +x-security: &default-security + security_opt: + - no-new-privileges:true + +services: + gitea: + image: docker.gitea.com/gitea:1.25.3 + container_name: gitea + restart: unless-stopped + networks: + - web_net + - db_net + ports: + - "222:22" + - "8418:3000" + environment: + - GITEA__database__HOST=gitea-db:3306 + - GITEA__database__NAME=gitea + - GITEA__database__USER=gitea + - GITEA__database__PASSWD=gitea + - GITEA__database__DB_TYPE=mysql + volumes: + - /docker/gitea/data:/data + depends_on: + gitea-db: + condition: service_healthy + logging: *default-logging + + gitea-db: + image: mysql:8 + container_name: gitea-db + restart: unless-stopped + networks: + - db_net + environment: + - MYSQL_ROOT_PASSWORD=gitea + - MYSQL_USER=gitea + - MYSQL_PASSWORD=gitea + - MYSQL_DATABASE=gitea + volumes: + - /docker/gitea/mysql:/var/lib/mysql + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + interval: 10s + logging: *default-logging + + tandoor_db: + restart: unless-stopped + image: postgres:16-alpine + container_name: tandoor_DB + environment: + - POSTGRES_DB=${TANDOOR_POSTGRES_DB} + - POSTGRES_USER=${TANDOOR_POSTGRES_USER} + - POSTGRES_PASSWORD=${TANDOOR_POSTGRES_PASSWORD} + volumes: + - ./tandoor/postgresql:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${TANDOOR_POSTGRES_USER} -d ${TANDOOR_POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 + + tandoor_web: + restart: unless-stopped + image: vabene1111/recipes + container_name: tandoor_web + environment: + - ALLOWED_HOSTS=recipes.kansaigaijin.com + - SECRET_KEY=${TANDOOR_SECRET_KEY} + - DB_ENGINE=${TANDOOR_DB_ENGINE} + - POSTGRES_HOST=${TANDOOR_POSTGRES_HOST} + - POSTGRES_DB=${TANDOOR_POSTGRES_DB} + - POSTGRES_PORT=${TANDOOR_POSTGRES_PORT} + - POSTGRES_USER=${TANDOOR_POSTGRES_USER} + - POSTGRES_PASSWORD=${TANDOOR_POSTGRES_PASSWORD} + ports: + - 8450:80 + volumes: + - ./tandoor/staticfiles:/opt/recipes/staticfiles + - ./tandoor/mediafiles:/opt/recipes/mediafiles + depends_on: + tandoor_db: + condition: service_healthy + + speedtest-tracker: + image: lscr.io/linuxserver/speedtest-tracker:latest + container_name: speedtest-tracker + restart: unless-stopped + networks: + - web_net + - db_net + ports: + - "8180:80" + environment: + - SPEEDTEST_SCHEDULE="0 */6 * * *" + - SPEEDTEST_SERVERS=7317 + - DB_HOST=speedtest-db + - DB_DATABASE=${SPEEDTEST_DB_NAME} + - DB_PASSWORD=${SPEEDTEST_DB_PASSWORD} + - DB_CONNECTION=mariadb + - DB_USERNAME=${SPEEDTEST_DB_USER} + - APP_KEY=${SPEEDTEST_APP_KEY} + volumes: + - /docker/speedtest-tracker/data:/config + depends_on: + speedtest-db: + condition: service_healthy + logging: *default-logging + + speedtest-db: + image: mariadb:11 + container_name: speedtest-db + restart: unless-stopped + networks: + - db_net + environment: + - MYSQL_USER=${SPEEDTEST_DB_USER} + - MYSQL_PASSWORD=${SPEEDTEST_DB_PASSWORD} + - MYSQL_DATABASE=${SPEEDTEST_DB_NAME} + - MYSQL_RANDOM_ROOT_PASSWORD=true + volumes: + - /docker/speedtest-tracker/db:/var/lib/mysql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + logging: *default-logging + + rustdesk-hbbs: + image: rustdesk/rustdesk-server:latest + container_name: hbbs + network_mode: host + restart: unless-stopped + command: hbbs + volumes: + - /docker/rustdesk/data:/root + logging: *default-logging + + rustdesk-hbbr: + image: rustdesk/rustdesk-server:latest + container_name: hbbr + network_mode: host + restart: unless-stopped + command: hbbr + volumes: + - /docker/rustdesk/data:/root + logging: *default-logging + + redbot: + image: phasecorex/red-discordbot + container_name: redbot + restart: unless-stopped + networks: + - internal_net + environment: + - TOKEN=${REDBOT_TOKEN} + volumes: + - /docker/redbot:/data + logging: *default-logging + + iperf3-server: + image: networkstatic/iperf3 + container_name: iperf3-server + restart: unless-stopped + networks: + - internal_net + ports: + - "5201:5201" + command: -s + logging: *default-logging + + neolink: + image: quantumentangledandy/neolink + container_name: neolink + ports: + - 8554:8554 + volumes: + - ./neolink/neolink.toml:/etc/neolink.toml + restart: unless-stopped + + linkwarden-db: + container_name: linkwarden-db + image: postgres:16-alpine + restart: always + environment: + - POSTGRES_PASSWORD=${LINKWARDEN_DB_PASSWORD} + volumes: + - ./linkwarden/pgdata:/var/lib/postgresql/data + + linkwarden: + container_name: linkwarden + environment: + - DATABASE_URL=postgresql://postgres:${LINKWARDEN_DB_PASSWORD}@linkwarden-db:5432/postgres + - NEXTAUTH_SECRET=${NEXTAUTH_SECRET} + - NEXTAUTH_URL=${NEXTAUTH_URL} + restart: always + image: ghcr.io/linkwarden/linkwarden:latest + ports: + - 3400:3000 + volumes: + - ./linkwarden/data:/data/data + depends_on: + - linkwarden-db + - meilisearch + + meilisearch: + container_name: meilisearch + image: getmeili/meilisearch:v1.12.8 + restart: always + volumes: + - ./linkwarden/meili_data:/meili_data + +networks: + web_net: + name: web_net + external: true + db_net: + name: db_net + external: true + internal_net: + name: internal_net + external: true