diff --git a/AGENTS.md b/AGENTS.md index f804ace..fc92cd4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,8 +8,8 @@ This document provides a comprehensive overview of the Docker infrastructure man ### Management & Monitoring - **Glances** - System monitoring dashboard with hardware access - **Portainer** - Container management UI (port 9443) -- **Watchtower** - Automated container updates (daily polling) - **Homepage** - Service dashboard (port 7575) +- **WUD (WhatsUpDocker)** - Docker image management utility ### Reverse Proxy & Security - **Nginx Proxy Manager** - SSL termination and reverse proxy (ports 80, 443, 81) @@ -42,6 +42,7 @@ This document provides a comprehensive overview of the Docker infrastructure man ### Document Processing - **Paperless-ngx** - Document management (port 8100) - MariaDB database, Redis broker, Tika text extraction, Gotenberg PDF processing +- **Paperless-AI** - AI-powered document processing for Paperless-ngx - **Stirling-PDF** - PDF manipulation tools (port 8090) - **OnlyOffice** - Document collaboration (port 8091) @@ -77,8 +78,6 @@ This document provides a comprehensive overview of the Docker infrastructure man - Separate trees for different family branches ### Finance & Tracking -- **WYGIWYH** - Financial tracking (port 9008) - - PostgreSQL database - **Speedtest Tracker** - Network monitoring (port 8180) - MariaDB database, automated testing @@ -94,6 +93,14 @@ This document provides a comprehensive overview of the Docker infrastructure man ### Network Tools - **iperf3-server** - Network performance testing (port 5201) +- **ntopng** - Network traffic monitoring tool (port 3939) + +### ROM Management +- **Retrom** - ROM library management service + - **retrom**: Main ROM service container (port 5111) + - **retrom-db**: PostgreSQL database for Retrom + - **retrom-adminer**: Adminer interface for database management (port 8080) + - **retrom-jaeger**: Distributed tracing for performance monitoring ### Discord Bot (OpenCode) - **Service**: Discord Agent Bot @@ -114,9 +121,7 @@ This document provides a comprehensive overview of the Docker infrastructure man - **Network**: `web_net` ### Other Tools -- **ByteStash** - File sharing service -- **KitchenOwl** - Recipe management -- **PinchFlat** - Cycling tracking +- **Newt** - AI service integration ## Network Architecture @@ -145,6 +150,9 @@ This document provides a comprehensive overview of the Docker infrastructure man ├── scrobble/ (Multi-scrobbler) ├── syncthing/ (File sync) ├── Homepage/ (Dashboard) +├── retrom/ (ROM library management) +├── wud/ (Docker image management) +├── ntopng/ (Network traffic monitoring) └── Various other service configs ``` @@ -187,7 +195,6 @@ This document provides a comprehensive overview of the Docker infrastructure man ## Maintenance Agents ### Automated Tasks -- **Watchtower** - Daily container updates - **Backup Script** - Regular data backups - **Speedtest Tracker** - Automated network testing diff --git a/README.md b/README.md index c2062ec..2e0b7fb 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,14 @@ This infrastructure manages a comprehensive self-hosted environment including me | Media & Arr Stack | Lidarr | Music management | 8686 | | Media & Arr Stack | Bazarr | Subtitle management | 6767 | | Media & Arr Stack | Prowlarr | Indexer manager | 9696 | +| Media & Arr Stack | FlareSolverr | Cloudflare bypass service | 8191 | | Media & Arr Stack | qBittorrent | Download client | 7070 | | Data Management | Paperless | Document management interface | 8100 | +| Data Management | Paperless-AI | AI-powered document processing | - | | Data Management | Stirling-PDF | PDF manipulation tools | 8090 | | Data Management | OnlyOffice | Document collaboration server | 8091 | | Local AI | Open WebUI | LLM interface | 3000 | +| Local AI | LiteLLM | AI gateway | 4000 | | Data Management | Immich | Photo management | 2283 | | Data Management | Syncthing | File synchronization | 8384 | | Management & Infrastructure | Gitea | Git service | 8418 | @@ -31,9 +34,16 @@ This infrastructure manages a comprehensive self-hosted environment including me | Management & Infrastructure | NPM Admin | NPM admin UI | 81 | | Data Management | Maloja | Music scrobbling | 42010 | | Data Management | Multi-Scrobbler | Cross-platform scrobbler | 9078 | -| Data Management | WYGIWYH | Financial tracking | 9008 | | Management & Infrastructure | Portainer | Container management UI | 9443 | | Management & Infrastructure | Homepage | Service dashboard | 7575 | +| Management & Infrastructure | WUD | Docker image management | 3000 | +| Management & Infrastructure | ntopng | Network traffic monitoring | 3939 | +| Management & Infrastructure | Glances | System monitoring dashboard | 61208 | +| Management & Infrastructure | Slskd | Soulseek file sharing client | 5031 | +| Management & Infrastructure | Retrom | ROM library management | 5111 | +| Management & Infrastructure | iperf3-server | Network performance testing | 5201 | +| Discord Bots | Discord Agent Bot | AI-powered Discord bot with Ollama | - | +| Discord Bots | RedBot | Alternative Discord bot | - | ### Discord Agent Integration @@ -117,7 +127,12 @@ TZ=Pacific/Auckland ├── redbot/ ├── maloja/ ├── scrobble/ -└── litellm/ +├── litellm/ +├── retrom/ +├── wud/ +├── ntopng/ +├── slskd/ +└── npm/ ``` ### NAS Storage (`/mnt/Nas-Storage/data/`) @@ -232,7 +247,7 @@ REDBOT_TOKEN=your_discord_bot_token ### Updates -Watchtower automatically updates containers daily. To manually update: +To manually update containers: ``` docker compose pull @@ -269,7 +284,7 @@ ls -l /dev/dri/renderD128 2. Network Segmentation: Database network is internal-only 3. Container Security: `no-new-privileges:true` on supported services 4. Reverse Proxy: Use NPM for SSL termination and authentication -5. Updates: Watchtower keeps containers updated +5. Updates: Regularly update containers with `docker compose pull && docker compose up -d` ## Troubleshooting diff --git a/RESTORE.md b/RESTORE.md index 249ea1b..515d6a2 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -195,7 +195,7 @@ docker run --rm \ ```bash # Start only database services -docker compose up -d paperless-db immich-postgres litellm-postgres wygiwyh-db gitea-db speedtest-db npm-db +docker compose up -d paperless-db immich-postgres litellm-postgres gitea-db speedtest-db npm-db # Wait for databases to be healthy docker compose ps @@ -226,11 +226,6 @@ docker exec -i immich_postgres psql -U postgres immich < /path/to/backup/databas docker exec -i litellm-postgres psql -U litellm litellm_db < /path/to/backup/database-dumps/litellm.sql ``` -### WYGIWYH PostgreSQL -``` -docker exec -i WYGIWYH-db psql -U ${WYGIWYH_DB_USER} ${WYGIWYH_DB_DATABASE} < /path/to/backup/database-dumps/wygiwyh.sql -``` - ### Gitea MySQL ``` docker exec -i gitea-db mysql -u root -pgitea gitea < /path/to/backup/database-dumps/gitea.sql diff --git a/docker-compose.yaml b/docker-compose.yaml index b97b08d..829627b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,6 +27,24 @@ services: - /run/user/1000/podman/podman.sock:/run/user/1000/podman/podman.sock - /var/run/docker.sock:/var/run/docker.sock logging: *default-logging + + ntopng: + image: ntop/ntopng:latest + container_name: ntopng + restart: unless-stopped + network_mode: host + cap_add: + - NET_ADMIN + - NET_RAW + command: > + --interface=eth0 + --http-port=3939 + --disable-login=1 + --community + volumes: + - /docker-local/ntopng/data:/var/lib/ntopng + - /docker-local/ntopng/redis:/var/lib/redis + logging: *default-logging portainer: image: portainer/portainer-ce:2.21.5 @@ -46,18 +64,24 @@ services: memory: 512M logging: *default-logging - watchtower: - image: containrrr/watchtower - container_name: watchtower - restart: unless-stopped - networks: - - internal_net - environment: - - WATCHTOWER_CLEANUP=true - - WATCHTOWER_POLL_INTERVAL=86400 + whatsupdocker: + image: getwud/wud:latest + container_name: wud volumes: - /var/run/docker.sock:/var/run/docker.sock - logging: *default-logging + - ./wud/store:/store + environment: + - WUD_SERVER_ENABLED=false + - WUD_REGISTRY_CUSTOM_LSCR_URL=https://lscr.io + healthcheck: + test: curl --fail http://localhost:${WUD_SERVER_PORT:-3000}/health || exit 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 10s + labels: + - 'wud.tag.include=^\d+\.\d+\.\d+$$' + - 'wud.link.template=https://github.com/getwud/wud/releases/tag/$${major}.$${minor}.$${patch}' homepage: image: ghcr.io/gethomepage/homepage:latest @@ -73,9 +97,9 @@ services: - TZ=${TZ} - HOMEPAGE_ALLOWED_HOSTS=${HOMEPAGE_ALLOWED_HOSTS} volumes: - - /docker/Homepage/config/icons:/app/public/icons - - /docker/Homepage/config/images:/app/public/images - - /docker/Homepage/config:/app/config + - ./Homepage/config/icons:/app/public/icons + - ./Homepage/config/images:/app/public/images + - ./Homepage/config:/app/config logging: *default-logging dockerproxy: @@ -228,7 +252,7 @@ services: logging: *default-logging lidarr: - image: ghcr.io/linuxserver-labs/prarr:lidarr-plugins + image: ghcr.io/hotio/lidarr:nightly container_name: lidarr restart: unless-stopped networks: @@ -261,20 +285,26 @@ services: - /docker/Arrs/Bazarr/config:/config logging: *default-logging - jellyseerr: - image: fallenbagel/jellyseerr:latest - container_name: jellyseerr - restart: unless-stopped - networks: - media_net: - ports: - - "5055:5055" + seerr: + image: ghcr.io/seerr-team/seerr:latest + init: true + container_name: seerr environment: - LOG_LEVEL=debug - TZ=${TZ} + networks: + media_net: + ports: + - 5055:5055 volumes: - - /docker/Arrs/Jellyseerr/config:/app/config - logging: *default-logging + - /docker/Arrs/Seerr/config:/app/config + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:5055/api/v1/status || exit 1 + start_period: 20s + timeout: 3s + interval: 15s + retries: 3 + restart: unless-stopped jellyfin: image: jellyfin/jellyfin:latest @@ -321,6 +351,59 @@ services: - /mnt/nas-storage/data/torrents/soulsync/incomplete:/data/torrents/soulsync/incomplete - /docker/slskd:/app logging: *default-logging + + retrom-db: + image: postgres + container_name: retrom-db + hostname: retrom-db + env_file: ./.env + restart: always + # set shared memory limit when using docker-compose + shm_size: 128mb + ports: + - 5432:5432 + environment: + TZ: "America/Los_Angeles" + PGTZ: "America/Los_Angeles" + POSTGRES_PASSWORD: ${DB_PASS:-password} + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_DB: ${DB_NAME:-retrom-dev} + + retrom-adminer: + container_name: retrom-adminer + image: adminer + restart: always + ports: + - 8080:8080 + + retrom: + container_name: retrom + hostname: retrom + image: ghcr.io/jmberesford/retrom-service:latest + ulimits: + nofile: + hard: 65536 + soft: 65536 + # environment: + # RUST_LOG: DEBUG + # RUST_BACKTRACE: 1 + ports: + - 5111:5101 + volumes: + - /mnt/nas-storage/data/media/romms:/app/library + - ./retrom/config:/app/config/ + - ./retrom/data:/app/data/ + depends_on: + - retrom-db + + retrom-jaeger: + image: jaegertracing/jaeger:2.2.0 + ports: + - 16686:16686 # web client + - 4317:4317 + - 4318:4318 + - 5778:5778 + - 9411:9411 # --- DOCUMENT & AI SUITE --- onlyoffice: @@ -562,6 +645,16 @@ services: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s logging: *default-logging + + immich-machine-learning: + image: ghcr.io/immich-app/immich-machine-learning:release + container_name: immich-machine-learning + restart: unless-stopped + networks: + - internal_net + volumes: + - /docker/immich/model-cache:/cache + logging: *default-logging immich-redis: image: valkey/valkey:8-bookworm @@ -762,6 +855,7 @@ services: - internal_net volumes: - ./discord-agent/data:/app/data + - /docker:/docker env_file: - .env environment: @@ -784,43 +878,7 @@ services: logging: *default-logging # --- FINANCE --- - wygiwyh-web: - image: eitchtee/wygiwyh:latest - container_name: WYGIWYH - environment: - # --- DATABASE SETTINGS --- - - SQL_ENGINE=django.db.backends.postgresql - - SQL_HOST=wygiwyh-db # Matches the service name below - - SQL_PORT=5432 - - SQL_DATABASE=${WYGIWYH_DB_DATABASE} - - SQL_USER=${WYGIWYH_DB_USER} - - SQL_PASSWORD=${WYGIWYH_DB_PASSWORD} - # --- APP SETTINGS --- - - SECRET_KEY=${WYGIWYH_SECRET_KEY} - - DJANGO_ALLOWED_HOSTS=${WYGIWYH_ALLOWED_HOSTS} - - CSRF_TRUSTED_ORIGINS=${WYGIWYH_CSRF_TRUSTED_ORIGINS} - - WYGIWYH_URL=${WYGIWYH_URL} - networks: - - web_net - - db_net - ports: - - 9008:8000 - depends_on: - - wygiwyh-db - restart: unless-stopped - - wygiwyh-db: - image: postgres:15-bookworm - container_name: WYGIWYH-db - environment: - - POSTGRES_USER=${WYGIWYH_DB_USER} - - POSTGRES_PASSWORD=${WYGIWYH_DB_PASSWORD} - - POSTGRES_DB=${WYGIWYH_DB_DATABASE} - networks: - - db_net - volumes: - - ./wygiwyh/postgres_data:/var/lib/postgresql/data/ - + # --- MUSIC & SCROBBLING --- maloja: image: krateng/maloja:latest