titlebot-ng-ng/run_all.sh

#!/usr/bin/env bash

# TitleBot-NG - Master Launch Script
# Starts all components of the TitleBot system
# THIS SCRIPT WAS CREATED VIA BLACKBOX AI -- HUMANIZATION REQUIRED   

set -e

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Detect OS
detect_os() {
    case "$(uname -s)" in
        Linux*)     echo "linux" ;;
        FreeBSD*)   echo "freebsd" ;;
        *)          echo "linux" ;;
    esac
}

OS=$(detect_os)
echo -e "${BLUE}Detected OS: $OS${NC}"

# Ensure Go is installed, attempting native package manager install if missing
ensure_go() {
    if command -v go >/dev/null 2>&1; then
        echo -e "${GREEN}Go found: $(go version)${NC}"
        return 0
    fi

    echo -e "${YELLOW}Go not found. Attempting to install via native package manager...${NC}"

    local pm=""
    if command -v apt-get >/dev/null 2>&1; then
        pm="apt"
    elif command -v pacman >/dev/null 2>&1; then
        pm="pacman"
    elif command -v dnf >/dev/null 2>&1; then
        pm="dnf"
    elif command -v pkg >/dev/null 2>&1 && [ "$OS" = "freebsd" ]; then
        pm="pkg"
    fi

    if [ -z "$pm" ]; then
        echo -e "${RED}Error: No supported package manager found (apt/pacman/dnf/pkg).${NC}"
        echo -e "${YELLOW}Please install Go manually: https://go.dev/dl/${NC}"
        exit 1
    fi

    local sudo_cmd=""
    if [ "$(id -u)" -ne 0 ]; then
        if command -v sudo >/dev/null 2>&1; then
            sudo_cmd="sudo"
        else
            echo -e "${RED}Error: Need root privileges to install Go but sudo is unavailable.${NC}"
            exit 1
        fi
    fi

    case "$pm" in
        apt)
            $sudo_cmd apt-get update
            $sudo_cmd apt-get install -y golang-go
            ;;
        pacman)
            $sudo_cmd pacman -Syu --noconfirm go
            ;;
        dnf)
            $sudo_cmd dnf install -y golang
            ;;
        pkg)
            $sudo_cmd pkg install -y go
            ;;
    esac

    if command -v go >/dev/null 2>&1; then
        echo -e "${GREEN}Go installed successfully: $(go version)${NC}"
    else
        echo -e "${RED}Error: Go installation failed. Please install Go manually: https://go.dev/dl/${NC}"
        exit 1
    fi
}

ensure_go

# Set OS-specific binary names
get_bin_name() {
    local base=$1
    case "$OS" in
        freebsd)
            case "$base" in
                jep)    echo "jepfbd" ;;
                vs)     echo "vsfbsd" ;;
                *)      echo "${base}_${OS}" ;;
            esac
            ;;
        *)
            echo "$base"
            ;;
    esac
}

JEP_BIN=$(get_bin_name "jep")
VS_BIN=$(get_bin_name "vs")

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"

# PID file directory
PID_DIR="$SCRIPT_DIR/.pids"
mkdir -p "$PID_DIR"

# Log directory
LOG_DIR="$SCRIPT_DIR/logs"
mkdir -p "$LOG_DIR"

# Cleanup function
cleanup() {
    echo -e "\n${YELLOW}Shutting down all services...${NC}"
    
    # Kill all background processes
    if [ -d "$PID_DIR" ]; then
        for pidfile in "$PID_DIR"/*.pid; do
            if [ -f "$pidfile" ]; then
                pid=$(cat "$pidfile")
                name=$(basename "$pidfile" .pid)
                if kill -0 "$pid" 2>/dev/null; then
                    echo -e "${YELLOW}Stopping $name (PID: $pid)${NC}"
                    kill "$pid" 2>/dev/null || true
                fi
                rm -f "$pidfile"
                rm -f "$PID_DIR/$name.cmd"
            fi
        done
    fi
    
    # Fallback: kill any lingering service processes even if pid files are stale/missing
    for bin in "$JEP_BIN" "$VS_BIN"; do
        pkill -x "$bin" 2>/dev/null || true
    done
    for script in scraper post_results get_votes get_user_list; do
        pkill -f "python3 $script.py" 2>/dev/null || true
    done

    rm -rf "${RESTART_STATE_DIR:-}" 2>/dev/null || true
    
    echo -e "${GREEN}All services stopped.${NC}"
    exit 0
}

# Set up trap for cleanup
trap cleanup SIGINT SIGTERM EXIT

# Function to start a Go service
start_go_service() {
    local name=$1
    local dir=$2
    local binary=$3
    local rebuild=0
    
    if [ -f "$dir/$binary" ]; then
        echo -e "${GREEN}Found existing $name binary ($dir/$binary), checking freshness...${NC}"
        # Only skip the build if the binary matches the current OS target and
        # is newer than every Go source file in the project (stale binaries rebuild).
        local os_ok=1
        case "$OS:$binary" in
            freebsd:jepfbd|freebsd:vsfbsd)   os_ok=0 ;;
            linux:jep|linux:vs)              os_ok=0 ;;
        esac
        if [ "$os_ok" -ne 0 ]; then
            echo -e "${YELLOW}$binary is for another platform; rebuilding ${name} for $OS${NC}"
            rebuild=1
        else
            local newest_src
            newest_src=$( (cd "$dir" && ls -t *.go go.mod go.sum 2>/dev/null) | head -1 )
            if [ -z "$newest_src" ] || [ "$dir/$binary" -nt "$dir/$newest_src" ]; then
                echo -e "${GREEN}✓ $name binary is current; skipping build${NC}"
                rebuild=0
            else
                echo -e "${YELLOW}$name binary is older than ${newest_src}; rebuilding${NC}"
                rebuild=1
            fi
        fi
    else
        rebuild=1
    fi

    if [ "$rebuild" = "1" ]; then
        echo -e "${BLUE}Building $name...${NC}"
        if [ "$OS" = "freebsd" ]; then
            (cd "$dir" && GOOS=freebsd go build -o "$binary") || {
                echo -e "${RED}Failed to build $name for FreeBSD${NC}"
                return 1
            }
        else
            (cd "$dir" && go build -o "$binary") || {
                echo -e "${RED}Failed to build $name${NC}"
                return 1
            }
        fi
    fi
    
    echo -e "${GREEN}Starting $name...${NC}"
    (cd "$dir" && ./"$binary" > "$LOG_DIR/$name.log" 2>&1) &
    echo $! > "$PID_DIR/$name.pid"
    echo "(cd \"$dir\" && .\"/$binary\" > \"$LOG_DIR/$name.log\" 2>&1) &" > "$PID_DIR/$name.cmd"
    
    # Brief wait to check if it started successfully
    sleep 1
    if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then
        echo -e "${GREEN}✓ $name started (PID: $(cat "$PID_DIR/$name.pid"))${NC}"
        return 0
    else
        echo -e "${RED}✗ $name failed to start${NC}"
        return 1
    fi
}

# Function to start a Python service
start_python_service() {
    local name=$1
    local script=$2
    local config_path=${3:-}
    local config_arg=""
    local label="$name"
    if [ -n "$config_path" ]; then
        config_arg="$config_path"
        label="${name}-$(basename "$config_path" .yaml)"
    fi
    
    echo -e "${GREEN}Starting $label...${NC}"
    
    # Activate virtual environment and run script
    (source .venv/bin/activate && python3 "$script" $config_arg > "$LOG_DIR/$label.log" 2>&1) &
    echo $! > "$PID_DIR/$label.pid"
    echo "(source .venv/bin/activate && python3 \"$script\" $config_arg > \"$LOG_DIR/$label.log\" 2>&1) &" > "$PID_DIR/$label.cmd"
    
    # Brief wait to check if it started successfully
    sleep 1
    if kill -0 $(cat "$PID_DIR/$label.pid") 2>/dev/null; then
        echo -e "${GREEN}✓ $label started (PID: $(cat "$PID_DIR/$label.pid"))${NC}"
        return 0
    else
        echo -e "${RED}✗ $label failed to start${NC}"
        return 1
    fi
}

# Ensure Python virtual environment exists, creating it if missing
ensure_venv() {
    if [ -d ".venv" ]; then
        echo -e "${GREEN}Python virtual environment found at .venv${NC}"
        return 0
    fi

    echo -e "${YELLOW}Python virtual environment not found. Creating it...${NC}"
    python3 -m venv .venv || {
        echo -e "${RED}Error: Failed to create virtual environment. Is python3-venv installed?${NC}"
        echo -e "${YELLOW}Try: sudo apt install python3-venv (or the equivalent for your distro)${NC}"
        exit 1
    }

    echo -e "${GREEN}Virtual environment created. Installing dependencies...${NC}"
    if source .venv/bin/activate && pip install matrix-nio pyyaml requests; then
        echo -e "${GREEN}Dependencies installed successfully.${NC}"
    else
        echo -e "${RED}Error: Failed to install Python dependencies.${NC}"
        exit 1
    fi
}

ensure_venv

# Check if config.yaml exists
if [ ! -f "configs/config.yaml" ]; then
    echo -e "${RED}Error: configs/config.yaml not found${NC}"
    echo -e "${YELLOW}Please copy configs/config_example.yaml to configs/config.yaml and configure it${NC}"
    exit 1
fi

# Discover active room configs (skip example/template files)
room_configs=()
for cfg in configs/*.yaml; do
    if [[ "$(basename "$cfg")" == *example* ]]; then
        continue
    fi
    room_configs+=("$cfg")
done
if [ ${#room_configs[@]} -eq 0 ]; then
    echo -e "${RED}Error: no room configs found in configs/*.yaml${NC}"
    exit 1
fi
echo -e "${BLUE}Found ${#room_configs[@]} room config(s): ${room_configs[*]}${NC}"

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}  TitleBot-NG - Starting All Services  ${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# Start services in order
echo -e "${YELLOW}Cleaning up any existing processes...${NC}"
for bin in "$JEP_BIN" "$VS_BIN"; do
    if pgrep -x "$bin" > /dev/null 2>&1; then
        echo -e "${YELLOW}Stopping existing $bin process...${NC}"
        pkill -x "$bin" 2>/dev/null || true
    fi
done
for script in scraper post_results get_votes get_user_list; do
    if pgrep -f "python3 $script.py" > /dev/null 2>&1; then
        echo -e "${YELLOW}Stopping existing $script process...${NC}"
        pkill -f "python3 $script.py" 2>/dev/null || true
    fi
done
sleep 1

echo -e "${YELLOW}Step 1: Starting scrapers (write room_slugs.json for voting paths)...${NC}"
for cfg in "${room_configs[@]}"; do
    start_python_service "scraper" "scraper.py" "$cfg"
done
sleep 2

echo -e "${YELLOW}Step 2: Starting backend services...${NC}"
start_go_service "jsonEndpoint" "jsonEndpoint" "$JEP_BIN"; true
sleep 2  # Give jsonEndpoint time to fully start

start_go_service "voteServer" "voteServer" "$VS_BIN"; true
sleep 2  # Give voteServer time to fully start

echo ""
echo -e "${YELLOW}Step 3: Starting remaining Matrix bots...${NC}"
for cfg in "${room_configs[@]}"; do
    start_python_service "post_results" "post_results.py" "$cfg"
    sleep 1
    start_python_service "get_votes" "get_votes.py" "$cfg"
    sleep 1
    start_python_service "get_user_list" "get_user_list.py" "$cfg"
done

echo ""
echo -e "${GREEN}========================================${NC}"
echo -e "${GREEN}  All services started successfully!   ${NC}"
echo -e "${GREEN}========================================${NC}"
echo ""
echo -e "${BLUE}Service Status:${NC}"
echo -e "  • JSON Endpoint:   http://localhost:9080"
echo -e "  • Vote Server:     http://localhost:9081"
for cfg in "${room_configs[@]}"; do
    label=$(basename "$cfg" .yaml)
    echo -e "  • Room ($label):      scraper / post_results / get_votes / get_user_list"
done
echo ""
echo -e "${BLUE}Logs are available in: $LOG_DIR${NC}"
echo -e "${BLUE}PIDs are stored in:    $PID_DIR${NC}"
echo ""
echo -e "${YELLOW}Press Ctrl+C to stop all services${NC}"
echo ""

# Minimum free memory (MB) required before an auto-restart is attempted.
# Prevents making an out-of-memory situation worse by launching new processes.
RESTART_MEM_MIN_MB=${RESTART_MEM_MIN_MB:-128}
# Max restart attempts per service within the restart window before backing off.
MAX_RESTARTS_PER_WINDOW=${MAX_RESTARTS_PER_WINDOW:-5}
RESTART_WINDOW_SECS=${RESTART_WINDOW_SECS:-300}

# Track restart attempts per service (name -> timestamp:count)
RESTART_STATE_DIR="$PID_DIR/restarts"
mkdir -p "$RESTART_STATE_DIR"

free_mem_mb() {
    # Returns approximate free memory in MB (0 if unavailable -> allow restart)
    case "$OS" in
        freebsd)
            local pages free_kb
            pages=$(sysctl -n vm.stats.vm.v_free_count 2>/dev/null) || return 0
            free_kb=$(( pages * 4096 / 1024 ))
            echo "$(( free_kb / 1024 ))"
            ;;
        linux)
            awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || return 0
            ;;
        *)
            return 0
            ;;
    esac
}

# Register a restart attempt; returns 0 if allowed, 1 if the service is
# restarting too often (crash loop) and should be left down for now.
check_restart_budget() {
    local name=$1 now
    now=$( date +%s )
    local state_file="$RESTART_STATE_DIR/$name"
    # shellcheck disable=SC2086
    local count=0 first="$now"
    if [ -f "$state_file" ]; then
        read -r first count < "$state_file"
    fi
    # Expire the window once it has passed.
    if [ $(( now - first )) -ge "$RESTART_WINDOW_SECS" ]; then
        count=0
        first=$now
    fi
    if [ "$count" -ge "$MAX_RESTARTS_PER_WINDOW" ]; then
        echo -e "${YELLOW}✗ $name: hit max restarts ($MAX_RESTARTS_PER_WINDOW in ${RESTART_WINDOW_SECS}s), leaving down. Clear $RESTART_STATE_DIR/$name to reset.${NC}"
        return 1
    fi
    count=$(( count + 1 ))
    echo "$first $count" > "$state_file"
    return 0
}

# Function to restart a dead service from its saved relaunch command
restart_service() {
    local name=$1
    local cmd_file="$PID_DIR/$name.cmd"
    
    if [ ! -f "$cmd_file" ]; then
        echo -e "${RED}✗ $name: no restart command stored, cannot auto-restart${NC}"
        return 1
    fi
    
    if ! check_restart_budget "$name"; then
        return 1
    fi
    
    # Do not relaunch more processes if the machine is nearly out of memory.
    local free_mb
    free_mb=$(free_mem_mb)
    if [ -n "$free_mb" ] && [ "$free_mb" -lt "$RESTART_MEM_MIN_MB" ]; then
        echo -e "${YELLOW}✗ $name: only ${free_mb}MB free (need >= ${RESTART_MEM_MIN_MB}MB), skipping restart to avoid OOM.${NC}"
        return 1
    fi
    
    echo -e "${YELLOW}Restarting $name...${NC}"
    rm -f "$PID_DIR/$name.pid"
    # shellcheck disable=SC2046
    eval "$(cat "$cmd_file")"
    echo $! > "$PID_DIR/$name.pid"
    
    sleep 1
    if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then
        echo -e "${GREEN}✓ $name restarted (PID: $(cat "$PID_DIR/$name.pid"))${NC}"
        # A restart that survives is a success; reset its budget.
        rm -f "$RESTART_STATE_DIR/$name"
        return 0
    else
        echo -e "${RED}✗ $name failed to restart (check $LOG_DIR/$name.log)${NC}"
        return 1
    fi
}

# Keep script running and monitor processes
while true; do
    # Check if all processes are still running
    all_running=true
    for pidfile in "$PID_DIR"/*.pid; do
        if [ -f "$pidfile" ]; then
            pid=$(cat "$pidfile")
            if ! kill -0 "$pid" 2>/dev/null; then
                name=$(basename "$pidfile" .pid)
                echo -e "${RED}Warning: $name (PID: $pid) has stopped unexpectedly${NC}"
                all_running=false
                restart_service "$name"
            fi
        fi
    done
    
    if [ "$all_running" = false ]; then
        echo -e "${YELLOW}Some services were restarted. Check logs in $LOG_DIR${NC}"
    fi
    
    sleep 10
done