478 lines
15 KiB
Bash
Executable File
478 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# TitleBot-NG - Master Launch Script
|
|
# Starts all components of the TitleBot system
|
|
# THIS SCRIPT WAS CREATED VIA BLACKBOX AI -- HUMANIZATION REQUIRED
|
|
|
|
set -e
|
|
|
|
# Color codes for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Detect OS
|
|
detect_os() {
|
|
case "$(uname -s)" in
|
|
Linux*) echo "linux" ;;
|
|
FreeBSD*) echo "freebsd" ;;
|
|
*) echo "linux" ;;
|
|
esac
|
|
}
|
|
|
|
OS=$(detect_os)
|
|
echo -e "${BLUE}Detected OS: $OS${NC}"
|
|
|
|
# Ensure Go is installed, attempting native package manager install if missing
|
|
ensure_go() {
|
|
if command -v go >/dev/null 2>&1; then
|
|
echo -e "${GREEN}Go found: $(go version)${NC}"
|
|
return 0
|
|
fi
|
|
|
|
echo -e "${YELLOW}Go not found. Attempting to install via native package manager...${NC}"
|
|
|
|
local pm=""
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
pm="apt"
|
|
elif command -v pacman >/dev/null 2>&1; then
|
|
pm="pacman"
|
|
elif command -v dnf >/dev/null 2>&1; then
|
|
pm="dnf"
|
|
elif command -v pkg >/dev/null 2>&1 && [ "$OS" = "freebsd" ]; then
|
|
pm="pkg"
|
|
fi
|
|
|
|
if [ -z "$pm" ]; then
|
|
echo -e "${RED}Error: No supported package manager found (apt/pacman/dnf/pkg).${NC}"
|
|
echo -e "${YELLOW}Please install Go manually: https://go.dev/dl/${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
local sudo_cmd=""
|
|
if [ "$(id -u)" -ne 0 ]; then
|
|
if command -v sudo >/dev/null 2>&1; then
|
|
sudo_cmd="sudo"
|
|
else
|
|
echo -e "${RED}Error: Need root privileges to install Go but sudo is unavailable.${NC}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
case "$pm" in
|
|
apt)
|
|
$sudo_cmd apt-get update
|
|
$sudo_cmd apt-get install -y golang-go
|
|
;;
|
|
pacman)
|
|
$sudo_cmd pacman -Syu --noconfirm go
|
|
;;
|
|
dnf)
|
|
$sudo_cmd dnf install -y golang
|
|
;;
|
|
pkg)
|
|
$sudo_cmd pkg install -y go
|
|
;;
|
|
esac
|
|
|
|
if command -v go >/dev/null 2>&1; then
|
|
echo -e "${GREEN}Go installed successfully: $(go version)${NC}"
|
|
else
|
|
echo -e "${RED}Error: Go installation failed. Please install Go manually: https://go.dev/dl/${NC}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
ensure_go
|
|
|
|
# Set OS-specific binary names
|
|
get_bin_name() {
|
|
local base=$1
|
|
case "$OS" in
|
|
freebsd)
|
|
case "$base" in
|
|
jep) echo "jepfbd" ;;
|
|
vs) echo "vsfbsd" ;;
|
|
*) echo "${base}_${OS}" ;;
|
|
esac
|
|
;;
|
|
*)
|
|
echo "$base"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
JEP_BIN=$(get_bin_name "jep")
|
|
VS_BIN=$(get_bin_name "vs")
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
# PID file directory
|
|
PID_DIR="$SCRIPT_DIR/.pids"
|
|
mkdir -p "$PID_DIR"
|
|
|
|
# Log directory
|
|
LOG_DIR="$SCRIPT_DIR/logs"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Cleanup function
|
|
cleanup() {
|
|
echo -e "\n${YELLOW}Shutting down all services...${NC}"
|
|
|
|
# Kill all background processes
|
|
if [ -d "$PID_DIR" ]; then
|
|
for pidfile in "$PID_DIR"/*.pid; do
|
|
if [ -f "$pidfile" ]; then
|
|
pid=$(cat "$pidfile")
|
|
name=$(basename "$pidfile" .pid)
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
echo -e "${YELLOW}Stopping $name (PID: $pid)${NC}"
|
|
kill "$pid" 2>/dev/null || true
|
|
fi
|
|
rm -f "$pidfile"
|
|
rm -f "$PID_DIR/$name.cmd"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Fallback: kill any lingering service processes even if pid files are stale/missing
|
|
for bin in "$JEP_BIN" "$VS_BIN"; do
|
|
pkill -x "$bin" 2>/dev/null || true
|
|
done
|
|
for script in scraper post_results get_votes get_user_list; do
|
|
pkill -f "python3 $script.py" 2>/dev/null || true
|
|
done
|
|
|
|
rm -rf "${RESTART_STATE_DIR:-}" 2>/dev/null || true
|
|
|
|
echo -e "${GREEN}All services stopped.${NC}"
|
|
exit 0
|
|
}
|
|
|
|
# Set up trap for cleanup
|
|
trap cleanup SIGINT SIGTERM EXIT
|
|
|
|
# Function to start a Go service
|
|
start_go_service() {
|
|
local name=$1
|
|
local dir=$2
|
|
local binary=$3
|
|
local rebuild=0
|
|
|
|
if [ -f "$dir/$binary" ]; then
|
|
echo -e "${GREEN}Found existing $name binary ($dir/$binary), checking freshness...${NC}"
|
|
# Only skip the build if the binary matches the current OS target and
|
|
# is newer than every Go source file in the project (stale binaries rebuild).
|
|
local os_ok=1
|
|
case "$OS:$binary" in
|
|
freebsd:jepfbd|freebsd:vsfbsd) os_ok=0 ;;
|
|
linux:jep|linux:vs) os_ok=0 ;;
|
|
esac
|
|
if [ "$os_ok" -ne 0 ]; then
|
|
echo -e "${YELLOW}$binary is for another platform; rebuilding ${name} for $OS${NC}"
|
|
rebuild=1
|
|
else
|
|
local newest_src
|
|
newest_src=$( (cd "$dir" && ls -t *.go go.mod go.sum 2>/dev/null) | head -1 )
|
|
if [ -z "$newest_src" ] || [ "$dir/$binary" -nt "$dir/$newest_src" ]; then
|
|
echo -e "${GREEN}✓ $name binary is current; skipping build${NC}"
|
|
rebuild=0
|
|
else
|
|
echo -e "${YELLOW}$name binary is older than ${newest_src}; rebuilding${NC}"
|
|
rebuild=1
|
|
fi
|
|
fi
|
|
else
|
|
rebuild=1
|
|
fi
|
|
|
|
if [ "$rebuild" = "1" ]; then
|
|
echo -e "${BLUE}Building $name...${NC}"
|
|
if [ "$OS" = "freebsd" ]; then
|
|
(cd "$dir" && GOOS=freebsd go build -o "$binary") || {
|
|
echo -e "${RED}Failed to build $name for FreeBSD${NC}"
|
|
return 1
|
|
}
|
|
else
|
|
(cd "$dir" && go build -o "$binary") || {
|
|
echo -e "${RED}Failed to build $name${NC}"
|
|
return 1
|
|
}
|
|
fi
|
|
fi
|
|
|
|
echo -e "${GREEN}Starting $name...${NC}"
|
|
(cd "$dir" && ./"$binary" > "$LOG_DIR/$name.log" 2>&1) &
|
|
echo $! > "$PID_DIR/$name.pid"
|
|
echo "(cd \"$dir\" && .\"/$binary\" > \"$LOG_DIR/$name.log\" 2>&1) &" > "$PID_DIR/$name.cmd"
|
|
|
|
# Brief wait to check if it started successfully
|
|
sleep 1
|
|
if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then
|
|
echo -e "${GREEN}✓ $name started (PID: $(cat "$PID_DIR/$name.pid"))${NC}"
|
|
return 0
|
|
else
|
|
echo -e "${RED}✗ $name failed to start${NC}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to start a Python service
|
|
start_python_service() {
|
|
local name=$1
|
|
local script=$2
|
|
local config_path=${3:-}
|
|
local config_arg=""
|
|
local label="$name"
|
|
if [ -n "$config_path" ]; then
|
|
config_arg="$config_path"
|
|
label="${name}-$(basename "$config_path" .yaml)"
|
|
fi
|
|
|
|
echo -e "${GREEN}Starting $label...${NC}"
|
|
|
|
# Activate virtual environment and run script
|
|
(source .venv/bin/activate && python3 "$script" $config_arg > "$LOG_DIR/$label.log" 2>&1) &
|
|
echo $! > "$PID_DIR/$label.pid"
|
|
echo "(source .venv/bin/activate && python3 \"$script\" $config_arg > \"$LOG_DIR/$label.log\" 2>&1) &" > "$PID_DIR/$label.cmd"
|
|
|
|
# Brief wait to check if it started successfully
|
|
sleep 1
|
|
if kill -0 $(cat "$PID_DIR/$label.pid") 2>/dev/null; then
|
|
echo -e "${GREEN}✓ $label started (PID: $(cat "$PID_DIR/$label.pid"))${NC}"
|
|
return 0
|
|
else
|
|
echo -e "${RED}✗ $label failed to start${NC}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Ensure Python virtual environment exists, creating it if missing
|
|
ensure_venv() {
|
|
if [ -d ".venv" ]; then
|
|
echo -e "${GREEN}Python virtual environment found at .venv${NC}"
|
|
return 0
|
|
fi
|
|
|
|
echo -e "${YELLOW}Python virtual environment not found. Creating it...${NC}"
|
|
python3 -m venv .venv || {
|
|
echo -e "${RED}Error: Failed to create virtual environment. Is python3-venv installed?${NC}"
|
|
echo -e "${YELLOW}Try: sudo apt install python3-venv (or the equivalent for your distro)${NC}"
|
|
exit 1
|
|
}
|
|
|
|
echo -e "${GREEN}Virtual environment created. Installing dependencies...${NC}"
|
|
if source .venv/bin/activate && pip install matrix-nio pyyaml requests; then
|
|
echo -e "${GREEN}Dependencies installed successfully.${NC}"
|
|
else
|
|
echo -e "${RED}Error: Failed to install Python dependencies.${NC}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
ensure_venv
|
|
|
|
# Check if config.yaml exists
|
|
if [ ! -f "configs/config.yaml" ]; then
|
|
echo -e "${RED}Error: configs/config.yaml not found${NC}"
|
|
echo -e "${YELLOW}Please copy configs/config_example.yaml to configs/config.yaml and configure it${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
# Discover active room configs (skip example/template files)
|
|
room_configs=()
|
|
for cfg in configs/*.yaml; do
|
|
if [[ "$(basename "$cfg")" == *example* ]]; then
|
|
continue
|
|
fi
|
|
room_configs+=("$cfg")
|
|
done
|
|
if [ ${#room_configs[@]} -eq 0 ]; then
|
|
echo -e "${RED}Error: no room configs found in configs/*.yaml${NC}"
|
|
exit 1
|
|
fi
|
|
echo -e "${BLUE}Found ${#room_configs[@]} room config(s): ${room_configs[*]}${NC}"
|
|
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo -e "${BLUE} TitleBot-NG - Starting All Services ${NC}"
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo ""
|
|
|
|
# Start services in order
|
|
echo -e "${YELLOW}Cleaning up any existing processes...${NC}"
|
|
for bin in "$JEP_BIN" "$VS_BIN"; do
|
|
if pgrep -x "$bin" > /dev/null 2>&1; then
|
|
echo -e "${YELLOW}Stopping existing $bin process...${NC}"
|
|
pkill -x "$bin" 2>/dev/null || true
|
|
fi
|
|
done
|
|
for script in scraper post_results get_votes get_user_list; do
|
|
if pgrep -f "python3 $script.py" > /dev/null 2>&1; then
|
|
echo -e "${YELLOW}Stopping existing $script process...${NC}"
|
|
pkill -f "python3 $script.py" 2>/dev/null || true
|
|
fi
|
|
done
|
|
sleep 1
|
|
|
|
echo -e "${YELLOW}Step 1: Starting scrapers (write room_slugs.json for voting paths)...${NC}"
|
|
for cfg in "${room_configs[@]}"; do
|
|
start_python_service "scraper" "scraper.py" "$cfg"
|
|
done
|
|
sleep 2
|
|
|
|
echo -e "${YELLOW}Step 2: Starting backend services...${NC}"
|
|
start_go_service "jsonEndpoint" "jsonEndpoint" "$JEP_BIN"; true
|
|
sleep 2 # Give jsonEndpoint time to fully start
|
|
|
|
start_go_service "voteServer" "voteServer" "$VS_BIN"; true
|
|
sleep 2 # Give voteServer time to fully start
|
|
|
|
echo ""
|
|
echo -e "${YELLOW}Step 3: Starting remaining Matrix bots...${NC}"
|
|
for cfg in "${room_configs[@]}"; do
|
|
start_python_service "post_results" "post_results.py" "$cfg"
|
|
sleep 1
|
|
start_python_service "get_votes" "get_votes.py" "$cfg"
|
|
sleep 1
|
|
start_python_service "get_user_list" "get_user_list.py" "$cfg"
|
|
done
|
|
|
|
echo ""
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo -e "${GREEN} All services started successfully! ${NC}"
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo ""
|
|
echo -e "${BLUE}Service Status:${NC}"
|
|
echo -e " • JSON Endpoint: http://localhost:9080"
|
|
echo -e " • Vote Server: http://localhost:9081"
|
|
for cfg in "${room_configs[@]}"; do
|
|
label=$(basename "$cfg" .yaml)
|
|
echo -e " • Room ($label): scraper / post_results / get_votes / get_user_list"
|
|
done
|
|
echo ""
|
|
echo -e "${BLUE}Logs are available in: $LOG_DIR${NC}"
|
|
echo -e "${BLUE}PIDs are stored in: $PID_DIR${NC}"
|
|
echo ""
|
|
echo -e "${YELLOW}Press Ctrl+C to stop all services${NC}"
|
|
echo ""
|
|
|
|
# Minimum free memory (MB) required before an auto-restart is attempted.
|
|
# Prevents making an out-of-memory situation worse by launching new processes.
|
|
RESTART_MEM_MIN_MB=${RESTART_MEM_MIN_MB:-128}
|
|
# Max restart attempts per service within the restart window before backing off.
|
|
MAX_RESTARTS_PER_WINDOW=${MAX_RESTARTS_PER_WINDOW:-5}
|
|
RESTART_WINDOW_SECS=${RESTART_WINDOW_SECS:-300}
|
|
|
|
# Track restart attempts per service (name -> timestamp:count)
|
|
RESTART_STATE_DIR="$PID_DIR/restarts"
|
|
mkdir -p "$RESTART_STATE_DIR"
|
|
|
|
free_mem_mb() {
|
|
# Returns approximate free memory in MB (0 if unavailable -> allow restart)
|
|
case "$OS" in
|
|
freebsd)
|
|
local pages free_kb
|
|
pages=$(sysctl -n vm.stats.vm.v_free_count 2>/dev/null) || return 0
|
|
free_kb=$(( pages * 4096 / 1024 ))
|
|
echo "$(( free_kb / 1024 ))"
|
|
;;
|
|
linux)
|
|
awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || return 0
|
|
;;
|
|
*)
|
|
return 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Register a restart attempt; returns 0 if allowed, 1 if the service is
|
|
# restarting too often (crash loop) and should be left down for now.
|
|
check_restart_budget() {
|
|
local name=$1 now
|
|
now=$( date +%s )
|
|
local state_file="$RESTART_STATE_DIR/$name"
|
|
# shellcheck disable=SC2086
|
|
local count=0 first="$now"
|
|
if [ -f "$state_file" ]; then
|
|
read -r first count < "$state_file"
|
|
fi
|
|
# Expire the window once it has passed.
|
|
if [ $(( now - first )) -ge "$RESTART_WINDOW_SECS" ]; then
|
|
count=0
|
|
first=$now
|
|
fi
|
|
if [ "$count" -ge "$MAX_RESTARTS_PER_WINDOW" ]; then
|
|
echo -e "${YELLOW}✗ $name: hit max restarts ($MAX_RESTARTS_PER_WINDOW in ${RESTART_WINDOW_SECS}s), leaving down. Clear $RESTART_STATE_DIR/$name to reset.${NC}"
|
|
return 1
|
|
fi
|
|
count=$(( count + 1 ))
|
|
echo "$first $count" > "$state_file"
|
|
return 0
|
|
}
|
|
|
|
# Function to restart a dead service from its saved relaunch command
|
|
restart_service() {
|
|
local name=$1
|
|
local cmd_file="$PID_DIR/$name.cmd"
|
|
|
|
if [ ! -f "$cmd_file" ]; then
|
|
echo -e "${RED}✗ $name: no restart command stored, cannot auto-restart${NC}"
|
|
return 1
|
|
fi
|
|
|
|
if ! check_restart_budget "$name"; then
|
|
return 1
|
|
fi
|
|
|
|
# Do not relaunch more processes if the machine is nearly out of memory.
|
|
local free_mb
|
|
free_mb=$(free_mem_mb)
|
|
if [ -n "$free_mb" ] && [ "$free_mb" -lt "$RESTART_MEM_MIN_MB" ]; then
|
|
echo -e "${YELLOW}✗ $name: only ${free_mb}MB free (need >= ${RESTART_MEM_MIN_MB}MB), skipping restart to avoid OOM.${NC}"
|
|
return 1
|
|
fi
|
|
|
|
echo -e "${YELLOW}Restarting $name...${NC}"
|
|
rm -f "$PID_DIR/$name.pid"
|
|
# shellcheck disable=SC2046
|
|
eval "$(cat "$cmd_file")"
|
|
echo $! > "$PID_DIR/$name.pid"
|
|
|
|
sleep 1
|
|
if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then
|
|
echo -e "${GREEN}✓ $name restarted (PID: $(cat "$PID_DIR/$name.pid"))${NC}"
|
|
# A restart that survives is a success; reset its budget.
|
|
rm -f "$RESTART_STATE_DIR/$name"
|
|
return 0
|
|
else
|
|
echo -e "${RED}✗ $name failed to restart (check $LOG_DIR/$name.log)${NC}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Keep script running and monitor processes
|
|
while true; do
|
|
# Check if all processes are still running
|
|
all_running=true
|
|
for pidfile in "$PID_DIR"/*.pid; do
|
|
if [ -f "$pidfile" ]; then
|
|
pid=$(cat "$pidfile")
|
|
if ! kill -0 "$pid" 2>/dev/null; then
|
|
name=$(basename "$pidfile" .pid)
|
|
echo -e "${RED}Warning: $name (PID: $pid) has stopped unexpectedly${NC}"
|
|
all_running=false
|
|
restart_service "$name"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$all_running" = false ]; then
|
|
echo -e "${YELLOW}Some services were restarted. Check logs in $LOG_DIR${NC}"
|
|
fi
|
|
|
|
sleep 10
|
|
done
|