297 lines
9.2 KiB
Bash
Executable File
297 lines
9.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# TitleBot-NG - Master Launch Script
|
|
# Starts all components of the TitleBot system
|
|
|
|
set -e
|
|
|
|
# Color codes for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Detect OS
|
|
detect_os() {
|
|
case "$(uname -s)" in
|
|
Linux*) echo "linux" ;;
|
|
FreeBSD*) echo "freebsd" ;;
|
|
*) echo "linux" ;;
|
|
esac
|
|
}
|
|
|
|
OS=$(detect_os)
|
|
echo -e "${BLUE}Detected OS: $OS${NC}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
# PID file directory
|
|
PID_DIR="$SCRIPT_DIR/.pids"
|
|
mkdir -p "$PID_DIR"
|
|
|
|
# Log directory
|
|
LOG_DIR="$SCRIPT_DIR/logs"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Track only services launched for this run, so stale restart commands cannot
|
|
# introduce services that are no longer configured.
|
|
service_labels=()
|
|
|
|
stop_services() {
|
|
for pidfile in "$PID_DIR"/*.pid; do
|
|
[ -f "$pidfile" ] || continue
|
|
local pid name
|
|
pid=$(cat "$pidfile")
|
|
name=$(basename "$pidfile" .pid)
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
echo -e "${YELLOW}Stopping $name (PID: $pid)${NC}"
|
|
kill "$pid" 2>/dev/null || true
|
|
fi
|
|
rm -f "$pidfile" "$PID_DIR/$name.cmd"
|
|
done
|
|
}
|
|
|
|
cleanup() {
|
|
local status=$?
|
|
trap - EXIT SIGINT SIGTERM
|
|
echo -e "\n${YELLOW}Shutting down all services...${NC}"
|
|
stop_services
|
|
if [ -n "${RESTART_STATE_DIR:-}" ]; then
|
|
rm -rf "$RESTART_STATE_DIR"
|
|
fi
|
|
echo -e "${GREEN}All services stopped.${NC}"
|
|
exit "$status"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
trap 'exit 130' SIGINT
|
|
trap 'exit 143' SIGTERM
|
|
|
|
# Function to start a Python service
|
|
start_python_service() {
|
|
local name=$1
|
|
local script=$2
|
|
local config_path=${3:-}
|
|
local config_args=()
|
|
local label="$name"
|
|
if [ -n "$config_path" ]; then
|
|
config_args+=("$config_path")
|
|
label="${name}-$(basename "$config_path")"
|
|
fi
|
|
|
|
service_labels+=("$label")
|
|
echo -e "${GREEN}Starting $label...${NC}"
|
|
|
|
# Activate virtual environment and run script
|
|
(source .venv/bin/activate && exec python3 "$script" "${config_args[@]}" > "$LOG_DIR/$label.log" 2>&1) &
|
|
echo $! > "$PID_DIR/$label.pid"
|
|
printf '(source .venv/bin/activate && exec python3 %q ' "$script" > "$PID_DIR/$label.cmd"
|
|
if [ ${#config_args[@]} -gt 0 ]; then
|
|
printf '%q ' "${config_args[@]}" >> "$PID_DIR/$label.cmd"
|
|
fi
|
|
printf '> %q 2>&1) &\n' "$LOG_DIR/$label.log" >> "$PID_DIR/$label.cmd"
|
|
|
|
# Brief wait to check if it started successfully
|
|
sleep 1
|
|
if kill -0 $(cat "$PID_DIR/$label.pid") 2>/dev/null; then
|
|
echo -e "${GREEN}✓ $label started (PID: $(cat "$PID_DIR/$label.pid"))${NC}"
|
|
return 0
|
|
else
|
|
echo -e "${RED}✗ $label failed to start${NC}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Ensure Python virtual environment exists, creating it if missing
|
|
ensure_venv() {
|
|
if [ -d ".venv" ]; then
|
|
echo -e "${GREEN}Python virtual environment found at .venv${NC}"
|
|
return 0
|
|
fi
|
|
|
|
echo -e "${YELLOW}Python virtual environment not found. Creating it...${NC}"
|
|
python3 -m venv .venv || {
|
|
echo -e "${RED}Error: Failed to create virtual environment. Is python3-venv installed?${NC}"
|
|
echo -e "${YELLOW}Try: sudo apt install python3-venv (or the equivalent for your distro)${NC}"
|
|
exit 1
|
|
}
|
|
|
|
echo -e "${GREEN}Virtual environment created. Installing dependencies...${NC}"
|
|
if source .venv/bin/activate && pip install -r requirements.txt; then
|
|
echo -e "${GREEN}Dependencies installed successfully.${NC}"
|
|
else
|
|
echo -e "${RED}Error: Failed to install Python dependencies.${NC}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
ensure_venv
|
|
|
|
# Discover active room configs (skip example/template files)
|
|
room_configs=()
|
|
for cfg in configs/*.yaml configs/*.yml; do
|
|
[ -f "$cfg" ] || continue
|
|
if [[ "$(basename "$cfg")" == "config_example.yaml" ]]; then
|
|
continue
|
|
fi
|
|
room_configs+=("$cfg")
|
|
done
|
|
if [ ${#room_configs[@]} -eq 0 ]; then
|
|
echo -e "${RED}Error: no room configs found in configs/*.yaml or configs/*.yml${NC}"
|
|
echo -e "${YELLOW}Copy configs/config_example.yaml to configs/your-room.yaml and configure it${NC}"
|
|
exit 1
|
|
fi
|
|
echo -e "${BLUE}Found ${#room_configs[@]} room config(s): ${room_configs[*]}${NC}"
|
|
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo -e "${BLUE} TitleBot-NG - Starting All Services ${NC}"
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo ""
|
|
|
|
# Stop the processes recorded by the previous run before starting room bots.
|
|
echo -e "${YELLOW}Cleaning up any existing processes...${NC}"
|
|
stop_services
|
|
|
|
echo -e "${YELLOW}Starting Matrix room bots...${NC}"
|
|
for cfg in "${room_configs[@]}"; do
|
|
start_python_service "scraper" "scraper.py" "$cfg" || true
|
|
start_python_service "post_results" "post_results.py" "$cfg" || true
|
|
done
|
|
|
|
echo ""
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo -e "${GREEN} Service startup complete ${NC}"
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo ""
|
|
echo -e "${BLUE}Service Status:${NC}"
|
|
for cfg in "${room_configs[@]}"; do
|
|
label=$(basename "$cfg")
|
|
echo -e " • Room ($label): scraper / post_results"
|
|
done
|
|
echo ""
|
|
echo -e "${BLUE}Logs are available in: $LOG_DIR${NC}"
|
|
echo -e "${BLUE}PIDs are stored in: $PID_DIR${NC}"
|
|
echo ""
|
|
echo -e "${YELLOW}Press Ctrl+C to stop all services${NC}"
|
|
echo ""
|
|
|
|
# Minimum free memory (MB) required before an auto-restart is attempted.
|
|
# Prevents making an out-of-memory situation worse by launching new processes.
|
|
RESTART_MEM_MIN_MB=${RESTART_MEM_MIN_MB:-128}
|
|
# Max restart attempts per service within the restart window before backing off.
|
|
MAX_RESTARTS_PER_WINDOW=${MAX_RESTARTS_PER_WINDOW:-5}
|
|
RESTART_WINDOW_SECS=${RESTART_WINDOW_SECS:-300}
|
|
|
|
# Track restart attempts per service (name -> timestamp:count)
|
|
RESTART_STATE_DIR="$PID_DIR/restarts"
|
|
mkdir -p "$RESTART_STATE_DIR"
|
|
|
|
free_mem_mb() {
|
|
# Returns approximate free memory in MB (0 if unavailable -> allow restart)
|
|
case "$OS" in
|
|
freebsd)
|
|
local pages free_kb
|
|
pages=$(sysctl -n vm.stats.vm.v_free_count 2>/dev/null) || return 0
|
|
free_kb=$(( pages * 4096 / 1024 ))
|
|
echo "$(( free_kb / 1024 ))"
|
|
;;
|
|
linux)
|
|
awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || return 0
|
|
;;
|
|
*)
|
|
return 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Register a restart attempt; returns 0 if allowed, 1 if the service is
|
|
# restarting too often (crash loop) and should be left down for now.
|
|
check_restart_budget() {
|
|
local name=$1 now
|
|
now=$( date +%s )
|
|
local state_file="$RESTART_STATE_DIR/$name"
|
|
# shellcheck disable=SC2086
|
|
local count=0 first="$now"
|
|
if [ -f "$state_file" ]; then
|
|
read -r first count < "$state_file"
|
|
fi
|
|
# Expire the window once it has passed.
|
|
if [ $(( now - first )) -ge "$RESTART_WINDOW_SECS" ]; then
|
|
count=0
|
|
first=$now
|
|
fi
|
|
if [ "$count" -ge "$MAX_RESTARTS_PER_WINDOW" ]; then
|
|
echo -e "${YELLOW}✗ $name: hit max restarts ($MAX_RESTARTS_PER_WINDOW in ${RESTART_WINDOW_SECS}s), leaving down. Clear $RESTART_STATE_DIR/$name to reset.${NC}"
|
|
return 1
|
|
fi
|
|
count=$(( count + 1 ))
|
|
echo "$first $count" > "$state_file"
|
|
return 0
|
|
}
|
|
|
|
# Function to restart a dead service from its saved relaunch command
|
|
restart_service() {
|
|
local name=$1
|
|
local cmd_file="$PID_DIR/$name.cmd"
|
|
|
|
if [ ! -f "$cmd_file" ]; then
|
|
echo -e "${RED}✗ $name: no restart command stored, cannot auto-restart${NC}"
|
|
return 1
|
|
fi
|
|
|
|
if ! check_restart_budget "$name"; then
|
|
return 1
|
|
fi
|
|
|
|
# Do not relaunch more processes if the machine is nearly out of memory.
|
|
local free_mb
|
|
free_mb=$(free_mem_mb)
|
|
if [ -n "$free_mb" ] && [ "$free_mb" -lt "$RESTART_MEM_MIN_MB" ]; then
|
|
echo -e "${YELLOW}✗ $name: only ${free_mb}MB free (need >= ${RESTART_MEM_MIN_MB}MB), skipping restart to avoid OOM.${NC}"
|
|
return 1
|
|
fi
|
|
|
|
echo -e "${YELLOW}Restarting $name...${NC}"
|
|
rm -f "$PID_DIR/$name.pid"
|
|
# shellcheck disable=SC2046
|
|
eval "$(cat "$cmd_file")"
|
|
echo $! > "$PID_DIR/$name.pid"
|
|
|
|
sleep 1
|
|
if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then
|
|
echo -e "${GREEN}✓ $name restarted (PID: $(cat "$PID_DIR/$name.pid"))${NC}"
|
|
# A restart that survives is a success; reset its budget.
|
|
rm -f "$RESTART_STATE_DIR/$name"
|
|
return 0
|
|
else
|
|
echo -e "${RED}✗ $name failed to restart (check $LOG_DIR/$name.log)${NC}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Keep script running and monitor processes
|
|
while true; do
|
|
# Check if all processes are still running
|
|
all_running=true
|
|
for name in "${service_labels[@]}"; do
|
|
pidfile="$PID_DIR/$name.pid"
|
|
if [ -f "$pidfile" ]; then
|
|
pid=$(cat "$pidfile")
|
|
if ! kill -0 "$pid" 2>/dev/null; then
|
|
name=$(basename "$pidfile" .pid)
|
|
echo -e "${RED}Warning: $name (PID: $pid) has stopped unexpectedly${NC}"
|
|
all_running=false
|
|
restart_service "$name" || true
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$all_running" = false ]; then
|
|
echo -e "${YELLOW}Some services were restarted. Check logs in $LOG_DIR${NC}"
|
|
fi
|
|
|
|
sleep 10
|
|
done
|