diff --git a/run_all.sh b/run_all.sh index 17c5c92..d999e11 100755 --- a/run_all.sh +++ b/run_all.sh @@ -145,6 +145,8 @@ cleanup() { for script in scraper post_results get_votes get_user_list; do pkill -f "python3 $script.py" 2>/dev/null || true done + + rm -rf "${RESTART_STATE_DIR:-}" 2>/dev/null || true echo -e "${GREEN}All services stopped.${NC}" exit 0 @@ -158,18 +160,48 @@ start_go_service() { local name=$1 local dir=$2 local binary=$3 + local rebuild=0 - echo -e "${BLUE}Building $name...${NC}" - if [ "$OS" = "freebsd" ]; then - (cd "$dir" && GOOS=freebsd go build -o "$binary") || { - echo -e "${RED}Failed to build $name for FreeBSD${NC}" - return 1 - } + if [ -f "$dir/$binary" ]; then + echo -e "${GREEN}Found existing $name binary ($dir/$binary), checking freshness...${NC}" + # Only skip the build if the binary matches the current OS target and + # is newer than every Go source file in the project (stale binaries rebuild). + local os_ok=1 + case "$OS:$binary" in + freebsd:jepfbd|freebsd:vsfbsd) os_ok=0 ;; + linux:jep|linux:vs) os_ok=0 ;; + esac + if [ "$os_ok" -ne 0 ]; then + echo -e "${YELLOW}$binary is for another platform; rebuilding ${name} for $OS${NC}" + rebuild=1 + else + local newest_src + newest_src=$( (cd "$dir" && ls -t *.go go.mod go.sum 2>/dev/null) | head -1 ) + if [ -z "$newest_src" ] || [ "$dir/$binary" -nt "$dir/$newest_src" ]; then + echo -e "${GREEN}✓ $name binary is current; skipping build${NC}" + rebuild=0 + else + echo -e "${YELLOW}$name binary is older than ${newest_src}; rebuilding${NC}" + rebuild=1 + fi + fi else - (cd "$dir" && go build -o "$binary") || { - echo -e "${RED}Failed to build $name${NC}" - return 1 - } + rebuild=1 + fi + + if [ "$rebuild" = "1" ]; then + echo -e "${BLUE}Building $name...${NC}" + if [ "$OS" = "freebsd" ]; then + (cd "$dir" && GOOS=freebsd go build -o "$binary") || { + echo -e "${RED}Failed to build $name for FreeBSD${NC}" + return 1 + } + else + (cd "$dir" && go build -o "$binary") || { + echo -e "${RED}Failed to build $name${NC}" + return 1 + } + fi fi echo -e "${GREEN}Starting $name...${NC}" @@ -327,6 +359,60 @@ echo "" echo -e "${YELLOW}Press Ctrl+C to stop all services${NC}" echo "" +# Minimum free memory (MB) required before an auto-restart is attempted. +# Prevents making an out-of-memory situation worse by launching new processes. +RESTART_MEM_MIN_MB=${RESTART_MEM_MIN_MB:-128} +# Max restart attempts per service within the restart window before backing off. +MAX_RESTARTS_PER_WINDOW=${MAX_RESTARTS_PER_WINDOW:-5} +RESTART_WINDOW_SECS=${RESTART_WINDOW_SECS:-300} + +# Track restart attempts per service (name -> timestamp:count) +RESTART_STATE_DIR="$PID_DIR/restarts" +mkdir -p "$RESTART_STATE_DIR" + +free_mem_mb() { + # Returns approximate free memory in MB (0 if unavailable -> allow restart) + case "$OS" in + freebsd) + local pages free_kb + pages=$(sysctl -n vm.stats.vm.v_free_count 2>/dev/null) || return 0 + free_kb=$(( pages * 4096 / 1024 )) + echo "$(( free_kb / 1024 ))" + ;; + linux) + awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || return 0 + ;; + *) + return 0 + ;; + esac +} + +# Register a restart attempt; returns 0 if allowed, 1 if the service is +# restarting too often (crash loop) and should be left down for now. +check_restart_budget() { + local name=$1 now + now=$( date +%s ) + local state_file="$RESTART_STATE_DIR/$name" + # shellcheck disable=SC2086 + local count=0 first="$now" + if [ -f "$state_file" ]; then + read -r first count < "$state_file" + fi + # Expire the window once it has passed. + if [ $(( now - first )) -ge "$RESTART_WINDOW_SECS" ]; then + count=0 + first=$now + fi + if [ "$count" -ge "$MAX_RESTARTS_PER_WINDOW" ]; then + echo -e "${YELLOW}✗ $name: hit max restarts ($MAX_RESTARTS_PER_WINDOW in ${RESTART_WINDOW_SECS}s), leaving down. Clear $RESTART_STATE_DIR/$name to reset.${NC}" + return 1 + fi + count=$(( count + 1 )) + echo "$first $count" > "$state_file" + return 0 +} + # Function to restart a dead service from its saved relaunch command restart_service() { local name=$1 @@ -337,6 +423,18 @@ restart_service() { return 1 fi + if ! check_restart_budget "$name"; then + return 1 + fi + + # Do not relaunch more processes if the machine is nearly out of memory. + local free_mb + free_mb=$(free_mem_mb) + if [ -n "$free_mb" ] && [ "$free_mb" -lt "$RESTART_MEM_MIN_MB" ]; then + echo -e "${YELLOW}✗ $name: only ${free_mb}MB free (need >= ${RESTART_MEM_MIN_MB}MB), skipping restart to avoid OOM.${NC}" + return 1 + fi + echo -e "${YELLOW}Restarting $name...${NC}" rm -f "$PID_DIR/$name.pid" # shellcheck disable=SC2046 @@ -346,6 +444,8 @@ restart_service() { sleep 1 if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then echo -e "${GREEN}✓ $name restarted (PID: $(cat "$PID_DIR/$name.pid"))${NC}" + # A restart that survives is a success; reset its budget. + rm -f "$RESTART_STATE_DIR/$name" return 0 else echo -e "${RED}✗ $name failed to restart (check $LOG_DIR/$name.log)${NC}"