Stop rebuilding every time it gets restarted

run_all.sh

@@ -146,6 +146,8 @@ cleanup() {
         pkill -f "python3 $script.py" 2>/dev/null || true
     done
 
+    rm -rf "${RESTART_STATE_DIR:-}" 2>/dev/null || true
+    
     echo -e "${GREEN}All services stopped.${NC}"
     exit 0
 }
@@ -158,7 +160,36 @@ start_go_service() {
     local name=$1
     local dir=$2
     local binary=$3
+    local rebuild=0
     
+    if [ -f "$dir/$binary" ]; then
+        echo -e "${GREEN}Found existing $name binary ($dir/$binary), checking freshness...${NC}"
+        # Only skip the build if the binary matches the current OS target and
+        # is newer than every Go source file in the project (stale binaries rebuild).
+        local os_ok=1
+        case "$OS:$binary" in
+            freebsd:jepfbd|freebsd:vsfbsd)   os_ok=0 ;;
+            linux:jep|linux:vs)              os_ok=0 ;;
+        esac
+        if [ "$os_ok" -ne 0 ]; then
+            echo -e "${YELLOW}$binary is for another platform; rebuilding ${name} for $OS${NC}"
+            rebuild=1
+        else
+            local newest_src
+            newest_src=$( (cd "$dir" && ls -t *.go go.mod go.sum 2>/dev/null) | head -1 )
+            if [ -z "$newest_src" ] || [ "$dir/$binary" -nt "$dir/$newest_src" ]; then
+                echo -e "${GREEN}✓ $name binary is current; skipping build${NC}"
+                rebuild=0
+            else
+                echo -e "${YELLOW}$name binary is older than ${newest_src}; rebuilding${NC}"
+                rebuild=1
+            fi
+        fi
+    else
+        rebuild=1
+    fi
+
+    if [ "$rebuild" = "1" ]; then
         echo -e "${BLUE}Building $name...${NC}"
         if [ "$OS" = "freebsd" ]; then
             (cd "$dir" && GOOS=freebsd go build -o "$binary") || {
@@ -171,6 +202,7 @@ start_go_service() {
                 return 1
             }
         fi
+    fi
     
     echo -e "${GREEN}Starting $name...${NC}"
     (cd "$dir" && ./"$binary" > "$LOG_DIR/$name.log" 2>&1) &
@@ -327,6 +359,60 @@ echo ""
 echo -e "${YELLOW}Press Ctrl+C to stop all services${NC}"
 echo ""
 
+# Minimum free memory (MB) required before an auto-restart is attempted.
+# Prevents making an out-of-memory situation worse by launching new processes.
+RESTART_MEM_MIN_MB=${RESTART_MEM_MIN_MB:-128}
+# Max restart attempts per service within the restart window before backing off.
+MAX_RESTARTS_PER_WINDOW=${MAX_RESTARTS_PER_WINDOW:-5}
+RESTART_WINDOW_SECS=${RESTART_WINDOW_SECS:-300}
+
+# Track restart attempts per service (name -> timestamp:count)
+RESTART_STATE_DIR="$PID_DIR/restarts"
+mkdir -p "$RESTART_STATE_DIR"
+
+free_mem_mb() {
+    # Returns approximate free memory in MB (0 if unavailable -> allow restart)
+    case "$OS" in
+        freebsd)
+            local pages free_kb
+            pages=$(sysctl -n vm.stats.vm.v_free_count 2>/dev/null) || return 0
+            free_kb=$(( pages * 4096 / 1024 ))
+            echo "$(( free_kb / 1024 ))"
+            ;;
+        linux)
+            awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || return 0
+            ;;
+        *)
+            return 0
+            ;;
+    esac
+}
+
+# Register a restart attempt; returns 0 if allowed, 1 if the service is
+# restarting too often (crash loop) and should be left down for now.
+check_restart_budget() {
+    local name=$1 now
+    now=$( date +%s )
+    local state_file="$RESTART_STATE_DIR/$name"
+    # shellcheck disable=SC2086
+    local count=0 first="$now"
+    if [ -f "$state_file" ]; then
+        read -r first count < "$state_file"
+    fi
+    # Expire the window once it has passed.
+    if [ $(( now - first )) -ge "$RESTART_WINDOW_SECS" ]; then
+        count=0
+        first=$now
+    fi
+    if [ "$count" -ge "$MAX_RESTARTS_PER_WINDOW" ]; then
+        echo -e "${YELLOW}✗ $name: hit max restarts ($MAX_RESTARTS_PER_WINDOW in ${RESTART_WINDOW_SECS}s), leaving down. Clear $RESTART_STATE_DIR/$name to reset.${NC}"
+        return 1
+    fi
+    count=$(( count + 1 ))
+    echo "$first $count" > "$state_file"
+    return 0
+}
+
 # Function to restart a dead service from its saved relaunch command
 restart_service() {
     local name=$1
@@ -337,6 +423,18 @@ restart_service() {
         return 1
     fi
     
+    if ! check_restart_budget "$name"; then
+        return 1
+    fi
+    
+    # Do not relaunch more processes if the machine is nearly out of memory.
+    local free_mb
+    free_mb=$(free_mem_mb)
+    if [ -n "$free_mb" ] && [ "$free_mb" -lt "$RESTART_MEM_MIN_MB" ]; then
+        echo -e "${YELLOW}✗ $name: only ${free_mb}MB free (need >= ${RESTART_MEM_MIN_MB}MB), skipping restart to avoid OOM.${NC}"
+        return 1
+    fi
+    
     echo -e "${YELLOW}Restarting $name...${NC}"
     rm -f "$PID_DIR/$name.pid"
     # shellcheck disable=SC2046
@@ -346,6 +444,8 @@ restart_service() {
     sleep 1
     if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then
         echo -e "${GREEN}✓ $name restarted (PID: $(cat "$PID_DIR/$name.pid"))${NC}"
+        # A restart that survives is a success; reset its budget.
+        rm -f "$RESTART_STATE_DIR/$name"
         return 0
     else
         echo -e "${RED}✗ $name failed to restart (check $LOG_DIR/$name.log)${NC}"