tweaks

dbutil.py

@@ -0,0 +1,31 @@
+import sqlite3
+import time
+
+
+LOCKED_MSGS = ("database is locked", "database is busy")
+
+
+def connect(db_path, timeout=30):
+    """Open a SQLite connection hardened against cross-process contention.
+
+    Enables WAL mode and a busy timeout so that the multiple bot processes
+    sharing a database file wait for each other instead of raising
+    ``OperationalError: database is locked`` immediately.
+    """
+    conn = sqlite3.connect(db_path, timeout=timeout)
+    conn.execute("PRAGMA busy_timeout=%d" % (timeout * 1000))
+    conn.execute("PRAGMA journal_mode=WAL")
+    return conn
+
+
+def commit(conn, retries=5, backoff=0.2):
+    """Commit, retrying briefly when the database is locked by another writer."""
+    for attempt in range(retries):
+        try:
+            conn.commit()
+            return
+        except sqlite3.OperationalError as e:
+            if str(e).lower() in LOCKED_MSGS and attempt < retries - 1:
+                time.sleep(backoff * (attempt + 1))
+                continue
+            raise

get_user_list.py

@@ -11,6 +11,7 @@ import time
 import sys
 from nio import AsyncClient, LoginResponse, RoomMessagesResponse, SyncResponse
 from room import Room
+import dbutil
 
 
 class UserListManager:
@@ -29,7 +30,7 @@ class UserListManager:
     
     def init_user_db(self):
         """Initialize the user list database."""
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         c.execute('''
             CREATE TABLE IF NOT EXISTS users (
@@ -67,7 +68,7 @@ class UserListManager:
         if 'most_recent_post' not in cols:
             c.execute('ALTER TABLE fetch_log ADD COLUMN most_recent_post INTEGER DEFAULT 0')
         
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
     
     async def get_room_members(self):
@@ -163,7 +164,7 @@ class UserListManager:
             print("No users to write to database")
             return
         
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         
         current_time = int(time.time())
@@ -209,13 +210,13 @@ class UserListManager:
             VALUES (?, ?, ?, ?)
         ''', (current_time, len(self.users), datetime.now().isoformat(), getattr(self, 'most_recent_post', 0)))
         
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
         print(f"Wrote {len(self.users)} users to {self.db_path}")
     
     def read_users_from_db(self):
         """Read all users from the database."""
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         c.execute('SELECT user_id, display_name, last_seen, last_updated, num_posts, last_active FROM users ORDER BY num_posts DESC, display_name')
         users = []

get_votes.py

@@ -56,8 +56,8 @@ async def main():
 
 def init_getvotes_table(room):
     """Initialize the getvotes tracking table in the database."""
-    import sqlite3
+    import dbutil
-    conn = sqlite3.connect(room.db_path)
+    conn = dbutil.connect(room.db_path)
     c = conn.cursor()
     c.execute('''
         CREATE TABLE IF NOT EXISTS getvotes (
@@ -70,14 +70,14 @@ def init_getvotes_table(room):
             UNIQUE(sender, timestamp)
         )
     ''')
-    conn.commit()
+    dbutil.commit(conn)
     conn.close()
 
 
 def get_last_getvotes_timestamp(room):
     """Get the timestamp of the last !getvotes command for this room."""
-    import sqlite3
+    import dbutil
-    conn = sqlite3.connect(room.db_path)
+    conn = dbutil.connect(room.db_path)
     c = conn.cursor()
     c.execute('''
         SELECT MAX(timestamp) FROM getvotes
@@ -91,10 +91,10 @@ def get_last_getvotes_timestamp(room):
 
 def record_getvotes(room, sender, timestamp, event_id=None):
     """Record a !getvotes detection; returns True if newly recorded (not duplicate)."""
-    import sqlite3
+    import dbutil
     import datetime
     
-    conn = sqlite3.connect(room.db_path)
+    conn = dbutil.connect(room.db_path)
     c = conn.cursor()
     dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds')
     c.execute('''
@@ -102,7 +102,7 @@ def record_getvotes(room, sender, timestamp, event_id=None):
         VALUES (?, ?, ?, ?)
     ''', (event_id, sender, int(timestamp), dt_str))
     inserted = c.rowcount == 1
-    conn.commit()
+    dbutil.commit(conn)
     conn.close()
     return inserted
 
@@ -154,10 +154,10 @@ def format_vote_results(room):
         return "❌ No vote data available yet. The voting system may not have received any votes."
     
     try:
-        import sqlite3
+        import dbutil
         
         # Read votes from database
-        conn = sqlite3.connect(vote_db)
+        conn = dbutil.connect(vote_db)
         c = conn.cursor()
         
         # Get all votes grouped by submission

post_results.py

@@ -1,17 +1,17 @@
 import yaml
 import re
 import asyncio
-import sqlite3
 import sys
 from nio import AsyncClient, LoginResponse, RoomSendResponse, RoomMessageText, SyncResponse
 import time
 from post import Post 
 from room import Room
+import dbutil
 
 
 def get_last_getvotes_timestamp(room):
     """Get the timestamp of the last !getvotes command for this room."""
-    conn = sqlite3.connect(room.db_path)
+    conn = dbutil.connect(room.db_path)
     c = conn.cursor()
     # Check if getvotes table exists
     c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='getvotes'")

room.py

@@ -13,6 +13,7 @@ import asyncio
 from nio import AsyncClient, LoginResponse, RoomMessageText, SyncResponse, RoomSendResponse
 from post import Post
 import requests
+import dbutil
 
 default_search_terms = ["!suggest", "!Suggest", "!suggestion", "!Suggestion", "SUGGEST", "!sug"]
 
@@ -61,7 +62,7 @@ class Room:
         return f"suggestions_{safe}.db"
 
     def init_db(self):
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         c.execute('''
             CREATE TABLE IF NOT EXISTS suggestions (
@@ -137,7 +138,7 @@ class Room:
                 UNIQUE(sender, timestamp)
             )
         ''')
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
 
     async def login_to_matrix(self):
@@ -297,7 +298,7 @@ class Room:
 
 
     def write_to_db(self):
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
 
         self.newly_inserted_posts = []
@@ -311,12 +312,12 @@ class Room:
             except sqlite3.IntegrityError:
                 continue
 
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
 
     def update_last_run(self):
         room_id = self.config.room_id
-        conn = sqlite3.connect("last_run.db")
+        conn = dbutil.connect("last_run.db")
         c = conn.cursor()
         c.execute('''
             CREATE TABLE IF NOT EXISTS last_run (
@@ -333,12 +334,12 @@ class Room:
         now = int(_time.time())
         now_str = datetime.datetime.now().isoformat(sep=' ', timespec='seconds')
         c.execute('INSERT OR REPLACE INTO last_run (room_id, timestamp, datetime) VALUES (?, ?, ?)', (room_id, now, now_str))
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
 
     def get_last_run(self):
         room_id = self.config.room_id
-        conn = sqlite3.connect("last_run.db")
+        conn = dbutil.connect("last_run.db")
         c = conn.cursor()
         c.execute('''
             CREATE TABLE IF NOT EXISTS last_run (
@@ -362,7 +363,7 @@ class Room:
 
     # New: record a !gettitles detection; returns True if newly recorded (not duplicate)
     def record_gettitles(self, sender, timestamp, event_id=None):
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds')
         c.execute('''
@@ -370,14 +371,14 @@ class Room:
             VALUES (?, ?, ?, ?)
         ''', (event_id, sender, int(timestamp), dt_str))
         inserted = c.rowcount == 1
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
         return inserted
 
     # New: record a !mayhem detection; returns True if newly recorded (not duplicate)
     def record_mayhem(self, sender, timestamp, event_id=None):
         """Record a !mayhem detection; returns True if newly recorded (not duplicate)"""
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds')
         c.execute('''
@@ -385,14 +386,14 @@ class Room:
             VALUES (?, ?, ?, ?)
         ''', (event_id, sender, int(timestamp), dt_str))
         inserted = c.rowcount == 1
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
         return inserted
 
     # New: record a !countvotes detection; returns True if newly recorded (not duplicate)
     def record_countvotes(self, sender, timestamp, event_id=None):
         """Record a !countvotes detection; returns True if newly recorded (not duplicate)"""
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds')
         c.execute('''
@@ -400,14 +401,14 @@ class Room:
             VALUES (?, ?, ?, ?)
         ''', (event_id, sender, int(timestamp), dt_str))
         inserted = c.rowcount == 1
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
         return inserted
 
     # New: record a !voteurl detection; returns True if newly recorded (not duplicate)
     def record_voteurl(self, sender, timestamp, event_id=None):
         """Record a !voteurl detection; returns True if newly recorded (not duplicate)"""
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds')
         c.execute('''
@@ -415,7 +416,7 @@ class Room:
             VALUES (?, ?, ?, ?)
         ''', (event_id, sender, int(timestamp), dt_str))
         inserted = c.rowcount == 1
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
         return inserted
 
@@ -611,7 +612,7 @@ class Room:
         self.formatted_sugs = "\n".join(lines)
 
     def read_db(self, since_timestamp=None, min_timestamp=None, only_unposted=False):
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         query = "SELECT user, display_name, message, timestamp, age, is_posted FROM suggestions"
         params = []
@@ -633,7 +634,7 @@ class Room:
         self.posts = [Post(poster=row[0], display_name=row[1], content=row[2], time=row[3], age=row[4], is_posted=bool(row[5])) for row in rows]
 
     def get_last_post(self):
-        conn = sqlite3.connect("last_post.db")
+        conn = dbutil.connect("last_post.db")
         c = conn.cursor()
         c.execute('''
             CREATE TABLE IF NOT EXISTS last_post (
@@ -654,7 +655,7 @@ class Room:
         return None
 
     def update_last_post(self):
-        conn = sqlite3.connect("last_post.db")
+        conn = dbutil.connect("last_post.db")
         c = conn.cursor()
         c.execute('''
             CREATE TABLE IF NOT EXISTS last_post (
@@ -672,7 +673,7 @@ class Room:
         now_str = datetime.datetime.now().isoformat(sep=' ', timespec='seconds')
         c.execute('INSERT OR REPLACE INTO last_post (room_id, timestamp, datetime) VALUES (?, ?, ?)',
                  (self.config.room_id, now, now_str))
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
 
     async def post_suggestions(self):
@@ -787,14 +788,14 @@ class Room:
         if posts is None:
             posts = self.posts
 
-        conn = sqlite3.connect(self.db_path)
+        conn = dbutil.connect(self.db_path)
         c = conn.cursor()
         for post in posts:
             c.execute(
                 "UPDATE suggestions SET is_posted = 1 WHERE user = ? AND message = ? AND timestamp = ?",
                 (post.poster, post.content, post.time)
             )
-        conn.commit()
+        dbutil.commit(conn)
         conn.close()
 
     async def close(self):

run_all.sh

@@ -133,6 +133,7 @@ cleanup() {
                     kill "$pid" 2>/dev/null || true
                 fi
                 rm -f "$pidfile"
+                rm -f "$PID_DIR/$name.cmd"
             fi
         done
     fi
@@ -174,6 +175,7 @@ start_go_service() {
     echo -e "${GREEN}Starting $name...${NC}"
     (cd "$dir" && ./"$binary" > "$LOG_DIR/$name.log" 2>&1) &
     echo $! > "$PID_DIR/$name.pid"
+    echo "(cd \"$dir\" && .\"/$binary\" > \"$LOG_DIR/$name.log\" 2>&1) &" > "$PID_DIR/$name.cmd"
     
     # Brief wait to check if it started successfully
     sleep 1
@@ -203,6 +205,7 @@ start_python_service() {
     # Activate virtual environment and run script
     (source .venv/bin/activate && python3 "$script" $config_arg > "$LOG_DIR/$label.log" 2>&1) &
     echo $! > "$PID_DIR/$label.pid"
+    echo "(source .venv/bin/activate && python3 \"$script\" $config_arg > \"$LOG_DIR/$label.log\" 2>&1) &" > "$PID_DIR/$label.cmd"
     
     # Brief wait to check if it started successfully
     sleep 1
@@ -324,6 +327,32 @@ echo ""
 echo -e "${YELLOW}Press Ctrl+C to stop all services${NC}"
 echo ""
 
+# Function to restart a dead service from its saved relaunch command
+restart_service() {
+    local name=$1
+    local cmd_file="$PID_DIR/$name.cmd"
+    
+    if [ ! -f "$cmd_file" ]; then
+        echo -e "${RED}✗ $name: no restart command stored, cannot auto-restart${NC}"
+        return 1
+    fi
+    
+    echo -e "${YELLOW}Restarting $name...${NC}"
+    rm -f "$PID_DIR/$name.pid"
+    # shellcheck disable=SC2046
+    eval "$(cat "$cmd_file")"
+    echo $! > "$PID_DIR/$name.pid"
+    
+    sleep 1
+    if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then
+        echo -e "${GREEN}✓ $name restarted (PID: $(cat "$PID_DIR/$name.pid"))${NC}"
+        return 0
+    else
+        echo -e "${RED}✗ $name failed to restart (check $LOG_DIR/$name.log)${NC}"
+        return 1
+    fi
+}
+
 # Keep script running and monitor processes
 while true; do
     # Check if all processes are still running
@@ -335,12 +364,13 @@ while true; do
                 name=$(basename "$pidfile" .pid)
                 echo -e "${RED}Warning: $name (PID: $pid) has stopped unexpectedly${NC}"
                 all_running=false
+                restart_service "$name"
             fi
         fi
     done
     
     if [ "$all_running" = false ]; then
-        echo -e "${YELLOW}Some services have stopped. Check logs in $LOG_DIR${NC}"
+        echo -e "${YELLOW}Some services were restarted. Check logs in $LOG_DIR${NC}"
     fi
     
     sleep 10