From 9cc7093c48728cea1c795092a877c14756dcead7 Mon Sep 17 00:00:00 2001 From: nolan Date: Wed, 12 Aug 2026 18:22:26 -0500 Subject: [PATCH] tweaks --- dbutil.py | 31 +++++++++++++++++++++++++++++++ get_user_list.py | 11 ++++++----- get_votes.py | 20 ++++++++++---------- post_results.py | 4 ++-- room.py | 43 ++++++++++++++++++++++--------------------- run_all.sh | 32 +++++++++++++++++++++++++++++++- 6 files changed, 102 insertions(+), 39 deletions(-) create mode 100644 dbutil.py diff --git a/dbutil.py b/dbutil.py new file mode 100644 index 0000000..0893b3e --- /dev/null +++ b/dbutil.py @@ -0,0 +1,31 @@ +import sqlite3 +import time + + +LOCKED_MSGS = ("database is locked", "database is busy") + + +def connect(db_path, timeout=30): + """Open a SQLite connection hardened against cross-process contention. + + Enables WAL mode and a busy timeout so that the multiple bot processes + sharing a database file wait for each other instead of raising + ``OperationalError: database is locked`` immediately. + """ + conn = sqlite3.connect(db_path, timeout=timeout) + conn.execute("PRAGMA busy_timeout=%d" % (timeout * 1000)) + conn.execute("PRAGMA journal_mode=WAL") + return conn + + +def commit(conn, retries=5, backoff=0.2): + """Commit, retrying briefly when the database is locked by another writer.""" + for attempt in range(retries): + try: + conn.commit() + return + except sqlite3.OperationalError as e: + if str(e).lower() in LOCKED_MSGS and attempt < retries - 1: + time.sleep(backoff * (attempt + 1)) + continue + raise \ No newline at end of file diff --git a/get_user_list.py b/get_user_list.py index 2a48bac..43abb81 100755 --- a/get_user_list.py +++ b/get_user_list.py @@ -11,6 +11,7 @@ import time import sys from nio import AsyncClient, LoginResponse, RoomMessagesResponse, SyncResponse from room import Room +import dbutil class UserListManager: @@ -29,7 +30,7 @@ class UserListManager: def init_user_db(self): """Initialize the user list database.""" - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS users ( @@ -67,7 +68,7 @@ class UserListManager: if 'most_recent_post' not in cols: c.execute('ALTER TABLE fetch_log ADD COLUMN most_recent_post INTEGER DEFAULT 0') - conn.commit() + dbutil.commit(conn) conn.close() async def get_room_members(self): @@ -163,7 +164,7 @@ class UserListManager: print("No users to write to database") return - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() current_time = int(time.time()) @@ -209,13 +210,13 @@ class UserListManager: VALUES (?, ?, ?, ?) ''', (current_time, len(self.users), datetime.now().isoformat(), getattr(self, 'most_recent_post', 0))) - conn.commit() + dbutil.commit(conn) conn.close() print(f"Wrote {len(self.users)} users to {self.db_path}") def read_users_from_db(self): """Read all users from the database.""" - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() c.execute('SELECT user_id, display_name, last_seen, last_updated, num_posts, last_active FROM users ORDER BY num_posts DESC, display_name') users = [] diff --git a/get_votes.py b/get_votes.py index 7bfcd2a..9fb625a 100755 --- a/get_votes.py +++ b/get_votes.py @@ -56,8 +56,8 @@ async def main(): def init_getvotes_table(room): """Initialize the getvotes tracking table in the database.""" - import sqlite3 - conn = sqlite3.connect(room.db_path) + import dbutil + conn = dbutil.connect(room.db_path) c = conn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS getvotes ( @@ -70,14 +70,14 @@ def init_getvotes_table(room): UNIQUE(sender, timestamp) ) ''') - conn.commit() + dbutil.commit(conn) conn.close() def get_last_getvotes_timestamp(room): """Get the timestamp of the last !getvotes command for this room.""" - import sqlite3 - conn = sqlite3.connect(room.db_path) + import dbutil + conn = dbutil.connect(room.db_path) c = conn.cursor() c.execute(''' SELECT MAX(timestamp) FROM getvotes @@ -91,10 +91,10 @@ def get_last_getvotes_timestamp(room): def record_getvotes(room, sender, timestamp, event_id=None): """Record a !getvotes detection; returns True if newly recorded (not duplicate).""" - import sqlite3 + import dbutil import datetime - conn = sqlite3.connect(room.db_path) + conn = dbutil.connect(room.db_path) c = conn.cursor() dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds') c.execute(''' @@ -102,7 +102,7 @@ def record_getvotes(room, sender, timestamp, event_id=None): VALUES (?, ?, ?, ?) ''', (event_id, sender, int(timestamp), dt_str)) inserted = c.rowcount == 1 - conn.commit() + dbutil.commit(conn) conn.close() return inserted @@ -154,10 +154,10 @@ def format_vote_results(room): return "❌ No vote data available yet. The voting system may not have received any votes." try: - import sqlite3 + import dbutil # Read votes from database - conn = sqlite3.connect(vote_db) + conn = dbutil.connect(vote_db) c = conn.cursor() # Get all votes grouped by submission diff --git a/post_results.py b/post_results.py index 175771a..4f38fad 100755 --- a/post_results.py +++ b/post_results.py @@ -1,17 +1,17 @@ import yaml import re import asyncio -import sqlite3 import sys from nio import AsyncClient, LoginResponse, RoomSendResponse, RoomMessageText, SyncResponse import time from post import Post from room import Room +import dbutil def get_last_getvotes_timestamp(room): """Get the timestamp of the last !getvotes command for this room.""" - conn = sqlite3.connect(room.db_path) + conn = dbutil.connect(room.db_path) c = conn.cursor() # Check if getvotes table exists c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='getvotes'") diff --git a/room.py b/room.py index a2a255f..35a261a 100755 --- a/room.py +++ b/room.py @@ -13,6 +13,7 @@ import asyncio from nio import AsyncClient, LoginResponse, RoomMessageText, SyncResponse, RoomSendResponse from post import Post import requests +import dbutil default_search_terms = ["!suggest", "!Suggest", "!suggestion", "!Suggestion", "SUGGEST", "!sug"] @@ -61,7 +62,7 @@ class Room: return f"suggestions_{safe}.db" def init_db(self): - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS suggestions ( @@ -137,7 +138,7 @@ class Room: UNIQUE(sender, timestamp) ) ''') - conn.commit() + dbutil.commit(conn) conn.close() async def login_to_matrix(self): @@ -297,7 +298,7 @@ class Room: def write_to_db(self): - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() self.newly_inserted_posts = [] @@ -311,12 +312,12 @@ class Room: except sqlite3.IntegrityError: continue - conn.commit() + dbutil.commit(conn) conn.close() def update_last_run(self): room_id = self.config.room_id - conn = sqlite3.connect("last_run.db") + conn = dbutil.connect("last_run.db") c = conn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS last_run ( @@ -333,12 +334,12 @@ class Room: now = int(_time.time()) now_str = datetime.datetime.now().isoformat(sep=' ', timespec='seconds') c.execute('INSERT OR REPLACE INTO last_run (room_id, timestamp, datetime) VALUES (?, ?, ?)', (room_id, now, now_str)) - conn.commit() + dbutil.commit(conn) conn.close() def get_last_run(self): room_id = self.config.room_id - conn = sqlite3.connect("last_run.db") + conn = dbutil.connect("last_run.db") c = conn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS last_run ( @@ -362,7 +363,7 @@ class Room: # New: record a !gettitles detection; returns True if newly recorded (not duplicate) def record_gettitles(self, sender, timestamp, event_id=None): - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds') c.execute(''' @@ -370,14 +371,14 @@ class Room: VALUES (?, ?, ?, ?) ''', (event_id, sender, int(timestamp), dt_str)) inserted = c.rowcount == 1 - conn.commit() + dbutil.commit(conn) conn.close() return inserted # New: record a !mayhem detection; returns True if newly recorded (not duplicate) def record_mayhem(self, sender, timestamp, event_id=None): """Record a !mayhem detection; returns True if newly recorded (not duplicate)""" - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds') c.execute(''' @@ -385,14 +386,14 @@ class Room: VALUES (?, ?, ?, ?) ''', (event_id, sender, int(timestamp), dt_str)) inserted = c.rowcount == 1 - conn.commit() + dbutil.commit(conn) conn.close() return inserted # New: record a !countvotes detection; returns True if newly recorded (not duplicate) def record_countvotes(self, sender, timestamp, event_id=None): """Record a !countvotes detection; returns True if newly recorded (not duplicate)""" - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds') c.execute(''' @@ -400,14 +401,14 @@ class Room: VALUES (?, ?, ?, ?) ''', (event_id, sender, int(timestamp), dt_str)) inserted = c.rowcount == 1 - conn.commit() + dbutil.commit(conn) conn.close() return inserted # New: record a !voteurl detection; returns True if newly recorded (not duplicate) def record_voteurl(self, sender, timestamp, event_id=None): """Record a !voteurl detection; returns True if newly recorded (not duplicate)""" - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() dt_str = datetime.datetime.fromtimestamp(int(timestamp)).isoformat(sep=' ', timespec='seconds') c.execute(''' @@ -415,7 +416,7 @@ class Room: VALUES (?, ?, ?, ?) ''', (event_id, sender, int(timestamp), dt_str)) inserted = c.rowcount == 1 - conn.commit() + dbutil.commit(conn) conn.close() return inserted @@ -611,7 +612,7 @@ class Room: self.formatted_sugs = "\n".join(lines) def read_db(self, since_timestamp=None, min_timestamp=None, only_unposted=False): - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() query = "SELECT user, display_name, message, timestamp, age, is_posted FROM suggestions" params = [] @@ -633,7 +634,7 @@ class Room: self.posts = [Post(poster=row[0], display_name=row[1], content=row[2], time=row[3], age=row[4], is_posted=bool(row[5])) for row in rows] def get_last_post(self): - conn = sqlite3.connect("last_post.db") + conn = dbutil.connect("last_post.db") c = conn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS last_post ( @@ -654,7 +655,7 @@ class Room: return None def update_last_post(self): - conn = sqlite3.connect("last_post.db") + conn = dbutil.connect("last_post.db") c = conn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS last_post ( @@ -672,7 +673,7 @@ class Room: now_str = datetime.datetime.now().isoformat(sep=' ', timespec='seconds') c.execute('INSERT OR REPLACE INTO last_post (room_id, timestamp, datetime) VALUES (?, ?, ?)', (self.config.room_id, now, now_str)) - conn.commit() + dbutil.commit(conn) conn.close() async def post_suggestions(self): @@ -787,14 +788,14 @@ class Room: if posts is None: posts = self.posts - conn = sqlite3.connect(self.db_path) + conn = dbutil.connect(self.db_path) c = conn.cursor() for post in posts: c.execute( "UPDATE suggestions SET is_posted = 1 WHERE user = ? AND message = ? AND timestamp = ?", (post.poster, post.content, post.time) ) - conn.commit() + dbutil.commit(conn) conn.close() async def close(self): diff --git a/run_all.sh b/run_all.sh index 1b78c26..17c5c92 100755 --- a/run_all.sh +++ b/run_all.sh @@ -133,6 +133,7 @@ cleanup() { kill "$pid" 2>/dev/null || true fi rm -f "$pidfile" + rm -f "$PID_DIR/$name.cmd" fi done fi @@ -174,6 +175,7 @@ start_go_service() { echo -e "${GREEN}Starting $name...${NC}" (cd "$dir" && ./"$binary" > "$LOG_DIR/$name.log" 2>&1) & echo $! > "$PID_DIR/$name.pid" + echo "(cd \"$dir\" && .\"/$binary\" > \"$LOG_DIR/$name.log\" 2>&1) &" > "$PID_DIR/$name.cmd" # Brief wait to check if it started successfully sleep 1 @@ -203,6 +205,7 @@ start_python_service() { # Activate virtual environment and run script (source .venv/bin/activate && python3 "$script" $config_arg > "$LOG_DIR/$label.log" 2>&1) & echo $! > "$PID_DIR/$label.pid" + echo "(source .venv/bin/activate && python3 \"$script\" $config_arg > \"$LOG_DIR/$label.log\" 2>&1) &" > "$PID_DIR/$label.cmd" # Brief wait to check if it started successfully sleep 1 @@ -324,6 +327,32 @@ echo "" echo -e "${YELLOW}Press Ctrl+C to stop all services${NC}" echo "" +# Function to restart a dead service from its saved relaunch command +restart_service() { + local name=$1 + local cmd_file="$PID_DIR/$name.cmd" + + if [ ! -f "$cmd_file" ]; then + echo -e "${RED}✗ $name: no restart command stored, cannot auto-restart${NC}" + return 1 + fi + + echo -e "${YELLOW}Restarting $name...${NC}" + rm -f "$PID_DIR/$name.pid" + # shellcheck disable=SC2046 + eval "$(cat "$cmd_file")" + echo $! > "$PID_DIR/$name.pid" + + sleep 1 + if kill -0 $(cat "$PID_DIR/$name.pid") 2>/dev/null; then + echo -e "${GREEN}✓ $name restarted (PID: $(cat "$PID_DIR/$name.pid"))${NC}" + return 0 + else + echo -e "${RED}✗ $name failed to restart (check $LOG_DIR/$name.log)${NC}" + return 1 + fi +} + # Keep script running and monitor processes while true; do # Check if all processes are still running @@ -335,12 +364,13 @@ while true; do name=$(basename "$pidfile" .pid) echo -e "${RED}Warning: $name (PID: $pid) has stopped unexpectedly${NC}" all_running=false + restart_service "$name" fi fi done if [ "$all_running" = false ]; then - echo -e "${YELLOW}Some services have stopped. Check logs in $LOG_DIR${NC}" + echo -e "${YELLOW}Some services were restarted. Check logs in $LOG_DIR${NC}" fi sleep 10