titlebot-ng-ng/scraper.py

import yaml
import re
import asyncio
import sqlite3
import sys
from nio import AsyncClient, LoginResponse, RoomSendResponse, RoomMessageText, SyncResponse
import time
from post import Post 
from room import Room 

async def main():
    config_path = sys.argv[1] if len(sys.argv) > 1 else "configs/config.yaml"
    room = Room(config_path)
    
    # Login to Matrix
    if not await room.login_to_matrix():
        print("Failed to login")
        return
    
    
    # await room.alert_post("SCRAPER LOGGED IN")
    
    # Initial sync
    assert room.client is not None
    await room.client.sync(timeout=3000)
    room.write_room_slugs()
    
    try:
        while True:
            # Get last run timestamp
            room.get_last_run()
            if room.last_run:
                print(f"Last run for this room was at UNIX timestamp {room.last_run}")
            
            # Scrape and write to database
            await room.get_convos()
            room.write_to_db()
            # Ack newly recorded suggestions with a ✅ reaction
            await room.acknowledge_new_suggestions()
            room.update_last_run()
            print(f"Stored {len(room.posts)} suggestions in database.")
            await asyncio.sleep(30)
    finally:
        # Close connection
        await room.close()

if __name__ == "__main__":
    asyncio.run(main())