"""Tiered log retention. Tier 1: request/response payloads are blanked after PAYLOAD_RETENTION_DAYS — the log row stays inspectable, it just loses the bodies. Tier 2: whole log rows are deleted after LOG_RETENTION_DAYS. Both run in batches so a large backlog never locks the database against the proxy's own log writes, followed by an incremental vacuum to hand freed pages back to the filesystem. A background loop triggers this every 6 hours. """ import asyncio from datetime import datetime, timedelta, timezone from sqlalchemy import text from app import config from app.database import SessionLocal PURGE_INTERVAL_SECONDS = 6 * 3600 BATCH_SIZE = 5000 def _cutoff(days: int) -> str: """Naive-UTC timestamp string, matching how rows are stored.""" moment = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days) return moment.strftime("%Y-%m-%d %H:%M:%S") def purge_once() -> dict: stats = {"payloads_blanked": 0, "rows_deleted": 0} db = SessionLocal() try: if config.PAYLOAD_RETENTION_DAYS > 0: cutoff = _cutoff(config.PAYLOAD_RETENTION_DAYS) while True: result = db.execute(text( "UPDATE request_logs SET request_body = '', response_body = '' " "WHERE id IN (SELECT id FROM request_logs " " WHERE timestamp < :cutoff " " AND (request_body != '' OR response_body != '') " " LIMIT :batch)" ), {"cutoff": cutoff, "batch": BATCH_SIZE}) db.commit() stats["payloads_blanked"] += result.rowcount if result.rowcount < BATCH_SIZE: break if config.LOG_RETENTION_DAYS > 0: cutoff = _cutoff(config.LOG_RETENTION_DAYS) while True: result = db.execute(text( "DELETE FROM request_logs WHERE id IN " "(SELECT id FROM request_logs WHERE timestamp < :cutoff LIMIT :batch)" ), {"cutoff": cutoff, "batch": BATCH_SIZE}) db.commit() stats["rows_deleted"] += result.rowcount if result.rowcount < BATCH_SIZE: break if stats["rows_deleted"] or stats["payloads_blanked"]: db.execute(text("PRAGMA incremental_vacuum")) db.commit() finally: db.close() return stats async def retention_loop() -> None: while True: try: stats = await asyncio.to_thread(purge_once) if stats["payloads_blanked"] or stats["rows_deleted"]: print(f"Retention: deleted {stats['rows_deleted']} log rows, " f"blanked {stats['payloads_blanked']} payloads.") except Exception as exc: # never let a purge failure kill the loop print(f"Retention run failed: {exc}") await asyncio.sleep(PURGE_INTERVAL_SECONDS)