Initial commit: API gateway with admin console

- FastAPI async gateway with httpx proxying to multiple upstreams - SQLite database with SQLAlchemy ORM - Admin console: manage services, users, API keys, endpoint access - Per-key, per-endpoint granular access control - OpenAPI document sync and caching (5-minute TTL) - Request/response logging with full transaction inspection - In-memory rate limiting (per-key, fixed-window) - Tiered log retention (7d payloads, 90d rows, incremental vacuum) - TLS verification toggle per service (for self-signed certificates) - Service connectivity validation with automatic endpoint refresh - Request browser with filters and deep-link inspection - Docker setup with persistent volume - Modal forms for create/edit flows Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
2026-07-29 14:00:22 +02:00
commit 77d7a50fa9
31 changed files with 2927 additions and 0 deletions
@@ -0,0 +1,77 @@
+"""Tiered log retention.
+
+Tier 1: request/response payloads are blanked after PAYLOAD_RETENTION_DAYS —
+        the log row stays inspectable, it just loses the bodies.
+Tier 2: whole log rows are deleted after LOG_RETENTION_DAYS.
+
+Both run in batches so a large backlog never locks the database against the
+proxy's own log writes, followed by an incremental vacuum to hand freed pages
+back to the filesystem. A background loop triggers this every 6 hours.
+"""
+import asyncio
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import text
+
+from app import config
+from app.database import SessionLocal
+
+PURGE_INTERVAL_SECONDS = 6 * 3600
+BATCH_SIZE = 5000
+
+
+def _cutoff(days: int) -> str:
+    """Naive-UTC timestamp string, matching how rows are stored."""
+    moment = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=days)
+    return moment.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def purge_once() -> dict:
+    stats = {"payloads_blanked": 0, "rows_deleted": 0}
+    db = SessionLocal()
+    try:
+        if config.PAYLOAD_RETENTION_DAYS > 0:
+            cutoff = _cutoff(config.PAYLOAD_RETENTION_DAYS)
+            while True:
+                result = db.execute(text(
+                    "UPDATE request_logs SET request_body = '', response_body = '' "
+                    "WHERE id IN (SELECT id FROM request_logs "
+                    "  WHERE timestamp < :cutoff "
+                    "    AND (request_body != '' OR response_body != '') "
+                    "  LIMIT :batch)"
+                ), {"cutoff": cutoff, "batch": BATCH_SIZE})
+                db.commit()
+                stats["payloads_blanked"] += result.rowcount
+                if result.rowcount < BATCH_SIZE:
+                    break
+
+        if config.LOG_RETENTION_DAYS > 0:
+            cutoff = _cutoff(config.LOG_RETENTION_DAYS)
+            while True:
+                result = db.execute(text(
+                    "DELETE FROM request_logs WHERE id IN "
+                    "(SELECT id FROM request_logs WHERE timestamp < :cutoff LIMIT :batch)"
+                ), {"cutoff": cutoff, "batch": BATCH_SIZE})
+                db.commit()
+                stats["rows_deleted"] += result.rowcount
+                if result.rowcount < BATCH_SIZE:
+                    break
+
+        if stats["rows_deleted"] or stats["payloads_blanked"]:
+            db.execute(text("PRAGMA incremental_vacuum"))
+            db.commit()
+    finally:
+        db.close()
+    return stats
+
+
+async def retention_loop() -> None:
+    while True:
+        try:
+            stats = await asyncio.to_thread(purge_once)
+            if stats["payloads_blanked"] or stats["rows_deleted"]:
+                print(f"Retention: deleted {stats['rows_deleted']} log rows, "
+                      f"blanked {stats['payloads_blanked']} payloads.")
+        except Exception as exc:  # never let a purge failure kill the loop
+            print(f"Retention run failed: {exc}")
+        await asyncio.sleep(PURGE_INTERVAL_SECONDS)