From d4fe8614c2d362d51630f972d39928ecf379e015 Mon Sep 17 00:00:00 2001 From: Alpinux Date: Wed, 6 May 2026 13:39:14 +0200 Subject: [PATCH] =?UTF-8?q?feat(cache):=20IP=E2=86=92ASN=20cache=20Postgre?= =?UTF-8?q?SQL=20+=20fix=20deploy=20venv=20permissions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - psycopg2-binary ajouté aux dépendances - Cache ip-api.com dans table ip_asn_cache (PostgreSQL, TTL 30 j) - ThreadedConnectionPool partagé par worker, schéma auto-créé au démarrage - Graceful degradation si DATABASE_URL absent - deploy-app.sh : pip tourne sous static-cdn (sudo -u) pour respecter les droits Co-Authored-By: Claude Sonnet 4.6 --- app/CHANGELOG.md | 9 ++++ app/VERSION | 2 +- app/app.py | 108 ++++++++++++++++++++++++++++++++++++++++-- app/requirements.txt | 1 + scripts/deploy-app.sh | 6 +-- 5 files changed, 117 insertions(+), 9 deletions(-) diff --git a/app/CHANGELOG.md b/app/CHANGELOG.md index 4e99fa9..cab8096 100644 --- a/app/CHANGELOG.md +++ b/app/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog — Alpinux Static +## [1.6.1] — 2026-05-06 + +### Modifié +- Cache IP→ASN migré de la mémoire process vers **PostgreSQL** (`ip_asn_cache`) — TTL 30 jours, partagé entre tous les workers gunicorn, pas de lock SQLite +- Pool de connexions `ThreadedConnectionPool` (1–3 conns par worker), schéma créé automatiquement au premier démarrage +- Graceful degradation : si `DATABASE_URL` absent ou PostgreSQL indisponible, l'app appelle ip-api.com sans cache + +--- + ## [1.6.0] — 2026-05-06 ### Ajouté diff --git a/app/VERSION b/app/VERSION index dc1e644..9c6d629 100644 --- a/app/VERSION +++ b/app/VERSION @@ -1 +1 @@ -1.6.0 +1.6.1 diff --git a/app/app.py b/app/app.py index a1e9415..d44703b 100644 --- a/app/app.py +++ b/app/app.py @@ -9,10 +9,18 @@ import subprocess import threading import time import urllib.request +from contextlib import contextmanager from pathlib import Path from datetime import datetime, timedelta from urllib.parse import urlencode + +try: + import psycopg2 + import psycopg2.pool as _pg_pool_lib + _HAS_PG = True +except ImportError: + _HAS_PG = False from PIL import Image from flask import (Flask, redirect, url_for, session, request, @@ -71,6 +79,10 @@ _BANNED_CACHE: tuple = (set(), []) _BANNED_CACHE_TS: float = 0 _BANNED_CACHE_TTL = 60 # 1 min +_pg_dsn = os.environ.get("DATABASE_URL", "") +_pg_pool_obj = None +_pg_schema_ok = False + _HIDDEN = frozenset({ ".git", "scripts", "app", @@ -616,22 +628,108 @@ def _ip_is_banned(ip: str) -> bool: except ValueError: return False +def _pg_init_pool() -> bool: + global _pg_pool_obj + if _pg_pool_obj is not None: + return True + if not _HAS_PG or not _pg_dsn: + return False + try: + _pg_pool_obj = _pg_pool_lib.ThreadedConnectionPool(1, 3, _pg_dsn) + return True + except Exception: + return False + +def _pg_ensure_schema(conn) -> None: + global _pg_schema_ok + if _pg_schema_ok: + return + with conn.cursor() as cur: + cur.execute(""" + CREATE TABLE IF NOT EXISTS ip_asn_cache ( + ip TEXT PRIMARY KEY, + asn TEXT NOT NULL DEFAULT '', + name TEXT NOT NULL DEFAULT '', + country TEXT NOT NULL DEFAULT '', + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """) + conn.commit() + _pg_schema_ok = True + +@contextmanager +def _pg(): + """Yields a psycopg2 connection from the pool, or None if unavailable.""" + if not _pg_init_pool(): + yield None + return + conn = None + try: + conn = _pg_pool_obj.getconn() + _pg_ensure_schema(conn) + except Exception: + if conn: + try: _pg_pool_obj.putconn(conn) + except: pass + yield None + return + try: + yield conn + finally: + try: _pg_pool_obj.putconn(conn) + except: pass + def _lookup_ip_asn(ip: str) -> dict: - """Returns {asn, name, country} via ip-api.com.""" - url = f"http://ip-api.com/json/{ip}?fields=as,org,countryCode" + """Returns {asn, name, country} — PostgreSQL cache (30 j) puis ip-api.com.""" + with _pg() as conn: + if conn: + try: + with conn.cursor() as cur: + cur.execute( + "SELECT asn, name, country FROM ip_asn_cache " + "WHERE ip = %s AND fetched_at > now() - interval '30 days'", + (ip,) + ) + row = cur.fetchone() + if row: + return {"asn": row[0], "name": row[1], "country": row[2]} + except Exception: + try: conn.rollback() + except: pass + + url = f"http://ip-api.com/json/{ip}?fields=as,countryCode" + result: dict = {"asn": "", "name": "", "country": ""} try: req = urllib.request.Request(url, headers={"User-Agent": "alpinux-static/1.0"}) with urllib.request.urlopen(req, timeout=5) as resp: d = json.loads(resp.read()) - as_field = d.get("as", "") # "AS12345 Some Name" + as_field = d.get("as", "") asn, name = "", as_field if as_field.startswith("AS"): parts = as_field.split(" ", 1) asn = parts[0][2:] name = parts[1] if len(parts) > 1 else "" - return {"asn": asn, "name": name, "country": d.get("countryCode", "")} + result = {"asn": asn, "name": name, "country": d.get("countryCode", "")} except Exception as e: - return {"asn": "", "name": "", "country": "", "error": str(e)} + result["error"] = str(e) + + with _pg() as conn: + if conn: + try: + with conn.cursor() as cur: + cur.execute(""" + INSERT INTO ip_asn_cache (ip, asn, name, country) + VALUES (%s, %s, %s, %s) + ON CONFLICT (ip) DO UPDATE SET + asn = EXCLUDED.asn, name = EXCLUDED.name, + country = EXCLUDED.country, fetched_at = now() + """, (ip, result["asn"], result["name"], result["country"])) + conn.commit() + except Exception: + try: conn.rollback() + except: pass + + return result def _lookup_as_prefixes(asn: str) -> list: """Returns IPv4 CIDRs for an ASN via RIPE Stat. Cached 30 days.""" diff --git a/app/requirements.txt b/app/requirements.txt index ec36a02..e9c7f5b 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -4,3 +4,4 @@ requests>=2.31 gunicorn>=21.0 python-dotenv>=1.0 Pillow>=10.0 +psycopg2-binary>=2.9 diff --git a/scripts/deploy-app.sh b/scripts/deploy-app.sh index b6acd36..aaabcce 100755 --- a/scripts/deploy-app.sh +++ b/scripts/deploy-app.sh @@ -58,9 +58,9 @@ echo -e "${CYAN}[2/4] Mise à jour de l'environnement Python…${RESET}" ssh "$REMOTE_HOST" bash <<'ENDSSH' set -e cd /opt/static-cdn - [ ! -d venv ] && python3 -m venv venv - venv/bin/pip install --quiet --upgrade pip - venv/bin/pip install --quiet -r requirements.txt + [ ! -d venv ] && sudo -u static-cdn python3 -m venv venv + sudo -u static-cdn venv/bin/pip install --quiet --upgrade pip + sudo -u static-cdn venv/bin/pip install --quiet -r requirements.txt ENDSSH echo -e " ${GREEN}✓ Dépendances installées${RESET}"