|
|
@@ -1,5 +1,5 @@
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
async postgres wrapper over asyncpg — two-layer API (friendly verbs + raw escape hatch)
|
|
|
|
async postgres wrapper over asyncpg - two-layer API (friendly verbs + raw escape hatch)
|
|
|
|
|
|
|
|
|
|
|
|
object pattern (one pool per process), attach to the app:
|
|
|
|
object pattern (one pool per process), attach to the app:
|
|
|
|
from psql import PsqlDB
|
|
|
|
from psql import PsqlDB
|
|
|
@@ -13,44 +13,29 @@ context manager:
|
|
|
|
async with PsqlDB(database="app", user="postgres") as db:
|
|
|
|
async with PsqlDB(database="app", user="postgres") as db:
|
|
|
|
await db.execute("CREATE TABLE ...")
|
|
|
|
await db.execute("CREATE TABLE ...")
|
|
|
|
|
|
|
|
|
|
|
|
lifecycle:
|
|
|
|
|
|
|
|
construction is sync, opens no socket. connect() builds the asyncpg pool, validates it
|
|
|
|
|
|
|
|
with `SELECT 1` (fail loud on bad host/credentials immediately, not on the first real
|
|
|
|
|
|
|
|
op), returns self; concurrent connect() calls are lock-serialized so only one pool is
|
|
|
|
|
|
|
|
ever live. close() closes the pool and nulls the reference (a closed instance reports
|
|
|
|
|
|
|
|
not-connected).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dsn:
|
|
|
|
dsn:
|
|
|
|
host/port default to None, not "localhost"/5432 — asyncpg only reads a dsn's embedded
|
|
|
|
host/port default to None, not "localhost"/5432 - asyncpg only reads a dsn's embedded
|
|
|
|
host/port when the host/port kwargs are falsy, so a hardcoded default would silently
|
|
|
|
host/port when the host/port kwargs are falsy, so a hardcoded default would silently
|
|
|
|
shadow the dsn's server. pass dsn=... in pool_kwargs alone (no host/port) to let the
|
|
|
|
shadow the dsn's server. pass dsn=... in pool_kwargs alone (no host/port) to let the
|
|
|
|
dsn's own host/port reach asyncpg; the no-dsn path still defaults to localhost:5432.
|
|
|
|
dsn's own host/port reach asyncpg; the no-dsn path still defaults to localhost:5432.
|
|
|
|
|
|
|
|
|
|
|
|
two-layer API:
|
|
|
|
two-layer API:
|
|
|
|
LAYER 1 — friendly, portable verbs for simple single-table CRUD. these hide the
|
|
|
|
LAYER 1 - friendly, portable verbs for simple single-table CRUD, byte-for-byte
|
|
|
|
dialect and are byte-for-byte identical to the `mysql` lib, so a dev swaps psql<->mysql
|
|
|
|
identical to the `mysql` lib (zero call-site changes swapping psql<->mysql):
|
|
|
|
with zero call-site changes: create_database, create_table, drop, insert, get,
|
|
|
|
create_database, create_table, drop, insert, get, get_one, delete, exists, upsert.
|
|
|
|
get_one, delete, exists, upsert.
|
|
|
|
LAYER 2 - raw escape hatch for the complex ~20% (joins, aggregates, CTEs, window
|
|
|
|
LAYER 2 — raw escape hatch for the complex ~20% (joins, aggregates, CTEs, window
|
|
|
|
functions): execute, fetch, fetchone, fetchval, transaction, using `$1, $2`
|
|
|
|
functions): execute, fetch, fetchone, fetchval, transaction. you write the SQL with
|
|
|
|
placeholders (asyncpg style). deliberately nothing in between - no query builder/ORM.
|
|
|
|
`$1, $2` placeholders (asyncpg style) + params; the wrapper still gives pooling,
|
|
|
|
|
|
|
|
parameterization, fail-loud errors, and row->dict conversion. raw SQL is psql-specific.
|
|
|
|
|
|
|
|
there is deliberately NOTHING in between — no query builder / ORM. a join goes through
|
|
|
|
|
|
|
|
raw fetch(), never a chainable .where()/.join().
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
rows:
|
|
|
|
|
|
|
|
layer 1 and fetch/fetchone return plain dicts ({column: value}), not asyncpg Record
|
|
|
|
|
|
|
|
objects — identical shape to the mysql lib.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
placeholders / injection safety:
|
|
|
|
placeholders / injection safety:
|
|
|
|
values are ALWAYS parameterized — layer 1 builds `$1, $2` internally; layer 2 takes
|
|
|
|
values are ALWAYS parameterized - layer 1 builds `$1, $2` internally; layer 2 takes
|
|
|
|
your `$1` placeholders + *params. never f-string/format a value into SQL. only
|
|
|
|
your `$1` placeholders + *params. never f-string/format a value into SQL. only
|
|
|
|
identifiers (table/column names) are interpolated, and they are quoted.
|
|
|
|
identifiers (table/column names) are interpolated, and they are quoted.
|
|
|
|
|
|
|
|
|
|
|
|
errors (FAIL LOUD — unlike the mongo lib's swallow-and-default):
|
|
|
|
errors (FAIL LOUD - unlike the mongo lib's swallow-and-default):
|
|
|
|
every method catches the driver error (asyncpg.PostgresError / InterfaceError, OSError
|
|
|
|
every method catches the driver error (asyncpg.PostgresError / InterfaceError, OSError
|
|
|
|
on connection loss), logs via getLogger(__name__), and re-raises. a None/[] return is
|
|
|
|
on connection loss), logs via getLogger(__name__), and re-raises. a None/[] return is
|
|
|
|
only ever a real result (no row, empty table) — never a swallowed failure. for anything
|
|
|
|
only ever a real result (no row, empty table) - never a swallowed failure. for anything
|
|
|
|
not wrapped, use the raw `.pool` property (the asyncpg.Pool).
|
|
|
|
not wrapped, use the raw `.pool` property (the asyncpg.Pool).
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
@@ -68,7 +53,7 @@ _DRIVER_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError)
|
|
|
|
def _quote_ident(identifier: str) -> str:
|
|
|
|
def _quote_ident(identifier: str) -> str:
|
|
|
|
"""quote a sql identifier (table/column), escaping embedded double-quotes
|
|
|
|
"""quote a sql identifier (table/column), escaping embedded double-quotes
|
|
|
|
|
|
|
|
|
|
|
|
identifiers can't be parameterized, so they are interpolated — quoting + doubling any
|
|
|
|
identifiers can't be parameterized, so they are interpolated - quoting + doubling any
|
|
|
|
embedded quote is the postgres-safe way to do that for caller-supplied names.
|
|
|
|
embedded quote is the postgres-safe way to do that for caller-supplied names.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
return '"' + identifier.replace('"', '""') + '"'
|
|
|
|
return '"' + identifier.replace('"', '""') + '"'
|
|
|
@@ -138,15 +123,18 @@ class PsqlDB:
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
async with self._connect_lock:
|
|
|
|
async with self._connect_lock:
|
|
|
|
if self._pool is not None:
|
|
|
|
if self._pool is not None:
|
|
|
|
await self.close()
|
|
|
|
await self._close_locked()
|
|
|
|
pool = await asyncpg.create_pool(**self._config)
|
|
|
|
pool = None
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
|
|
|
|
pool = await asyncpg.create_pool(**self._config)
|
|
|
|
await pool.fetchval("SELECT 1")
|
|
|
|
await pool.fetchval("SELECT 1")
|
|
|
|
except _DRIVER_ERRORS:
|
|
|
|
except _DRIVER_ERRORS:
|
|
|
|
log.exception("psql.connect() validation failed")
|
|
|
|
log.exception("psql.connect() failed")
|
|
|
|
|
|
|
|
if pool is not None:
|
|
|
|
await pool.close()
|
|
|
|
await pool.close()
|
|
|
|
raise
|
|
|
|
raise
|
|
|
|
except BaseException:
|
|
|
|
except BaseException:
|
|
|
|
|
|
|
|
if pool is not None:
|
|
|
|
await pool.close()
|
|
|
|
await pool.close()
|
|
|
|
raise
|
|
|
|
raise
|
|
|
|
self._pool = pool
|
|
|
|
self._pool = pool
|
|
|
@@ -154,7 +142,17 @@ class PsqlDB:
|
|
|
|
|
|
|
|
|
|
|
|
async def close(self) -> None:
|
|
|
|
async def close(self) -> None:
|
|
|
|
"""close the pool on shutdown and null the reference (so pool/connected checks
|
|
|
|
"""close the pool on shutdown and null the reference (so pool/connected checks
|
|
|
|
report not-connected against a dead pool)"""
|
|
|
|
report not-connected against a dead pool)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
guarded by the same lock as connect() - a close() racing an in-flight connect()
|
|
|
|
|
|
|
|
waits for it rather than no-opping against a not-yet-installed pool and leaving
|
|
|
|
|
|
|
|
the just-built one live.
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
async with self._connect_lock:
|
|
|
|
|
|
|
|
await self._close_locked()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def _close_locked(self) -> None:
|
|
|
|
|
|
|
|
"""close the pool and null the reference; caller must hold `_connect_lock`"""
|
|
|
|
if self._pool is None:
|
|
|
|
if self._pool is None:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
try:
|
|
|
|
try:
|
|
|
@@ -166,29 +164,29 @@ class PsqlDB:
|
|
|
|
self._pool = None
|
|
|
|
self._pool = None
|
|
|
|
|
|
|
|
|
|
|
|
async def __aenter__(self) -> "PsqlDB":
|
|
|
|
async def __aenter__(self) -> "PsqlDB":
|
|
|
|
|
|
|
|
"""enter: connect() and return self"""
|
|
|
|
return await self.connect()
|
|
|
|
return await self.connect()
|
|
|
|
|
|
|
|
|
|
|
|
async def __aexit__(self, exc_type, exc, tb) -> None:
|
|
|
|
async def __aexit__(self, exc_type, exc, tb) -> None:
|
|
|
|
|
|
|
|
"""exit: close(), ignoring exc_type/exc/tb"""
|
|
|
|
await self.close()
|
|
|
|
await self.close()
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
@property
|
|
|
|
def pool(self) -> asyncpg.Pool:
|
|
|
|
def pool(self) -> asyncpg.Pool:
|
|
|
|
"""raw asyncpg.Pool escape hatch; full driver surface, raises
|
|
|
|
"""raw asyncpg.Pool escape hatch for copy/prepare/listen-notify/cursors and
|
|
|
|
|
|
|
|
anything not wrapped; full driver surface, raises"""
|
|
|
|
use for copy/prepare/listen-notify/cursors and anything not wrapped.
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
if self._pool is None:
|
|
|
|
if self._pool is None:
|
|
|
|
raise RuntimeError("psql: not connected; call await db.connect() first")
|
|
|
|
raise RuntimeError("psql: not connected; call await db.connect() first")
|
|
|
|
return self._pool
|
|
|
|
return self._pool
|
|
|
|
|
|
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
|
|
# -------------------------------------------------------------------------
|
|
|
|
# layer 2 — raw escape hatch (you write the SQL, $1 placeholders)
|
|
|
|
# layer 2 - raw escape hatch (you write the SQL, $1 placeholders)
|
|
|
|
|
|
|
|
|
|
|
|
async def execute(self, query: str, *params: Any) -> str:
|
|
|
|
async def execute(self, query: str, *params: Any) -> str:
|
|
|
|
"""run a statement (INSERT/UPDATE/DELETE/DDL); return asyncpg's status string
|
|
|
|
"""run a statement (INSERT/UPDATE/DELETE/DDL); return asyncpg's status string
|
|
|
|
|
|
|
|
|
|
|
|
the status string is e.g. "INSERT 0 1" / "UPDATE 3" / "DELETE 2" — parse it or use
|
|
|
|
e.g. "INSERT 0 1" / "UPDATE 3" / "DELETE 2" - parse it, or use the layer-1 verbs
|
|
|
|
the layer-1 verbs (insert/delete) which return structured values instead.
|
|
|
|
(insert/delete) which return structured values instead.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
return await self.pool.execute(query, *params)
|
|
|
|
return await self.pool.execute(query, *params)
|
|
|
@@ -208,8 +206,7 @@ class PsqlDB:
|
|
|
|
async def fetchone(self, query: str, *params: Any) -> Optional[dict]:
|
|
|
|
async def fetchone(self, query: str, *params: Any) -> Optional[dict]:
|
|
|
|
"""run a query and return the first row as a dict, or None if no rows
|
|
|
|
"""run a query and return the first row as a dict, or None if no rows
|
|
|
|
|
|
|
|
|
|
|
|
named fetchone (not asyncpg's fetchrow) to match the mysql lib's layer-2 surface;
|
|
|
|
named fetchone (not asyncpg's fetchrow) to match the mysql lib's layer-2 surface.
|
|
|
|
maps to the driver's fetchrow internally.
|
|
|
|
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
row = await self.pool.fetchrow(query, *params)
|
|
|
|
row = await self.pool.fetchrow(query, *params)
|
|
|
@@ -234,15 +231,15 @@ class PsqlDB:
|
|
|
|
await conn.execute("INSERT ...", a)
|
|
|
|
await conn.execute("INSERT ...", a)
|
|
|
|
await conn.execute("UPDATE ...", b)
|
|
|
|
await conn.execute("UPDATE ...", b)
|
|
|
|
commits on clean exit, rolls back and re-raises on any error. `conn` is a raw
|
|
|
|
commits on clean exit, rolls back and re-raises on any error. `conn` is a raw
|
|
|
|
asyncpg connection (use its $1-placeholder execute/fetch/... directly).
|
|
|
|
asyncpg connection ($1-placeholder execute/fetch/... directly).
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
return _Transaction(self.pool)
|
|
|
|
return _Transaction(self.pool)
|
|
|
|
|
|
|
|
|
|
|
|
# -------------------------------------------------------------------------
|
|
|
|
# -------------------------------------------------------------------------
|
|
|
|
# layer 1 — friendly portable verbs (identical across psql/mysql)
|
|
|
|
# layer 1 - friendly portable verbs (identical across psql/mysql)
|
|
|
|
|
|
|
|
|
|
|
|
async def create_database(self, name: str) -> None:
|
|
|
|
async def create_database(self, name: str) -> None:
|
|
|
|
"""CREATE DATABASE name (raises if it already exists — postgres has no IF NOT
|
|
|
|
"""CREATE DATABASE name (raises if it already exists - postgres has no IF NOT
|
|
|
|
EXISTS for CREATE DATABASE; catch the duplicate error or check first)"""
|
|
|
|
EXISTS for CREATE DATABASE; catch the duplicate error or check first)"""
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
await self.pool.execute(f"CREATE DATABASE {_quote_ident(name)}")
|
|
|
|
await self.pool.execute(f"CREATE DATABASE {_quote_ident(name)}")
|
|
|
@@ -278,7 +275,7 @@ class PsqlDB:
|
|
|
|
"""INSERT one row from {column: value}; return the inserted rowcount (1)
|
|
|
|
"""INSERT one row from {column: value}; return the inserted rowcount (1)
|
|
|
|
|
|
|
|
|
|
|
|
values are parameterized ($1, $2, ...). returns the number of rows inserted (1 on
|
|
|
|
values are parameterized ($1, $2, ...). returns the number of rows inserted (1 on
|
|
|
|
success) — the portable return shape shared with the mysql lib.
|
|
|
|
success) - the portable return shape shared with the mysql lib.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
cols = list(values.keys())
|
|
|
|
cols = list(values.keys())
|
|
|
|
placeholders = ", ".join(f"${i + 1}" for i in range(len(cols)))
|
|
|
|
placeholders = ", ".join(f"${i + 1}" for i in range(len(cols)))
|
|
|
@@ -294,7 +291,7 @@ class PsqlDB:
|
|
|
|
async def get(self, table: str, conditions: Optional[Dict[str, Any]] = None) -> List[dict]:
|
|
|
|
async def get(self, table: str, conditions: Optional[Dict[str, Any]] = None) -> List[dict]:
|
|
|
|
"""SELECT * rows matching equality `conditions` (col = val AND ...) as dicts
|
|
|
|
"""SELECT * rows matching equality `conditions` (col = val AND ...) as dicts
|
|
|
|
|
|
|
|
|
|
|
|
conditions=None/{} returns all rows. simple equality only — anything more complex
|
|
|
|
conditions=None/{} returns all rows. simple equality only - anything more complex
|
|
|
|
goes through raw fetch().
|
|
|
|
goes through raw fetch().
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
where, params = _where(conditions)
|
|
|
|
where, params = _where(conditions)
|
|
|
@@ -342,7 +339,7 @@ class PsqlDB:
|
|
|
|
raise
|
|
|
|
raise
|
|
|
|
|
|
|
|
|
|
|
|
async def upsert(self, table: str, values: Dict[str, Any], conflict: Sequence[str]) -> int:
|
|
|
|
async def upsert(self, table: str, values: Dict[str, Any], conflict: Sequence[str]) -> int:
|
|
|
|
"""INSERT ... ON CONFLICT (conflict_cols) DO UPDATE — insert or update on key clash
|
|
|
|
"""INSERT ... ON CONFLICT (conflict_cols) DO UPDATE - insert or update on key clash
|
|
|
|
|
|
|
|
|
|
|
|
`conflict` is the list of columns forming the unique/pk constraint to upsert on.
|
|
|
|
`conflict` is the list of columns forming the unique/pk constraint to upsert on.
|
|
|
|
the wrapper emits ON CONFLICT here (the mysql lib emits ON DUPLICATE KEY UPDATE for
|
|
|
|
the wrapper emits ON CONFLICT here (the mysql lib emits ON DUPLICATE KEY UPDATE for
|
|
|
@@ -380,7 +377,7 @@ class _Transaction:
|
|
|
|
await self._tx.start()
|
|
|
|
await self._tx.start()
|
|
|
|
except BaseException:
|
|
|
|
except BaseException:
|
|
|
|
# start() (or transaction()) failing after acquire would otherwise leak the
|
|
|
|
# start() (or transaction()) failing after acquire would otherwise leak the
|
|
|
|
# pooled connection — __aexit__ is not called when __aenter__ raises. release
|
|
|
|
# pooled connection - __aexit__ is not called when __aenter__ raises. release
|
|
|
|
# it and reset so a failed transaction start never burns a pool slot.
|
|
|
|
# it and reset so a failed transaction start never burns a pool slot.
|
|
|
|
await self._pool.release(self._conn)
|
|
|
|
await self._pool.release(self._conn)
|
|
|
|
self._conn = None
|
|
|
|
self._conn = None
|
|
|
|