A decorator wraps a function/class to modify or extend its behavior without changing its source.
import functools
import time
def timer(func):
@functools.wraps(func) # preserves __name__, __doc__, etc.
def wrapper(*args, **kwargs):
start = time.perf_counter()
result = func(*args, **kwargs)
print(f"{func.__name__} took {time.perf_counter() - start:.4f}s")
return result
return wrapper
@timer
def slow_add(a, b):
time.sleep(0.1)
return a + bDecorators with arguments need an extra layer of nesting:
def retry(times=3, exceptions=(Exception,)):
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
last_exc = None
for attempt in range(times):
try:
return func(*args, **kwargs)
except exceptions as e:
last_exc = e
raise last_exc
return wrapper
return decorator
@retry(times=5, exceptions=(ConnectionError,))
def fetch(): ...Class-based decorators (useful for stateful decorators):
class CountCalls:
def __init__(self, func):
functools.update_wrapper(self, func)
self.func = func
self.count = 0
def __call__(self, *args, **kwargs):
self.count += 1
return self.func(*args, **kwargs)Built-ins worth knowing: @functools.lru_cache, @functools.cache, @functools.singledispatch (function overloading by type), @functools.cached_property, @contextlib.contextmanager.
An iterator implements __iter__ and __next__. A generator function (uses yield) is a convenient way to build one lazily.
def fibonacci():
a, b = 0, 1
while True:
yield a
a, b = b, a + b
fib = fibonacci()
first_10 = [next(fib) for _ in range(10)]Generator expressions avoid building intermediate lists — critical for memory efficiency over large data:
total = sum(x**2 for x in range(10_000_000)) # no list materializedyield from delegates to a sub-generator:
def chain(*iterables):
for it in iterables:
yield from itSending values into a generator turns it into a rudimentary coroutine:
def running_average():
total, count = 0, 0
average = None
while True:
value = yield average
total += value
count += 1
average = total / count
avg = running_average()
next(avg) # prime it
print(avg.send(10)) # 10.0
print(avg.send(20)) # 15.0Generators also support .throw() and .close() for exception injection and cleanup — this is the mechanism asyncio was originally built on before native async/await.
Implement __enter__/__exit__ for the with statement, or use @contextlib.contextmanager for a generator-based shortcut.
from contextlib import contextmanager
@contextmanager
def db_transaction(conn):
cursor = conn.cursor()
try:
yield cursor
conn.commit()
except Exception:
conn.rollback()
raise
finally:
cursor.close()Class-based version, useful when you need more control or reusable state:
class Timer:
def __enter__(self):
self.start = time.perf_counter()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.elapsed = time.perf_counter() - self.start
return False # don't suppress exceptionscontextlib.ExitStack lets you manage a dynamic number of context managers, and contextlib.suppress(Exception) is a clean alternative to a bare try/except/pass.
Descriptors are the mechanism behind @property, methods, staticmethod, classmethod, and ORMs like Django's. A descriptor defines __get__, __set__, and/or __delete__ and lives on the class, not the instance.
class Positive:
def __set_name__(self, owner, name):
self.name = "_" + name
def __get__(self, obj, objtype=None):
if obj is None:
return self
return getattr(obj, self.name)
def __set__(self, obj, value):
if value < 0:
raise ValueError(f"{self.name} must be positive")
setattr(obj, self.name, value)
class Product:
price = Positive()
def __init__(self, price):
self.price = price- Data descriptors (define
__set__or__delete__) take priority over instance__dict__. - Non-data descriptors (only
__get__) can be overridden by instance attributes — this is how regular methods work.
A metaclass controls how classes themselves are created — type is the default metaclass. Use metaclasses when you need to intercept class creation, not instance creation.
class SingletonMeta(type):
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super().__call__(*args, **kwargs)
return cls._instances[cls]
class Config(metaclass=SingletonMeta):
pass__init_subclass__ is a lighter-weight alternative for many cases where you'd reach for a metaclass — it hooks into subclass creation without the complexity:
class Plugin:
registry = []
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
Plugin.registry.append(cls)
class MyPlugin(Plugin):
pass # auto-registeredPython uses the C3 linearization algorithm to resolve attribute lookup order in multiple inheritance.
class A:
def greet(self): print("A")
class B(A):
def greet(self): print("B"); super().greet()
class C(A):
def greet(self): print("C"); super().greet()
class D(B, C):
pass
D().greet() # B, C, A
print(D.__mro__) # (D, B, C, A, object)super() doesn't mean "my parent class" — it means "the next class in the MRO," which is why cooperative multiple inheritance works. This underlies mixin-based design.
The GIL (Global Interpreter Lock) allows only one thread to execute Python bytecode at a time. This makes threading useful for I/O-bound work but not CPU-bound work (note: PEP 703's free-threaded/no-GIL build is available as of Python 3.13+, but most deployments still run with the GIL).
| Model | Best for | Why |
|---|---|---|
threading |
I/O-bound (network, disk) | GIL released during I/O waits |
multiprocessing |
CPU-bound | Separate processes, separate GILs |
asyncio |
High-concurrency I/O | Single-threaded event loop, no OS thread overhead |
import asyncio
async def fetch(url, session):
async with session.get(url) as resp:
return await resp.text()
async def main(urls):
import aiohttp
async with aiohttp.ClientSession() as session:
tasks = [fetch(u, session) for u in urls]
return await asyncio.gather(*tasks)
asyncio.run(main(["https://example.com"] * 10))from concurrent.futures import ProcessPoolExecutor
def cpu_heavy(n):
return sum(i * i for i in range(n))
with ProcessPoolExecutor() as executor:
results = list(executor.map(cpu_heavy, [10**7] * 4))Key asyncio primitives: asyncio.gather, asyncio.wait_for (timeouts), asyncio.Queue, asyncio.Lock, TaskGroup (3.11+, structured concurrency with automatic error propagation).
- Reference counting is the primary mechanism; an object is freed when its refcount hits zero.
- Cyclic garbage collector (
gcmodule) handles reference cycles that refcounting alone can't clean up. __slots__eliminates the per-instance__dict__, cutting memory use and speeding attribute access for classes with many instances:
class Point:
__slots__ = ("x", "y")
def __init__(self, x, y):
self.x, self.y = x, yweakreflets you reference an object without increasing its refcount — useful for caches that shouldn't keep objects alive.sys.getsizeof,tracemalloc, andobjgraphare the go-to tools for diagnosing memory issues.
import cProfile
cProfile.run("my_function()")
# line-by-line, needs `pip install line_profiler`
# @profile then: kernprof -lv script.pyCommon wins, roughly in order of impact:
- Use the right data structure (
set/dictfor membership tests — O(1) vs O(n) for lists) - Avoid repeated attribute lookups in hot loops (cache as local variable)
- Use built-ins and vectorized libraries (
numpy) over manual Python loops functools.lru_cachefor expensive pure functions- For genuinely CPU-bound bottlenecks:
Cython,Numba(@njit), or rewrite the hot path in Rust/C viapyo3/cffi
from typing import TypeVar, Generic, Protocol, overload, TypedDict, Literal
T = TypeVar("T")
class Stack(Generic[T]):
def __init__(self) -> None:
self._items: list[T] = []
def push(self, item: T) -> None:
self._items.append(item)
def pop(self) -> T:
return self._items.pop()
# Structural typing — "if it walks like a duck"
class Comparable(Protocol):
def __lt__(self, other) -> bool: ...
def my_sort(items: list[Comparable]) -> list[Comparable]:
return sorted(items)
class Movie(TypedDict):
title: str
year: int
Mode = Literal["r", "w", "a"]Modern syntax (3.12+): type IntList = list[int] and generic classes as class Stack[T]: ... directly, no TypeVar boilerplate needed. Run mypy or pyright in CI to actually enforce these — annotations alone do nothing at runtime.
from dataclasses import dataclass, field
@dataclass(frozen=True, order=True)
class Point:
x: float
y: float
tags: list[str] = field(default_factory=list, compare=False)dataclasses(stdlib): boilerplate reduction, no validation.attrs: predates dataclasses, more features (validators, converters), faster.pydantic: adds runtime validation and parsing — the standard choice for API payloads, config, and settings (BaseModel,BaseSettings).
| Method | Purpose |
|---|---|
__repr__ / __str__ |
debugging vs. display representation |
__eq__ / __hash__ |
equality + hashability (must define together) |
__len__, __getitem__, __setitem__ |
container behavior, enables slicing/iteration |
__enter__ / __exit__ |
context manager protocol |
__iter__ / __next__ |
iterator protocol |
__call__ |
makes an instance callable like a function |
__new__ |
controls instance creation (before __init__) |
__get__/__set__/__delete__ |
descriptor protocol |
__class_getitem__ |
enables MyClass[int] generic syntax |
# Exception groups (3.11+) for concurrent/multi-error scenarios
try:
async with asyncio.TaskGroup() as tg:
tg.create_task(job1())
tg.create_task(job2())
except* ValueError as eg:
for e in eg.exceptions:
print("value error:", e)
except* TimeoutError as eg:
print("timeouts:", eg.exceptions)# Custom exception hierarchies carry structured data, not just messages
class APIError(Exception):
def __init__(self, message, status_code):
super().__init__(message)
self.status_code = status_coderaise NewError(...) from original_exc preserves the causal chain (__cause__) instead of just chaining accidentally (__context__).
from itertools import islice, groupby, chain, product, tee, accumulate
from functools import reduce, partial, cache
# Windowed pairs
def pairwise(iterable):
a, b = tee(iterable)
next(b, None)
return zip(a, b)
# Partial application
add_10 = partial(lambda x, y: x + y, 10)
# Reduce
total = reduce(lambda acc, x: acc + x, range(10), 0)itertools is the single most underused stdlib module for advanced Python — groupby, chain.from_iterable, combinations/permutations, and accumulate replace huge amounts of manual loop logic.
Pick one thread and I'll go much deeper with runnable examples and edge cases:
- Build a real async web scraper or API client with
asyncio+aiohttp - Walk through implementing your own ORM-lite using descriptors + metaclasses
- Profile and optimize an actual slow function you have
- Set up strict typing (
mypy --strict) on an existing codebase - Design a plugin architecture using
__init_subclass__
Tell me which direction is useful and I'll build it out with you.