diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42de293..abd82d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,10 +162,10 @@ jobs: services: coordinode: # Pinned by digest, not by tag: the proto submodule pins a server - # version, and a tag can be re-pushed, so `:0.7.0` alone does not name - # one fixed server. The digest below is 0.7.0; bump both together with + # version, and a tag can be re-pushed, so `:0.7.2` alone does not name + # one fixed server. The digest below is 0.7.2; bump both together with # the submodule. - image: ghcr.io/structured-world/coordinode@sha256:2e76528dde460fd715549a8ff06e52c5187b12ff03cdbca59b371f46fa664829 + image: ghcr.io/structured-world/coordinode@sha256:3a3657306cbb050f9ab9562965e7645f332d63a30b360813128434559c6cfae7 ports: - 7080:7080 - 7084:7084 diff --git a/README.md b/README.md index 91785a5..0b120db 100644 --- a/README.md +++ b/README.md @@ -102,9 +102,10 @@ with CoordinodeClient("localhost:7080") as db: raise ``` -Requires a CoordiNode server of **v0.7.0 or newer** — the release this client -is integration-tested against; schema management uses the v2 schema service -that release introduced. `health()` exercises a different service, so a +Requires a CoordiNode server of **v0.7.2 or newer**, the release this client +is integration-tested against. Schema management uses the v2 schema service +0.7 introduced, and a TIMESTAMP property arrives as signed epoch microseconds +only from 0.7.2 on. `health()` exercises a different service, so a server without the transaction RPCs passes the health check and then refuses `transaction()`. diff --git a/coordinode-embedded/Cargo.lock b/coordinode-embedded/Cargo.lock index b63aee5..cc78e8f 100644 --- a/coordinode-embedded/Cargo.lock +++ b/coordinode-embedded/Cargo.lock @@ -629,7 +629,7 @@ dependencies = [ [[package]] name = "coordinode-cluster" -version = "0.7.0" +version = "0.7.2" dependencies = [ "coordinode-storage", "coordinode-workspace-hack", @@ -644,8 +644,9 @@ dependencies = [ [[package]] name = "coordinode-core" -version = "0.7.0" +version = "0.7.2" dependencies = [ + "atomic-waker", "bytes", "crc32c", "rmp", @@ -663,7 +664,7 @@ dependencies = [ [[package]] name = "coordinode-embed" -version = "0.7.0" +version = "0.7.2" dependencies = [ "coordinode-core", "coordinode-lsm-tree", @@ -700,9 +701,9 @@ dependencies = [ [[package]] name = "coordinode-lsm-tree" -version = "5.11.16" +version = "5.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3bc9ff6ba54549c8bc4e562c3c9bb0c776b97af8796694b3837e407e8aa008" +checksum = "b5f4f179a6dfb6bf7757603318757f8e5b389e651e1da4c7a667ee34ee776720" dependencies = [ "arc-swap", "cpufeatures 0.3.1", @@ -726,7 +727,7 @@ dependencies = [ [[package]] name = "coordinode-modality" -version = "0.7.0" +version = "0.7.2" dependencies = [ "coordinode-core", "coordinode-lsm-tree", @@ -743,7 +744,7 @@ dependencies = [ [[package]] name = "coordinode-query" -version = "0.7.0" +version = "0.7.2" dependencies = [ "chrono", "coordinode-cluster", @@ -772,7 +773,7 @@ dependencies = [ [[package]] name = "coordinode-raft" -version = "0.7.0" +version = "0.7.2" dependencies = [ "bytes", "coordinode-core", @@ -797,9 +798,10 @@ dependencies = [ [[package]] name = "coordinode-search" -version = "0.7.0" +version = "0.7.2" dependencies = [ "aes-gcm 0.11.1", + "coordinode-core", "coordinode-storage", "coordinode-workspace-hack", "hmac 0.13.0", @@ -815,7 +817,7 @@ dependencies = [ [[package]] name = "coordinode-storage" -version = "0.7.0" +version = "0.7.2" dependencies = [ "byteorder-lite", "bytes", @@ -827,7 +829,7 @@ dependencies = [ "crc32fast", "dashmap", "flume", - "fs4", + "fs4 1.1.0", "log", "lz4_flex 0.14.0", "metrics", @@ -847,7 +849,7 @@ dependencies = [ [[package]] name = "coordinode-vector" -version = "0.7.0" +version = "0.7.2" dependencies = [ "coordinode-core", "coordinode-storage", @@ -866,17 +868,27 @@ dependencies = [ [[package]] name = "coordinode-wire" -version = "0.7.0" +version = "0.7.2" dependencies = [ + "arc-swap", "bytes", "coordinode-workspace-hack", + "futures-util", + "hyper-util", + "metrics", + "parking_lot", "prost", "rustls", "rustls-pki-types", "rustls-rustcrypto", + "rustls-webpki 0.103.15", "structured-zstd", "thiserror", + "tokio", + "tokio-rustls", "tonic", + "tower-service", + "tracing", ] [[package]] @@ -1003,7 +1015,8 @@ dependencies = [ "fnv", "foldhash 0.1.5", "foldhash 0.2.0", - "fs4", + "fs4 0.13.1", + "fs4 1.1.0", "futures-channel", "futures-core", "futures-macro", @@ -1243,6 +1256,7 @@ dependencies = [ "tonic-build", "tonic-prost", "tonic-prost-build", + "tonic-types", "tower", "tower-layer", "tower-service", @@ -1853,6 +1867,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fs4" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e72ed92b67c146290f88e9c89d60ca163ea417a446f61ffd7b72df3e7f1dfd5" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -4092,9 +4116,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "structured-zstd" -version = "0.0.58" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19112515370199811e7e499f32b633b93a2525ea2cf6f7aa3b1e758d65fd932" +checksum = "ddca575bd714f7e8e2ad3d2c3ce8f99f8589d6128d219659831c74e1bbbc4f83" dependencies = [ "fastrand", "twox-hash", @@ -4153,7 +4177,7 @@ dependencies = [ "downcast-rs", "fastdivide", "fnv", - "fs4", + "fs4 0.13.1", "htmlescape", "itertools 0.14.0", "levenshtein_automata", diff --git a/coordinode-embedded/python/coordinode_embedded/__init__.py b/coordinode-embedded/python/coordinode_embedded/__init__.py index 52de969..96fb342 100644 --- a/coordinode-embedded/python/coordinode_embedded/__init__.py +++ b/coordinode-embedded/python/coordinode_embedded/__init__.py @@ -22,6 +22,6 @@ """ from ._coordinode_embedded import Hnsw, LocalClient -from ._types import MultiVector, Path +from ._types import MultiVector, Path, Timestamp -__all__ = ["Hnsw", "LocalClient", "MultiVector", "Path"] +__all__ = ["Hnsw", "LocalClient", "MultiVector", "Path", "Timestamp"] diff --git a/coordinode-embedded/python/coordinode_embedded/_types.py b/coordinode-embedded/python/coordinode_embedded/_types.py index a6cc614..b10af90 100644 --- a/coordinode-embedded/python/coordinode_embedded/_types.py +++ b/coordinode-embedded/python/coordinode_embedded/_types.py @@ -15,8 +15,12 @@ from __future__ import annotations +from datetime import UTC, datetime, timedelta + +# Both packages ship from this repository at one version: the tags come from +# `coordinode` whole. try: # pragma: no cover - exercised by whichever install shape is in use - from coordinode._types import MultiVector, Path + from coordinode._types import MultiVector, Path, Timestamp except ImportError: class MultiVector(list): # type: ignore[no-redef] @@ -29,5 +33,25 @@ class Path(dict): # type: ignore[no-redef] __slots__ = () + _EPOCH = datetime(1970, 1, 1, tzinfo=UTC) + + class Timestamp(int): # type: ignore[no-redef] + """An instant: microseconds since the Unix epoch, negative before 1970.""" + + __slots__ = () + + @classmethod + def from_datetime(cls, moment: datetime) -> Timestamp: + """The instant a timezone-aware datetime names. A naive one names none.""" + if moment.tzinfo is None or moment.utcoffset() is None: + raise ValueError("a naive datetime names no instant; attach a timezone") + # Integer arithmetic on the timedelta: a float of seconds loses + # microseconds this far from the epoch. + return cls((moment - _EPOCH) // timedelta(microseconds=1)) + + def to_datetime(self) -> datetime: + """This instant as a UTC datetime. Raises OverflowError outside years 1..9999.""" + return _EPOCH + timedelta(microseconds=int(self)) + -__all__ = ["MultiVector", "Path"] +__all__ = ["MultiVector", "Path", "Timestamp"] diff --git a/coordinode-embedded/src/lib.rs b/coordinode-embedded/src/lib.rs index a3ec67e..8d2aeeb 100644 --- a/coordinode-embedded/src/lib.rs +++ b/coordinode-embedded/src/lib.rs @@ -22,7 +22,7 @@ use coordinode_embed::{Database, DatabaseError}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyBytes, PyDict, PyList}; +use pyo3::types::{PyBytes, PyDict, PyFloat, PyInt, PyList, PyString}; use rmpv::Value as MsgpackValue; // ── Value → PyObject conversion ────────────────────────────────────────────── @@ -63,6 +63,70 @@ fn path_type(py: Python<'_>) -> Option> { tag_type(py, &PATH_TYPE, "Path") } +/// The `Timestamp` tag, from this package's own `_types` module. +/// +/// An int subclass, for the same reason as [`multi_vector_type`]: without the +/// tag a timestamp read back is an ordinary int, and writing it out again +/// would store an integer. It also turns a timezone-aware `datetime` into the +/// instant it names. Resolved and remembered on the same terms as +/// [`MULTI_VECTOR_TYPE`]. +static TIMESTAMP_TYPE: PyOnceLock> = PyOnceLock::new(); + +fn timestamp_type(py: Python<'_>) -> Option> { + tag_type(py, &TIMESTAMP_TYPE, "Timestamp") +} + +/// `datetime.datetime`, remembered on the same terms as the tags. Looked up +/// through the module rather than `PyDateTime`, which the stable ABI this +/// extension is built for does not expose. +static DATETIME_TYPE: PyOnceLock> = PyOnceLock::new(); + +fn datetime_type(py: Python<'_>) -> Option> { + if let Some(cls) = DATETIME_TYPE.get(py) { + return Some(cls.bind(py).clone()); + } + let cls = py + .import("datetime") + .and_then(|module| module.getattr("datetime")) + .ok()?; + // A racing thread may have set it first; either value is the same class. + let _ = DATETIME_TYPE.set(py, cls.clone().unbind()); + Some(cls) +} + +/// The instant a tagged timestamp or a timezone-aware `datetime` names, in +/// microseconds since the epoch; `None` for any other value. A naive +/// `datetime` is refused: it names no instant until a zone is chosen for it. +fn py_timestamp(obj: &Bound<'_, PyAny>) -> PyResult> { + // The common parameters are none of these and are known by their exact + // type, without the tag lookup and two instance checks below. + if obj.is_exact_instance_of::() + || obj.is_exact_instance_of::() + || obj.is_exact_instance_of::() + { + return Ok(None); + } + let py = obj.py(); + let Some(cls) = timestamp_type(py) else { + return Ok(None); + }; + if obj.is_instance(&cls)? { + return obj.extract::().map(Some); + } + let Some(datetime) = datetime_type(py) else { + return Ok(None); + }; + if obj.is_instance(&datetime)? { + // The tag's own conversion: one definition of the epoch arithmetic, + // and the same refusal of a naive datetime the gRPC client gives. + return cls + .call_method1("from_datetime", (obj,))? + .extract::() + .map(Some); + } + Ok(None) +} + /// Fetch a tag class from this package's `_types`, remembering it in `cell`. /// /// The cell holds successes only, so a lookup that fails is retried on the @@ -93,8 +157,12 @@ fn value_to_py(py: Python<'_>, v: Value) -> PyResult> { Value::Int(i) => Ok(i.into_pyobject(py)?.into_any().unbind()), Value::Float(f) => Ok(f.into_pyobject(py)?.into_any().unbind()), Value::String(s) => Ok(s.into_pyobject(py)?.into_any().unbind()), - // Timestamp: expose as raw microseconds; callers use datetime.fromtimestamp(ts/1e6) - Value::Timestamp(ts) => Ok(ts.into_pyobject(py)?.into_any().unbind()), + // Microseconds since the epoch, tagged so a value read here and written + // straight back stays a timestamp instead of becoming an integer. + Value::Timestamp(ts) => match timestamp_type(py) { + Some(cls) => Ok(cls.call1((ts,))?.unbind()), + None => Ok(ts.into_pyobject(py)?.into_any().unbind()), + }, // Sized from the vector's length in one go. Growing by append costs a // reallocation every time the list outgrows its capacity, and an // embedding is the one value here that routinely runs to hundreds or @@ -272,6 +340,11 @@ fn py_to_value(obj: &Bound<'_, PyAny>) -> PyResult { // bool before int — Python bool is a subclass of int return Ok(Value::Bool(b)); } + // Before int as well: a tagged timestamp is an int subclass, and the int + // branch would store it as an integer. + if let Some(micros) = py_timestamp(obj)? { + return Ok(Value::Timestamp(micros)); + } if let Ok(i) = obj.extract::() { return Ok(Value::Int(i)); } diff --git a/coordinode-rs b/coordinode-rs index d7ef29a..db58b79 160000 --- a/coordinode-rs +++ b/coordinode-rs @@ -1 +1 @@ -Subproject commit d7ef29a447ecd49968b06b2f03bb0fd8a926f23d +Subproject commit db58b79c1051652100e03047284eab71953100ea diff --git a/coordinode/coordinode/__init__.py b/coordinode/coordinode/__init__.py index fcb205f..2eeb7a2 100644 --- a/coordinode/coordinode/__init__.py +++ b/coordinode/coordinode/__init__.py @@ -18,7 +18,7 @@ result = await db.cypher("MATCH (n) RETURN count(n) AS total") """ -from coordinode._types import MultiVector, Path +from coordinode._types import MultiVector, Path, Timestamp from coordinode.client import ( AsyncCoordinodeClient, AsyncTransaction, @@ -50,10 +50,12 @@ "Transaction", "AsyncTransaction", # Values whose wire type only a tag can carry: a plain nested list encodes - # as a list and a plain dict as a map, so sending either of these types - # requires the constructor, not just the shape. + # as a list, a plain dict as a map and a plain int as an integer, so + # sending one of these types requires the constructor, not just the shape. + # An aware datetime encodes as a timestamp too. "MultiVector", "Path", + "Timestamp", # How a write is acknowledged: member count or majority, and how durably. "WriteConcern", "NodeResult", diff --git a/coordinode/coordinode/_types.py b/coordinode/coordinode/_types.py index 703de12..615aa2f 100644 --- a/coordinode/coordinode/_types.py +++ b/coordinode/coordinode/_types.py @@ -4,11 +4,12 @@ from __future__ import annotations +from datetime import UTC, datetime, timedelta from typing import Any # We import proto types lazily to avoid hard-fail when stubs aren't generated yet. -PyValue = int | float | str | bool | bytes | list[float] | list[Any] | dict[str, Any] | None +PyValue = int | float | str | bool | bytes | datetime | list[float] | list[Any] | dict[str, Any] | None class MultiVector(list): @@ -25,6 +26,35 @@ class MultiVector(list): __slots__ = () +_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) + + +class Timestamp(int): + """An instant: microseconds since the Unix epoch, negative before 1970. + + A plain ``int``, so it reads, compares and does arithmetic like the + microsecond count the embedded client returns. As with + :class:`MultiVector`, the type exists to carry the distinction back to the + wire: a value that arrived as a timestamp re-encodes as one, while an + ordinary int stays an integer. + """ + + __slots__ = () + + @classmethod + def from_datetime(cls, moment: datetime) -> Timestamp: + """The instant a timezone-aware datetime names. A naive one names none.""" + if moment.tzinfo is None or moment.utcoffset() is None: + raise ValueError("a naive datetime names no instant; attach a timezone") + # Integer arithmetic on the timedelta: a float of seconds loses + # microseconds this far from the epoch. + return cls((moment - _EPOCH) // timedelta(microseconds=1)) + + def to_datetime(self) -> datetime: + """This instant as a UTC datetime. Raises OverflowError outside years 1..9999.""" + return _EPOCH + timedelta(microseconds=int(self)) + + class Path(dict): """A graph path: the node ids it runs through, and the hops between them. @@ -69,15 +99,17 @@ def _set_sequence(pv: Any, values: Any) -> None: """Encode a Python sequence into the field that matches what it holds. A non-empty run of plain numbers is a Vector; anything else is a list of - values. bool is a subclass of int, so it is excluded explicitly: [True, - False] is a list of booleans, not a Vector of 1.0 and 0.0. + values. bool and Timestamp are subclasses of int, so they are excluded + explicitly: [True, False] is a list of booleans, not a Vector of 1.0 and + 0.0, and a list holding timestamps keeps them as timestamps rather than + rounding their microseconds into floats. """ from coordinode._proto.coordinode.v1.common.types_pb2 import ( # type: ignore[import] PropertyList, Vector, ) - if values and all(isinstance(v, int | float) and not isinstance(v, bool) for v in values): + if values and all(isinstance(v, int | float) and not isinstance(v, bool | Timestamp) for v in values): pv.vector_value.CopyFrom(Vector(values=[float(v) for v in values])) else: pv.list_value.CopyFrom(PropertyList(values=[to_property_value(v) for v in values])) @@ -96,6 +128,11 @@ def to_property_value(py_val: PyValue) -> Any: pass # unset oneof → null semantics elif isinstance(py_val, bool): pv.bool_value = py_val + elif isinstance(py_val, Timestamp): + # Before the int branch below, which would otherwise take it. + pv.timestamp_value.unix_micros = int(py_val) + elif isinstance(py_val, datetime): + pv.timestamp_value.unix_micros = int(Timestamp.from_datetime(py_val)) elif isinstance(py_val, int): pv.int_value = py_val elif isinstance(py_val, float): @@ -145,8 +182,7 @@ def from_property_value(pv: Any) -> PyValue: elif kind == "bytes_value": return pv.bytes_value elif kind == "timestamp_value": - ts = pv.timestamp_value - return {"wall_time": ts.wall_time, "logical": ts.logical} + return Timestamp(pv.timestamp_value.unix_micros) elif kind == "vector_value": return list(pv.vector_value.values) elif kind == "list_value": diff --git a/coordinode/coordinode/client.py b/coordinode/coordinode/client.py index dee06a9..3c08685 100644 --- a/coordinode/coordinode/client.py +++ b/coordinode/coordinode/client.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import datetime import functools import inspect import logging @@ -438,6 +439,7 @@ def __init__(self, proto_constraint: Any) -> None: from coordinode._proto.coordinode.v2.graph.schema_pb2 import ( # type: ignore[import] ConstraintKind, ConstraintState, + IndexBuildState, ) self.name: str = proto_constraint.name @@ -455,11 +457,22 @@ def __init__(self, proto_constraint: Any) -> None: self.state: str = ConstraintState.Name(proto_constraint.state).removeprefix("CONSTRAINT_STATE_") #: The index the constraint owns, for ``UNIQUE`` and ``NODE_KEY``; else ``None``. self.backing_index: str | None = proto_constraint.backing_index or None + has_build = proto_constraint.HasField("build") + #: Operation id of the build that checks the owned index against stored + #: data and decides a ``VALIDATING`` constraint; ``None`` for the kinds + #: checked on each node alone. + self.build_operation: int | None = proto_constraint.build.operation if has_build else None + #: ``"ACCEPTED"``, ``"RUNNING"``, ``"PUBLISHED"``, ``"FAILED"`` or + #: ``"CANCELLED"``; ``None`` without a build. + self.build_state: str | None = ( + IndexBuildState.Name(proto_constraint.build.state).removeprefix("INDEX_BUILD_STATE_") if has_build else None + ) def __repr__(self) -> str: return ( f"ConstraintInfo(name={self.name!r}, label={self.label!r}, properties={self.properties!r}, " - f"kind={self.kind!r}, state={self.state!r}, backing_index={self.backing_index!r})" + f"kind={self.kind!r}, state={self.state!r}, backing_index={self.backing_index!r}, " + f"build_operation={self.build_operation!r}, build_state={self.build_state!r})" ) @@ -1970,12 +1983,18 @@ async def create_constraint( name: str = "", property_type: Any = None, if_not_exists: bool = False, + wait: float | None = None, ) -> ConstraintInfo: """Create a named constraint on the nodes of ``label``. - A ``"unique"`` or ``"node_key"`` constraint owns an index and returns - once that index is checked against the stored nodes; stored nodes that - break any constraint make the call fail with nothing created. + A ``"unique"`` or ``"node_key"`` constraint owns an index. It is + enforced on every write from the start, while a build checks the index + against the stored nodes. The call waits for that build up to ``wait``: + it returns the constraint ``ACTIVE`` once the build finished, or + ``VALIDATING`` with the build still running when the wait ran out; the + build then goes on without the call, and :meth:`get_constraints` shows + its outcome. Stored nodes that break the constraint within the wait + make the call fail with nothing created. Args: label: The label whose nodes are constrained. @@ -1988,7 +2007,13 @@ async def create_constraint( property_type: The required type for ``"property_type"``, as a type spec (see :meth:`create_label`). if_not_exists: Return a constraint of the same name, or an - equivalent one, instead of refusing. + equivalent one, instead of refusing; one still validating is + returned with its running build, so a retry after a lost reply + does not start a second build. + wait: Seconds to wait for the build of the owned index. ``None``: + the server's default. ``0``: return once the constraint and its + build are admitted. The call's deadline is extended by it, so + the deadline does not expire while the server is still waiting. """ from coordinode._proto.coordinode.v2.graph.schema_pb2 import ( # type: ignore[import] ConstraintKind, @@ -2007,6 +2032,8 @@ async def create_constraint( props = [properties] if isinstance(properties, str) else list(properties) if (key == "property_type") != (property_type is not None): raise ValueError("property_type is required for kind 'property_type' and only for it") + if wait is not None and wait < 0: + raise ValueError(f"wait must be zero or more seconds; got {wait!r}") req = CreateConstraintRequest( name=name, label=label, @@ -2016,7 +2043,12 @@ async def create_constraint( ) if property_type is not None: req.property_type.CopyFrom(self._property_type_proto(property_type, "the constraint")) - created = await self._schema_stub.CreateConstraint(req, timeout=self._timeout) + timeout = self._timeout + if wait is not None: + # A zero wait is sent too: absent takes the server's default. + req.wait.FromTimedelta(datetime.timedelta(seconds=wait)) + timeout += wait + created = await self._schema_stub.CreateConstraint(req, timeout=timeout) return ConstraintInfo(created) async def drop_constraint(self, name: str, *, if_exists: bool = False) -> None: @@ -2610,6 +2642,7 @@ def create_constraint( name: str = "", property_type: Any = None, if_not_exists: bool = False, + wait: float | None = None, ) -> ConstraintInfo: """Create a named constraint; see :meth:`AsyncCoordinodeClient.create_constraint`.""" return self._run( @@ -2620,6 +2653,7 @@ def create_constraint( name=name, property_type=property_type, if_not_exists=if_not_exists, + wait=wait, ) ) diff --git a/demo/Dockerfile.jupyter b/demo/Dockerfile.jupyter index 02b73c4..4007ee0 100644 --- a/demo/Dockerfile.jupyter +++ b/demo/Dockerfile.jupyter @@ -2,11 +2,11 @@ # Python stack comes from the same uv.lock the tests run on. # uv exports the lock; only its binary is copied into the image. -FROM ghcr.io/astral-sh/uv:0.12.23@sha256:61d393e44e249f2e4b526b6c7ddcecce245946826e608e11c93ad4f5bba55b21 AS uv +FROM ghcr.io/astral-sh/uv:0.13.1@sha256:4a1f3666816dafadd20ceb71eac9a7d56d7ef30279f7d64048519ab901ada336 AS uv # quay.io is where Project Jupyter publishes its images now; the Docker Hub # copy stopped receiving updates in October 2023. -FROM quay.io/jupyter/scipy-notebook:python-3.13.15 +FROM quay.io/jupyter/scipy-notebook:python-3.13.16 USER root RUN apt-get update && apt-get install -y --no-install-recommends gcc git && rm -rf /var/lib/apt/lists/* diff --git a/demo/README.md b/demo/README.md index eb0cb69..27dcb8d 100644 --- a/demo/README.md +++ b/demo/README.md @@ -19,8 +19,9 @@ Interactive notebooks for LlamaIndex, LangChain, and LangGraph integrations. > alternative was filtering model-written Cypher by a session tag, which leaks > the first time the filter misses a case. `COORDINODE_AGENT_DB` moves that > file; deleting it starts the agent with no memory. -> The Docker Compose stack below pins the CoordiNode **server** image v0.7.0 by -> digest. Do not move it below that: the client manages schema through the v2 +> The Docker Compose stack below pins the CoordiNode **server** image v0.7.2 by +> digest. Do not move it below that: the client reads timestamps in the signed +> epoch-microsecond form 0.7.2 introduced, manages schema through the v2 > schema service 0.7 introduced, and speaks the two-axis write concern 0.6 > introduced, which an older server ignores in favour of its own default. > diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml index 8da5a7e..c991e92 100644 --- a/demo/docker-compose.yml +++ b/demo/docker-compose.yml @@ -4,8 +4,8 @@ services: # COORDINODE_IMAGE at a locally built tag to try a server build before it # is published. # Digest-pinned so a re-pushed tag cannot change the demo underneath - # you; this one is 0.7.0. Override COORDINODE_IMAGE to try another. - image: ${COORDINODE_IMAGE:-ghcr.io/structured-world/coordinode@sha256:2e76528dde460fd715549a8ff06e52c5187b12ff03cdbca59b371f46fa664829} + # you; this one is 0.7.2. Override COORDINODE_IMAGE to try another. + image: ${COORDINODE_IMAGE:-ghcr.io/structured-world/coordinode@sha256:3a3657306cbb050f9ab9562965e7645f332d63a30b360813128434559c6cfae7} container_name: demo-coordinode ports: - "127.0.0.1:37080:7080" # gRPC (native API) — localhost-only diff --git a/demo/notebooks/04_whats_new_in_0_5.ipynb b/demo/notebooks/04_whats_new_in_0_5.ipynb index c845de6..7c3fd53 100644 --- a/demo/notebooks/04_whats_new_in_0_5.ipynb +++ b/demo/notebooks/04_whats_new_in_0_5.ipynb @@ -3,7 +3,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "# What 0.5 Added\n\n[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/structured-world/coordinode-python/blob/main/demo/notebooks/04_whats_new_in_0_5.ipynb)\n\nThe client surface that arrived with CoordiNode 0.5, exercised end to end:\n\n| Feature | What it is for |\n|---------|----------------|\n| `create_nodes_batch` | Create many nodes in one atomic write instead of a loop of round trips |\n| `element_id` | A stable identifier that survives restarts and replication |\n| `schema_revision` | Tells you when a label's shape last changed, so caches know to refresh |\n| `write_concern` | Chooses how durable an acknowledgement is, from fire-and-forget to majority |\n| `read_concern` / `read_preference` | Chooses how fresh a read is, and which replica answers it |\n| `at_timestamp` | Reads the database as it was at a point in time |\n| `transaction()` | Commits several statements as one unit, or rolls back all of them |\n\n> **Needs a server, v0.7.0 or newer** (the release this client is tested\n> against; 0.7 moved schema management to the v2 schema service the\n> `schema_revision` cell reads, and 0.6 changed the write concern to the two\n> axes shown below). These are distribution and durability features, so they\n> exist on the client that talks to a CoordiNode server. Set `COORDINODE_ADDR`\n> before running. The embedded engine has no Raft and no replicas, so the cells\n> below stop with an explanation rather than pretending.\n>\n> The version floor is not covered by the connect cell's health check, which\n> exercises a different service: a server without the transaction RPCs passes\n> it and then refuses them.", + "source": "# What 0.5 Added\n\n[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/structured-world/coordinode-python/blob/main/demo/notebooks/04_whats_new_in_0_5.ipynb)\n\nThe client surface that arrived with CoordiNode 0.5, exercised end to end:\n\n| Feature | What it is for |\n|---------|----------------|\n| `create_nodes_batch` | Create many nodes in one atomic write instead of a loop of round trips |\n| `element_id` | A stable identifier that survives restarts and replication |\n| `schema_revision` | Tells you when a label's shape last changed, so caches know to refresh |\n| `write_concern` | Chooses how durable an acknowledgement is, from fire-and-forget to majority |\n| `read_concern` / `read_preference` | Chooses how fresh a read is, and which replica answers it |\n| `at_timestamp` | Reads the database as it was at a point in time |\n| `transaction()` | Commits several statements as one unit, or rolls back all of them |\n\n> **Needs a server, v0.7.2 or newer** (the release this client is tested\n> against; 0.7 moved schema management to the v2 schema service the\n> `schema_revision` cell reads, and 0.6 changed the write concern to the two\n> axes shown below). These are distribution and durability features, so they\n> exist on the client that talks to a CoordiNode server. Set `COORDINODE_ADDR`\n> before running. The embedded engine has no Raft and no replicas, so the cells\n> below stop with an explanation rather than pretending.\n>\n> The version floor is not covered by the connect cell's health check, which\n> exercises a different service: a server without the transaction RPCs passes\n> it and then refuses them.", "id": "cell-00-ccfb99" }, { diff --git a/docker-compose.yml b/docker-compose.yml index b46c2ab..d2472ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,10 @@ version: "3.9" services: coordinode: - # The same 0.7.0 digest the demo stack and the integration job run, so the + # The same 0.7.2 digest the demo stack and the integration job run, so the # quick start does not hand a newcomer an older server than everything else # here is tested against. Pinned by digest because a tag can be re-pushed. - image: ghcr.io/structured-world/coordinode@sha256:2e76528dde460fd715549a8ff06e52c5187b12ff03cdbca59b371f46fa664829 + image: ghcr.io/structured-world/coordinode@sha256:3a3657306cbb050f9ab9562965e7645f332d63a30b360813128434559c6cfae7 # Named for this repository rather than plain `coordinode`: the server # repository runs its own compose stack that claims that name, and a bare # name is global to the daemon. diff --git a/proto b/proto index 9ee64b9..f5e8d9e 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 9ee64b930f96a61ba527fcdf43370e87045b4294 +Subproject commit f5e8d9e2d36c7cfc20e25b6a32e8ed89fa7468f1 diff --git a/tests/unit/test_embedded_values.py b/tests/unit/test_embedded_values.py index 8e0d08f..17f66c4 100644 --- a/tests/unit/test_embedded_values.py +++ b/tests/unit/test_embedded_values.py @@ -6,6 +6,8 @@ from __future__ import annotations +from datetime import datetime, timedelta, timezone + import pytest ce = pytest.importorskip("coordinode_embedded") @@ -13,7 +15,7 @@ # From the embedded package rather than `coordinode`, so this file runs where # only the wheel is installed. It re-exports `coordinode`'s definitions when # that package is present, so this is the same class either way. -from coordinode_embedded import MultiVector, Path # noqa: E402 (after the skip guard) +from coordinode_embedded import MultiVector, Path, Timestamp # noqa: E402 (after the skip guard) @pytest.fixture @@ -84,3 +86,38 @@ def test_a_plain_dict_is_not_promoted(self, db) -> None: assert not isinstance(rows[0]["m"], Path) assert rows[0]["m"] == {"nodes": [1, 2], "rels": []} + + +class TestTimestampRoundTrip: + """A timestamp written as a datetime reads back as one and writes back as one. + + The engine took a timestamp parameter for a plain integer and refused a + datetime, and handed a stored timestamp back as a plain int that a write + then stored as an integer: the same instant, but no longer a TIMESTAMP. + """ + + def test_a_datetime_is_stored_and_survives_a_read_modify_write(self, db) -> None: + moment = datetime(1969, 7, 20, 20, 17, 40, 123456, tzinfo=timezone(timedelta(hours=2))) + db.cypher("CREATE (:Event {tag: 'ts', at: $at})", params={"at": moment}) + + first = db.cypher("MATCH (n:Event {tag: 'ts'}) RETURN n.at AS at")[0]["at"] + assert isinstance(first, Timestamp), f"a timestamp came back as {type(first).__name__}" + assert first == -14_190_139_876_544 + assert first.to_datetime() == moment + + db.cypher("MATCH (n:Event {tag: 'ts'}) SET n.copy = $at", params={"at": first}) + second = db.cypher("MATCH (n:Event {tag: 'ts'}) RETURN n.copy AS at")[0]["at"] + assert isinstance(second, Timestamp), "the rewrite must not change the type" + assert second == first + + def test_a_plain_int_is_not_promoted(self, db) -> None: + """Only the tag or a datetime makes a timestamp; an int stays an int.""" + rows = db.cypher("RETURN $n AS n", params={"n": 1_700_000_000_000_000}) + + assert not isinstance(rows[0]["n"], Timestamp) + assert rows[0]["n"] == 1_700_000_000_000_000 + + def test_a_naive_datetime_is_refused(self, db) -> None: + """A naive datetime names no instant until a zone is chosen for it.""" + with pytest.raises(ValueError): + db.cypher("RETURN $at AS at", params={"at": datetime(2024, 1, 1)}) diff --git a/tests/unit/test_schema_crud.py b/tests/unit/test_schema_crud.py index ede2577..bb33ef5 100644 --- a/tests/unit/test_schema_crud.py +++ b/tests/unit/test_schema_crud.py @@ -211,6 +211,38 @@ def test_type_constraint_names_its_type_and_owns_no_index(self): assert (c.kind, c.property_type, c.state) == ("PROPERTY_TYPE", "INT64", "VALIDATING") assert c.backing_index is None + def test_validating_constraint_reports_its_build(self): + # A create whose wait ran out returns the constraint VALIDATING with + # its build still running; the caller needs the build to follow it. + c = ConstraintInfo( + schema_pb2.Constraint( + name="c", + label="L", + kind=schema_pb2.CONSTRAINT_KIND_UNIQUE, + state=schema_pb2.CONSTRAINT_STATE_VALIDATING, + backing_index="c_idx", + build=schema_pb2.IndexBuild( + operation=42, + state=schema_pb2.INDEX_BUILD_STATE_RUNNING, + indexed=7, + ), + ) + ) + assert c.state == "VALIDATING" + assert (c.build_operation, c.build_state) == (42, "RUNNING") + + def test_constraint_without_a_build(self): + # NOT_NULL is checked on each node alone and has no build. + c = ConstraintInfo( + schema_pb2.Constraint( + name="c", + label="L", + kind=schema_pb2.CONSTRAINT_KIND_NOT_NULL, + state=schema_pb2.CONSTRAINT_STATE_ACTIVE, + ) + ) + assert (c.build_operation, c.build_state) == (None, None) + # ── element_id ─────────────────────────────────────────────────────────────── @@ -379,6 +411,40 @@ async def _inner() -> None: assert (sent.name, sent.label, list(sent.properties)) == ("l_key", "L", ["a", "b"]) assert sent.kind == schema_pb2.CONSTRAINT_KIND_NODE_KEY and sent.if_not_exists assert not sent.HasField("property_type") + # No wait named: the server's own default applies, so none is sent. + assert not sent.HasField("wait") + + asyncio.run(_inner()) + + def test_wait_is_sent_and_extends_the_deadline(self): + # The server holds the call for up to `wait` while the backing index + # builds. A deadline of the client's timeout alone would expire first + # and report a failure for a constraint the server went on to create. + async def _inner() -> None: + client = self._client() + await client.create_constraint("L", "email", "unique", wait=2.5) + call = client._schema_stub.CreateConstraint.call_args + sent = call.args[0] + assert (sent.wait.seconds, sent.wait.nanos) == (2, 500_000_000) + assert call.kwargs["timeout"] == pytest.approx(client._timeout + 2.5) + + asyncio.run(_inner()) + + def test_zero_wait_is_sent_rather_than_dropped(self): + # Zero means "return once admitted", unlike an absent wait, which + # takes the server's default; dropping it would change the meaning. + async def _inner() -> None: + client = self._client() + await client.create_constraint("L", "email", "unique", wait=0) + sent = client._schema_stub.CreateConstraint.call_args.args[0] + assert sent.HasField("wait") and sent.wait.ToTimedelta().total_seconds() == 0 + + asyncio.run(_inner()) + + def test_negative_wait_raises(self): + async def _inner() -> None: + with pytest.raises(ValueError, match="wait"): + await self._client().create_constraint("L", "email", "unique", wait=-1) asyncio.run(_inner()) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 5c2ebf3..94e2a08 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -4,6 +4,8 @@ to the PropertyValue oneof shows up here rather than in user code. """ +from datetime import UTC, datetime, timedelta, timezone + import pytest from coordinode._proto.coordinode.v1.common.types_pb2 import ( @@ -13,7 +15,10 @@ PropertyValue, Vector, ) -from coordinode._types import from_property_value, to_property_value +from coordinode._proto.coordinode.v1.common.types_pb2 import ( + Timestamp as TimestampProto, +) +from coordinode._types import Timestamp, from_property_value, to_property_value # Proto generation is part of `make install`, so a missing stub is a broken # checkout rather than a reason to quietly skip these. @@ -184,3 +189,93 @@ def test_a_plain_dict_is_not_a_path(self): """Only a value that came back as one re-encodes as one.""" pv = to_property_value({"nodes": [1, 2], "rels": []}) assert pv.WhichOneof("value") == "map_value" + + +# ── timestamps ────────────────────────────────────────────────────────────── + + +class TestTimestamp: + def test_decodes_to_epoch_microseconds(self): + """A timestamp reads as the microsecond count the embedded client returns. + + It used to come back as a {"wall_time", "logical"} dict, a shape no + other path produced and that nothing could write back. + """ + pv = PropertyValue(timestamp_value=TimestampProto(unix_micros=1_700_000_000_123_456)) + decoded = from_property_value(pv) + assert decoded == 1_700_000_000_123_456 + assert isinstance(decoded, Timestamp) + + def test_an_instant_before_1970_is_negative(self): + pv = PropertyValue(timestamp_value=TimestampProto(unix_micros=-86_400_000_000)) + assert from_property_value(pv) == -86_400_000_000 + + def test_survives_a_round_trip(self): + """Reading a timestamp and writing it back keeps its wire type. + + As an ordinary int it would re-encode as an integer, and a TIMESTAMP + property rejects that. + """ + original = PropertyValue(timestamp_value=TimestampProto(unix_micros=-1)) + re_encoded = to_property_value(from_property_value(original)) + assert re_encoded.WhichOneof("value") == "timestamp_value" + assert re_encoded.timestamp_value.unix_micros == -1 + + def test_a_plain_int_is_not_a_timestamp(self): + """Only a value that came back as one, or was built as one, encodes as one.""" + assert to_property_value(1_700_000_000_000_000).WhichOneof("value") == "int_value" + + def test_an_aware_datetime_encodes_exactly(self): + moment = datetime(1969, 7, 20, 20, 17, 40, 123456, tzinfo=timezone(timedelta(hours=2))) + pv = to_property_value(moment) + assert pv.WhichOneof("value") == "timestamp_value" + assert pv.timestamp_value.unix_micros == -14_190_139_876_544 + + def test_a_naive_datetime_is_refused(self): + """A naive datetime names no instant until a zone is chosen for it.""" + with pytest.raises(ValueError): + to_property_value(datetime(2024, 1, 1)) + + def test_to_datetime_is_utc(self): + assert Timestamp(951_782_400_000_000).to_datetime() == datetime(2000, 2, 29, tzinfo=UTC) + assert Timestamp(-1).to_datetime() == datetime(1969, 12, 31, 23, 59, 59, 999999, tzinfo=UTC) + + def test_from_datetime_round_trips(self): + moment = datetime(2023, 11, 14, 22, 13, 20, 123456, tzinfo=UTC) + assert Timestamp.from_datetime(moment) == 1_700_000_000_123_456 + assert Timestamp.from_datetime(moment).to_datetime() == moment + + def test_zero_is_the_epoch(self): + assert Timestamp(0).to_datetime() == datetime(1970, 1, 1, tzinfo=UTC) + pv = to_property_value(Timestamp(0)) + assert pv.WhichOneof("value") == "timestamp_value" + assert pv.timestamp_value.unix_micros == 0 + + def test_from_datetime_before_epoch_is_negative(self): + moment = datetime(1969, 12, 31, 23, 59, 59, 999999, tzinfo=UTC) + assert Timestamp.from_datetime(moment) == -1 + + def test_from_datetime_refuses_naive(self): + with pytest.raises(ValueError): + Timestamp.from_datetime(datetime(2024, 1, 1)) + + @pytest.mark.parametrize( + "values", + [ + [Timestamp(1_700_000_000_123_456), Timestamp(-1)], + [Timestamp(1_700_000_000_123_456), 7], + ], + ) + def test_a_list_holding_timestamps_keeps_them(self, values): + """A list read back with timestamps in it writes back as that list. + + Timestamp is an int, so a run of them used to pass for a Vector and + went out as floats: another wire type, and microseconds lost to the + float's mantissa. + """ + pv = to_property_value(values) + assert pv.WhichOneof("value") == "list_value" + assert from_property_value(pv) == values + kinds = [v.WhichOneof("value") for v in pv.list_value.values] + assert kinds[0] == "timestamp_value" + assert pv.list_value.values[0].timestamp_value.unix_micros == 1_700_000_000_123_456