From db9896f8553eddba42d9ccdfec7cffa0eba9aede Mon Sep 17 00:00:00 2001 From: desty Date: Thu, 24 Sep 2026 12:36:10 +0900 Subject: [PATCH 1/2] fix: keep vector fields in from_existing on RESP2 unified FT.INFO replies With redis-py 8, protocol=2 and legacy_responses=False, FT.INFO packs vector params into the field's flags list. parse_vector_attrs ignored flags, found no dims, and the caller dropped the field without a trace, so from_existing() + create() rebuilt the index without its vectors. Unpack key/value pairs from flags (skipping valueless modifiers such as INDEXMISSING), and log a warning when a vector field is skipped. Fixes #712 --- redisvl/redis/connection.py | 22 +++++++++ tests/unit/test_convert_index_info.py | 64 +++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index f8d53d2f..0a9b1c31 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -318,6 +318,10 @@ def normalize_index_definition(index_info: dict[str, Any]) -> dict[str, Any]: return _normalize_index_info_mapping(index_info.get("index_definition")) +# FT.INFO field modifiers that appear in "flags" without a value. +_VECTOR_VALUELESS_FLAGS = {"INDEXMISSING", "INDEXEMPTY", "SORTABLE", "NOINDEX", "UNF"} + + def normalize_index_fields(index_info: dict[str, Any]) -> list[dict[str, Any]]: """Return FT.INFO field entries in mapping form.""" return [ @@ -373,6 +377,18 @@ def parse_vector_attrs(attrs): for key, value in attrs.items() if key not in {"identifier", "attribute", "type", "flags"} } + # redis-py 8 on RESP2 with legacy_responses=False puts the vector + # params in "flags" as key/value pairs, possibly followed by + # valueless modifiers: ["algorithm", "FLAT", "dim", 4, "INDEXMISSING"] + flags = attrs.get("flags") or [] + i = 0 + while i < len(flags): + key = str(flags[i]) + if key.upper() in _VECTOR_VALUELESS_FLAGS or i + 1 >= len(flags): + i += 1 + continue + vector_attrs.setdefault(key.lower(), flags[i + 1]) + i += 2 else: # Check if we have any attributes beyond the type declaration if len(attrs) <= 6: @@ -604,6 +620,12 @@ def parse_attrs(attrs, field_type=None): if attrs is None: # Vector field attributes cannot be parsed on this Redis version # Skip this field - it cannot be properly reconstructed + logger.warning( + "Skipping vector field %r of index %r: could not parse its " + "attributes from FT.INFO", + name, + index_name, + ) continue field["attrs"] = attrs else: diff --git a/tests/unit/test_convert_index_info.py b/tests/unit/test_convert_index_info.py index 53f0eeba..d26605ab 100644 --- a/tests/unit/test_convert_index_info.py +++ b/tests/unit/test_convert_index_info.py @@ -241,3 +241,67 @@ def test_convert_index_info_default_stopwords(): assert result["index"]["name"] == "test_default_stopwords" assert "stopwords" not in result["index"] # Should not be present + + +def test_convert_index_info_resp2_unified_vector_flags(): + """Vector params packed into ``flags`` (redis-py 8, RESP2, legacy_responses=False).""" + index_info = { + "index_name": "test_index", + "index_definition": {"key_type": "HASH", "prefixes": ["doc"]}, + "attributes": [ + { + "identifier": "emb", + "attribute": "emb", + "type": "VECTOR", + "flags": [ + "algorithm", + "HNSW", + "data_type", + "FLOAT32", + "dim", + 8, + "distance_metric", + "L2", + "M", + 24, + "ef_construction", + 300, + "INDEXMISSING", + ], + } + ], + } + + result = convert_index_info_to_schema(index_info) + + assert result["fields"] == [ + { + "name": "emb", + "type": "vector", + "attrs": { + "dims": 8, + "distance_metric": "l2", + "algorithm": "hnsw", + "datatype": "float32", + "m": 24, + "ef_construction": 300, + }, + } + ] + + +def test_convert_index_info_warns_on_unparseable_vector_field(caplog): + """A vector field without params is skipped with a warning, not silently.""" + index_info = { + "index_name": "test_index", + "index_definition": {"key_type": "HASH", "prefixes": ["doc"]}, + "attributes": [ + {"identifier": "emb", "attribute": "emb", "type": "VECTOR", "flags": []} + ], + } + + with caplog.at_level("WARNING"): + result = convert_index_info_to_schema(index_info) + + assert result["fields"] == [] + assert "'emb'" in caplog.text and "'test_index'" in caplog.text From d4d7a2e32274b53b46187d4c5ba31eafd05b1543 Mon Sep 17 00:00:00 2001 From: desty Date: Fri, 16 Oct 2026 14:33:23 +0900 Subject: [PATCH 2/2] fix: harden RESP2 unified vector flag parsing per review - Pair only known vector parameter keys from flags, so an unknown bare token no longer shifts the remaining key/value pairs. - Treat an unparseable dim/dims as missing, so the field is skipped with a warning instead of raising out of from_existing(). - Name the consequence in the skip warning. - Exercise protocol=2 + legacy_responses=False in the existing from_existing integration tests (sync and async). - Note why INDEXMISSING is not recovered on vector fields. --- redisvl/redis/connection.py | 54 +++++++++++++++---- tests/integration/test_async_search_index.py | 8 +++ tests/integration/test_search_index.py | 8 +++ tests/unit/test_convert_index_info.py | 56 ++++++++++++++++++++ 4 files changed, 116 insertions(+), 10 deletions(-) diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 0a9b1c31..fba7f194 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -318,8 +318,32 @@ def normalize_index_definition(index_info: dict[str, Any]) -> dict[str, Any]: return _normalize_index_info_mapping(index_info.get("index_definition")) -# FT.INFO field modifiers that appear in "flags" without a value. -_VECTOR_VALUELESS_FLAGS = {"INDEXMISSING", "INDEXEMPTY", "SORTABLE", "NOINDEX", "UNF"} +# Vector parameter names that parse_vector_attrs reads from FT.INFO. In a +# RESP2 unified ``flags`` list, only these are paired with the next token, so +# an unknown bare token costs one position instead of shifting every pair. +_VECTOR_PARAM_KEYS = frozenset( + { + "algorithm", + "data_type", + "datatype", + "type", + "dim", + "dims", + "distance_metric", + "initial_cap", + "block_size", + "m", + "ef_construction", + "ef_runtime", + "epsilon", + "compression", + "reduce", + "graph_max_degree", + "construction_window_size", + "search_window_size", + "training_threshold", + } +) def normalize_index_fields(index_info: dict[str, Any]) -> list[dict[str, Any]]: @@ -383,11 +407,11 @@ def parse_vector_attrs(attrs): flags = attrs.get("flags") or [] i = 0 while i < len(flags): - key = str(flags[i]) - if key.upper() in _VECTOR_VALUELESS_FLAGS or i + 1 >= len(flags): + key = str(flags[i]).lower() + if key not in _VECTOR_PARAM_KEYS or i + 1 >= len(flags): i += 1 continue - vector_attrs.setdefault(key.lower(), flags[i + 1]) + vector_attrs.setdefault(key, flags[i + 1]) i += 2 else: # Check if we have any attributes beyond the type declaration @@ -424,10 +448,18 @@ def parse_vector_attrs(attrs): normalized = {} # Handle dims/dim field - REQUIRED for vector fields - if "dim" in vector_attrs: - normalized["dims"] = int(vector_attrs.pop("dim")) - elif "dims" in vector_attrs: - normalized["dims"] = int(vector_attrs["dims"]) + if "dim" in vector_attrs or "dims" in vector_attrs: + raw_dims = ( + vector_attrs.pop("dim") + if "dim" in vector_attrs + else vector_attrs["dims"] + ) + try: + normalized["dims"] = int(raw_dims) + except (ValueError, TypeError): + # Unparseable dims: fall through to the required-dims check + # below, which skips the field with a warning. + pass else: # If dims is missing from normal parsing, try scanning the raw attrs # This handles edge cases where the format is unexpected @@ -622,7 +654,9 @@ def parse_attrs(attrs, field_type=None): # Skip this field - it cannot be properly reconstructed logger.warning( "Skipping vector field %r of index %r: could not parse its " - "attributes from FT.INFO", + "attributes from FT.INFO. The recovered schema is incomplete, " + "and recreating the index from it (e.g. create(overwrite=True)) " + "drops this field.", name, index_name, ) diff --git a/tests/integration/test_async_search_index.py b/tests/integration/test_async_search_index.py index ed2109f2..3504832a 100644 --- a/tests/integration/test_async_search_index.py +++ b/tests/integration/test_async_search_index.py @@ -547,6 +547,14 @@ async def test_batch_search(async_index): reason="legacy_responses requires redis-py 8", ), ), + pytest.param( + {"protocol": 2, "legacy_responses": False}, + id="resp2-new-response-format", + marks=pytest.mark.skipif( + int(redis.__version__.split(".")[0]) < 8, + reason="legacy_responses requires redis-py 8", + ), + ), ], ) async def test_client_from_existing_and_batch_search( diff --git a/tests/integration/test_search_index.py b/tests/integration/test_search_index.py index 783d328d..38f03918 100644 --- a/tests/integration/test_search_index.py +++ b/tests/integration/test_search_index.py @@ -610,6 +610,14 @@ def test_batch_search(index): reason="legacy_responses requires redis-py 8", ), ), + pytest.param( + {"protocol": 2, "legacy_responses": False}, + id="resp2-new-response-format", + marks=pytest.mark.skipif( + int(redis.__version__.split(".")[0]) < 8, + reason="legacy_responses requires redis-py 8", + ), + ), ], ) def test_client_from_existing_and_batch_search( diff --git a/tests/unit/test_convert_index_info.py b/tests/unit/test_convert_index_info.py index d26605ab..b08b7b44 100644 --- a/tests/unit/test_convert_index_info.py +++ b/tests/unit/test_convert_index_info.py @@ -274,6 +274,9 @@ def test_convert_index_info_resp2_unified_vector_flags(): result = convert_index_info_to_schema(index_info) + # INDEXMISSING is deliberately not recovered as ``index_missing``: a vector + # field can't be recreated with it yet, so restoring it here would turn a + # quiet loss into a failure on ``create()``. assert result["fields"] == [ { "name": "emb", @@ -305,3 +308,56 @@ def test_convert_index_info_warns_on_unparseable_vector_field(caplog): assert result["fields"] == [] assert "'emb'" in caplog.text and "'test_index'" in caplog.text + + +def test_convert_index_info_resp2_unified_flags_skip_unknown_tokens(): + """An unknown bare token in ``flags`` does not shift the key/value pairs.""" + index_info = { + "index_name": "test_index", + "index_definition": {"key_type": "HASH", "prefixes": ["doc"]}, + "attributes": [ + { + "identifier": "emb", + "attribute": "emb", + "type": "VECTOR", + "flags": [ + "algorithm", + "FLAT", + "data_type", + "FLOAT32", + "dim", + 4, + "NEWMOD", + "distance_metric", + "IP", + ], + } + ], + } + + result = convert_index_info_to_schema(index_info) + + assert result["fields"][0]["attrs"]["distance_metric"] == "ip" + assert result["fields"][0]["attrs"]["dims"] == 4 + + +def test_convert_index_info_skips_vector_field_with_unparseable_dims(caplog): + """A non-numeric ``dim`` skips the field with a warning instead of raising.""" + index_info = { + "index_name": "test_index", + "index_definition": {"key_type": "HASH", "prefixes": ["doc"]}, + "attributes": [ + { + "identifier": "emb", + "attribute": "emb", + "type": "VECTOR", + "flags": ["algorithm", "FLAT", "dim", "FUTUREMOD"], + } + ], + } + + with caplog.at_level("WARNING"): + result = convert_index_info_to_schema(index_info) + + assert result["fields"] == [] + assert "'emb'" in caplog.text