Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to Pydantic V2 models #281

Merged
merged 7 commits into from
Feb 14, 2025
Merged

Upgrade to Pydantic V2 models #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a29fe84
pydantic facelift to v2
tylerhutcherson Feb 12, 2025
9a8074f
use an enum for field type literals
tylerhutcherson Feb 12, 2025
134e9e3
refactor vectorizers to work with new pydantic base model
tylerhutcherson Feb 13, 2025
f521060
refactor the rerankers to conform to new pydantic
tylerhutcherson Feb 13, 2025
efac198
fix remaining issues
tylerhutcherson Feb 13, 2025
107ec3a
fix pydantic model_validate
tylerhutcherson Feb 14, 2025
3f10f1c
fix notebook
tylerhutcherson Feb 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 118 additions & 44 deletions docs/user_guide/01_getting_started.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,20 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"<redisvl.index.index.SearchIndex at 0x10faca900>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from redis import Redis\n",
"\n",
Expand All @@ -238,7 +249,7 @@
{
"data": {
"text/plain": [
"<redisvl.index.index.SearchIndex at 0x7f8670a51190>"
"<redisvl.index.index.SearchIndex at 0x10faca900>"
]
},
"execution_count": 5,
Expand Down Expand Up @@ -293,8 +304,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m11:53:23\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
"\u001b[32m11:53:23\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. user_simple\n"
"\u001b[32m11:50:15\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
"\u001b[32m11:50:15\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. user_simple\n"
]
}
],
Expand All @@ -320,15 +331,15 @@
"│ user_simple │ HASH │ ['user_simple_docs'] │ [] │ 0 │\n",
"╰──────────────┴────────────────┴──────────────────────┴─────────────────┴────────────╯\n",
"Index Fields:\n",
"╭────────────────┬────────────────┬─────────┬────────────────┬────────────────╮\n",
"│ Name │ Attribute │ Type │ Field Option │ Option Value │\n",
"├────────────────┼────────────────┼─────────┼────────────────┼────────────────┤\n",
"│ user │ user │ TAG │ SEPARATOR │ , │\n",
"│ credit_score │ credit_score │ TAG │ SEPARATOR │ , │\n",
"│ job │ job │ TEXT │ WEIGHT │ 1 │\n",
"│ age │ age │ NUMERIC │ │ │\n",
"│ user_embedding │ user_embedding │ VECTOR │ │ │\n",
"╰────────────────┴────────────────┴─────────┴────────────────┴────────────────╯\n"
"╭────────────────┬────────────────┬─────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬─────────────────┬────────────────╮\n",
"│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n",
"├────────────────┼────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────────────────┼────────────────┤\n",
"│ user │ user │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n",
"│ credit_score │ credit_score │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n",
"│ job │ job │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n",
"│ age │ age │ NUMERIC │ │ │ │ │ │ │ │ │\n",
"│ user_embedding │ user_embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 3 │ distance_metric │ COSINE │\n",
"╰────────────────┴────────────────┴─────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴─────────────────┴────────────────╯\n"
]
}
],
Expand All @@ -354,7 +365,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"['user_simple_docs:d424b73c516442f7919cc11ed3bb1882', 'user_simple_docs:6da16f88342048e79b3500bec5448805', 'user_simple_docs:ef5a590ef85e4d4888fd8ebe79ae1e8c']\n"
"['user_simple_docs:01JM2NWFWNH0BNA640MT5DS8BD', 'user_simple_docs:01JM2NWFWNF4S2V4E4HYG25CVA', 'user_simple_docs:01JM2NWFWNBFXJJ4PV9F4KMJSE']\n"
]
}
],
Expand Down Expand Up @@ -388,7 +399,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"['user_simple_docs:9806a362604f4700b17513cc94fcf10d']\n"
"['user_simple_docs:01JM2NWJGYMJ0QTR5YB4MB0BX9']\n"
]
}
],
Expand Down Expand Up @@ -476,9 +487,50 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 13,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'index': {'name': 'user_simple', 'prefix': 'user_simple_docs'},\n",
" 'fields': [{'name': 'user', 'type': 'tag'},\n",
" {'name': 'credit_score', 'type': 'tag'},\n",
" {'name': 'job', 'type': 'text'},\n",
" {'name': 'age', 'type': 'numeric'},\n",
" {'name': 'user_embedding',\n",
" 'type': 'vector',\n",
" 'attrs': {'dims': 3,\n",
" 'distance_metric': 'cosine',\n",
" 'algorithm': 'flat',\n",
" 'datatype': 'float32'}}]}"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"schema"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<redisvl.index.index.AsyncSearchIndex at 0x10facacf0>"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from redisvl.index import AsyncSearchIndex\n",
"from redis.asyncio import Redis\n",
Expand All @@ -491,7 +543,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 15,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -532,7 +584,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -620,24 +672,24 @@
"│ Stat Key │ Value │\n",
"├─────────────────────────────┼─────────────┤\n",
"│ num_docs │ 4 │\n",
"│ num_terms │ 0 │\n",
"│ num_terms │ 4 │\n",
"│ max_doc_id │ 4 │\n",
"│ num_records │ 20 │\n",
"│ num_records │ 22 │\n",
"│ percent_indexed │ 1 │\n",
"│ hash_indexing_failures │ 0 │\n",
"│ number_of_uses │ 2 │\n",
"│ bytes_per_record_avg │ 1 │\n",
"│ doc_table_size_mb │ 0.00044632 │\n",
"│ inverted_sz_mb │ 1.90735e-05 │\n",
"│ key_table_size_mb │ 0.000138283 │\n",
"│ offset_bits_per_record_avg │ nan │\n",
"│ offset_vectors_sz_mb │ 0 │\n",
"│ offsets_per_term_avg │ 0 │\n",
"│ records_per_doc_avg │ 5 │\n",
"│ number_of_uses │ 5 │\n",
"│ bytes_per_record_avg │ 50.9091 │\n",
"│ doc_table_size_mb │ 0.000423431 │\n",
"│ inverted_sz_mb │ 0.00106812 │\n",
"│ key_table_size_mb │ 0.000165939 │\n",
"│ offset_bits_per_record_avg │ 8 │\n",
"│ offset_vectors_sz_mb │ 5.72205e-06 │\n",
"│ offsets_per_term_avg │ 0.272727 │\n",
"│ records_per_doc_avg │ 5.5 │\n",
"│ sortable_values_size_mb │ 0 │\n",
"│ total_indexing_time │ 1.796 │\n",
"│ total_inverted_index_blocks │ 11 │\n",
"│ vector_index_sz_mb │ 0.235603 │\n",
"│ total_indexing_time │ 0.197 │\n",
"│ total_inverted_index_blocks │ 12 │\n",
"│ vector_index_sz_mb │ 0.0201416 │\n",
"╰─────────────────────────────┴─────────────╯\n"
]
}
Expand All @@ -657,7 +709,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"Below we will clean up after our work. First, you can optionally flush all data from Redis associated with the index by\n",
"Below we will clean up after our work. First, you can flush all data from Redis associated with the index by\n",
"using the `.clear()` method. This will leave the secondary index in place for future insertions or updates.\n",
"\n",
"But if you want to clean up everything, including the index, just use `.delete()`\n",
Expand All @@ -666,31 +718,53 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 19,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"4"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# (optionally) clear all data from Redis associated with the index\n",
"# Clear all data from Redis associated with the index\n",
"await index.clear()"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 20,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# but the index is still in place\n",
"# Butm the index is still in place\n",
"await index.exists()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"# remove / delete the index in its entirety\n",
"# Remove / delete the index in its entirety\n",
"await index.delete()"
]
}
Expand All @@ -711,7 +785,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.9"
"version": "3.13.2"
},
"orig_nbformat": 4
},
Expand Down
4 changes: 2 additions & 2 deletions docs/user_guide/08_semantic_router.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@
"source": [
"router2 = SemanticRouter.from_dict(router.to_dict(), redis_url=\"redis://localhost:6379\")\n",
"\n",
"assert router2 == router"
"assert router2.to_dict() == router.to_dict()"
]
},
{
Expand Down Expand Up @@ -449,7 +449,7 @@
"source": [
"router3 = SemanticRouter.from_yaml(\"router.yaml\", redis_url=\"redis://localhost:6379\")\n",
"\n",
"assert router3 == router2 == router"
"assert router3.to_dict() == router2.to_dict() == router.to_dict()"
]
},
{
Expand Down
45 changes: 23 additions & 22 deletions redisvl/extensions/llmcache/schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import Any, Dict, List, Optional

from pydantic.v1 import BaseModel, Field, root_validator, validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

from redisvl.extensions.constants import (
CACHE_VECTOR_FIELD_NAME,
Expand Down Expand Up @@ -34,22 +34,23 @@ class CacheEntry(BaseModel):
filters: Optional[Dict[str, Any]] = Field(default=None)
"""Optional filter data stored on the cache entry for customizing retrieval"""

@root_validator(pre=True)
@model_validator(mode="before")
@classmethod
def generate_id(cls, values):
# Ensure entry_id is set
if not values.get("entry_id"):
values["entry_id"] = hashify(values["prompt"], values.get("filters"))
return values

@validator("metadata")
@field_validator("metadata")
@classmethod
def non_empty_metadata(cls, v):
if v is not None and not isinstance(v, dict):
raise TypeError("Metadata must be a dictionary.")
return v

def to_dict(self, dtype: str) -> Dict:
data = self.dict(exclude_none=True)
data = self.model_dump(exclude_none=True)
data["prompt_vector"] = array_to_buffer(self.prompt_vector, dtype)
if self.metadata is not None:
data["metadata"] = serialize(self.metadata)
Expand Down Expand Up @@ -79,33 +80,33 @@ class CacheHit(BaseModel):
filters: Optional[Dict[str, Any]] = Field(default=None)
"""Optional filter data stored on the cache entry for customizing retrieval"""

@root_validator(pre=True)
# Allow extra fields to simplify handling filters
model_config = ConfigDict(extra="allow")

@model_validator(mode="before")
@classmethod
def validate_cache_hit(cls, values):
def validate_cache_hit(cls, values: Dict[str, Any]) -> Dict[str, Any]:
# Deserialize metadata if necessary
if "metadata" in values and isinstance(values["metadata"], str):
values["metadata"] = deserialize(values["metadata"])

# Separate filters from other fields
known_fields = set(cls.__fields__.keys())
filters = {k: v for k, v in values.items() if k not in known_fields}

# Add filters to values
if filters:
values["filters"] = filters

# Remove filter fields from the main values
for k in filters:
values.pop(k)
# Collect any extra fields and store them as filters
extra_data = values.pop("__pydantic_extra__", {}) or {}
if extra_data:
current_filters = values.get("filters") or {}
if not isinstance(current_filters, dict):
current_filters = {}
current_filters.update(extra_data)
values["filters"] = current_filters

return values

def to_dict(self) -> Dict:
data = self.dict(exclude_none=True)
if self.filters:
data.update(self.filters)
def to_dict(self) -> Dict[str, Any]:
"""Convert this model to a dictionary, merging filters into the result."""
data = self.model_dump(exclude_none=True)
if data.get("filters"):
data.update(data["filters"])
del data["filters"]

return data


Expand Down
4 changes: 2 additions & 2 deletions redisvl/extensions/llmcache/semantic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ def __init__(

# Create semantic cache schema and index
schema = SemanticCacheIndexSchema.from_params(
name, prefix, vectorizer.dims, vectorizer.dtype
name, prefix, vectorizer.dims, vectorizer.dtype # type: ignore
)
schema = self._modify_schema(schema, filterable_fields)
self._index = SearchIndex(schema=schema)
Expand All @@ -141,7 +141,7 @@ def __init__(
existing_index = SearchIndex.from_existing(
name, redis_client=self._index.client
)
if existing_index.schema != self._index.schema:
if existing_index.schema.to_dict() != self._index.schema.to_dict():
raise ValueError(
f"Existing index {name} schema does not match the user provided schema for the semantic cache. "
"If you wish to overwrite the index schema, set overwrite=True during initialization."
Expand Down
Loading