Skip to content

Commit

Permalink
Add ability to have predicates on uuid timestamp
Browse files Browse the repository at this point in the history
  • Loading branch information
@cevian
cevian committed Oct 27, 2023
1 parent 2ae8fa6 commit 9d6f580
Show file tree
Hide file tree
Showing 2 changed files with 133 additions and 47 deletions.
164 changes: 120 additions & 44 deletions nbs/00_vector.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"metadata": {},
"outputs": [],
"source": [
"_ = load_dotenv(find_dotenv())\n",
"_ = load_dotenv(find_dotenv(), override=True)\n",
"service_url = os.environ['TIMESCALE_SERVICE_URL']"
]
},
Expand Down Expand Up @@ -455,15 +455,25 @@
" operator = self.operators_mapping[operator]\n",
" else:\n",
" raise ValueError(\"Invalid clause format\")\n",
" \n",
"\n",
" index = len(params)+1\n",
" param_name = f\"${index}\"\n",
"\n",
" if field == '__uuid_timestamp':\n",
" #convert str to timestamp in the database, it's better at it than python\n",
" if isinstance(value, str):\n",
" where_conditions.append(f\"uuid_timestamp(id) {operator} ({param_name}::text)::timestamptz\")\n",
" else:\n",
" where_conditions.append(f\"uuid_timestamp(id) {operator} {param_name}\")\n",
" params.append(value)\n",
" continue\n",
"\n",
" field_cast = ''\n",
" if isinstance(value, int):\n",
" field_cast = '::int'\n",
" elif isinstance(value, float):\n",
" field_cast = '::numeric' \n",
"\n",
" index = len(params)+1\n",
" param_name = f\"${index}\"\n",
" where_conditions.append(f\"(metadata->>'{field}'){field_cast} {operator} {param_name}\")\n",
" params.append(value) \n",
"\n",
Expand Down Expand Up @@ -830,7 +840,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L321){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L475){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### QueryBuilder.get_create_query\n",
"\n",
Expand All @@ -841,7 +851,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L321){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L475){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### QueryBuilder.get_create_query\n",
"\n",
Expand Down Expand Up @@ -1137,7 +1147,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Async.create_tables\n",
"\n",
Expand All @@ -1148,7 +1158,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Async.create_tables\n",
"\n",
Expand Down Expand Up @@ -1176,7 +1186,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Async.create_tables\n",
"\n",
Expand All @@ -1187,7 +1197,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Async.create_tables\n",
"\n",
Expand Down Expand Up @@ -1227,7 +1237,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L753){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L944){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Async.search\n",
"\n",
Expand All @@ -1250,7 +1260,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L753){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L944){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Async.search\n",
"\n",
Expand Down Expand Up @@ -1490,24 +1500,52 @@
" (uuid_from_time(specific_datetime), {\"key\": \"val\"}, \"the brown fox\", [1.0, 1.2])\n",
"])\n",
"assert not await vec.table_is_empty()\n",
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7)))\n",
"assert len(rec) == 1\n",
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": specific_datetime-timedelta(days=7), \"__end_date\": specific_datetime+timedelta(days=7)})\n",
"assert len(rec) == 1\n",
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": str(specific_datetime-timedelta(days=7)), \"__end_date\": str(specific_datetime+timedelta(days=7))})\n",
"assert len(rec) == 1\n",
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": str(specific_datetime-timedelta(days=7))})\n",
"assert len(rec) == 2\n",
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__end_date\": str(specific_datetime+timedelta(days=7))})\n",
"assert len(rec) == 1\n",
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2)))\n",
"assert len(rec) == 0\n",
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": specific_datetime-timedelta(days=7), \"__end_date\": specific_datetime-timedelta(days=2)})\n",
"assert len(rec) == 0\n",
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": str(specific_datetime-timedelta(days=7)), \"__end_date\": str(specific_datetime-timedelta(days=2))})\n",
"assert len(rec) == 0\n",
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7)))\n",
"assert len(rec) == 2\n",
"\n",
"#check all the possible ways to specify a date range\n",
"async def search_date(start_date, end_date, expected):\n",
" #using uuid_time_filter\n",
" rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date, end_date))\n",
" assert len(rec) == expected\n",
" \n",
" #using filters\n",
" filter = {}\n",
" if start_date is not None:\n",
" filter[\"__start_date\"] = start_date\n",
" if end_date is not None:\n",
" filter[\"__end_date\"] = end_date\n",
" rec = await vec.search([1.0, 2.0], limit=4, filter=filter)\n",
" assert len(rec) == expected\n",
" #using filters with string dates\n",
" filter = {}\n",
" if start_date is not None:\n",
" filter[\"__start_date\"] = str(start_date)\n",
" if end_date is not None:\n",
" filter[\"__end_date\"] = str(end_date)\n",
" rec = await vec.search([1.0, 2.0], limit=4, filter=filter)\n",
" assert len(rec) == expected\n",
" #using predicates\n",
" predicates = []\n",
" if start_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \">=\", start_date))\n",
" if end_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \"<\", end_date))\n",
" rec = await vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
" assert len(rec) == expected\n",
" #using predicates with string dates\n",
" predicates = []\n",
" if start_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \">=\", str(start_date)))\n",
" if end_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \"<\", str(end_date)))\n",
" rec = await vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
" assert len(rec) == expected\n",
"\n",
"await search_date(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7), 1)\n",
"await search_date(specific_datetime-timedelta(days=7), None, 2)\n",
"await search_date(None, specific_datetime+timedelta(days=7), 1)\n",
"await search_date(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2), 0)\n",
"\n",
"#check timedelta handling\n",
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date=specific_datetime, time_delta=timedelta(days=7)))\n",
"assert len(rec) == 1\n",
"#end is exclusive\n",
Expand Down Expand Up @@ -1877,7 +1915,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L955){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1147){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Sync.create_tables\n",
"\n",
Expand All @@ -1888,7 +1926,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L955){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1147){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Sync.create_tables\n",
"\n",
Expand Down Expand Up @@ -1916,7 +1954,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L935){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1127){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Sync.upsert\n",
"\n",
Expand All @@ -1932,7 +1970,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L935){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1127){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Sync.upsert\n",
"\n",
Expand Down Expand Up @@ -1965,7 +2003,7 @@
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1072){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1262){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Sync.search\n",
"\n",
Expand All @@ -1988,7 +2026,7 @@
"text/plain": [
"---\n",
"\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1072){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1262){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Sync.search\n",
"\n",
Expand Down Expand Up @@ -2205,15 +2243,53 @@
" #time in 2018\n",
" (uuid_from_time(specific_datetime), {\"key\": \"val\"}, \"the brown fox\", [1.0, 1.2])\n",
"])\n",
"\n",
"def search_date(start_date, end_date, expected):\n",
" #using uuid_time_filter\n",
" rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date, end_date))\n",
" assert len(rec) == expected\n",
" \n",
" #using filters\n",
" filter = {}\n",
" if start_date is not None:\n",
" filter[\"__start_date\"] = start_date\n",
" if end_date is not None:\n",
" filter[\"__end_date\"] = end_date\n",
" rec = vec.search([1.0, 2.0], limit=4, filter=filter)\n",
" assert len(rec) == expected\n",
" #using filters with string dates\n",
" filter = {}\n",
" if start_date is not None:\n",
" filter[\"__start_date\"] = str(start_date)\n",
" if end_date is not None:\n",
" filter[\"__end_date\"] = str(end_date)\n",
" rec = vec.search([1.0, 2.0], limit=4, filter=filter)\n",
" assert len(rec) == expected\n",
" #using predicates\n",
" predicates = []\n",
" if start_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \">=\", start_date))\n",
" if end_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \"<\", end_date))\n",
" rec = vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
" assert len(rec) == expected\n",
" #using predicates with string dates\n",
" predicates = []\n",
" if start_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \">=\", str(start_date)))\n",
" if end_date is not None:\n",
" predicates.append((\"__uuid_timestamp\", \"<\", str(end_date)))\n",
" rec = vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
" assert len(rec) == expected\n",
"\n",
"assert not vec.table_is_empty()\n",
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7)))\n",
"assert len(rec) == 1\n",
"rec = vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": specific_datetime-timedelta(days=7), \"__end_date\": specific_datetime+timedelta(days=7)})\n",
"assert len(rec) == 1\n",
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2)))\n",
"assert len(rec) == 0\n",
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7)))\n",
"assert len(rec) == 2\n",
"\n",
"search_date(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7), 1)\n",
"search_date(specific_datetime-timedelta(days=7), None, 2)\n",
"search_date(None, specific_datetime+timedelta(days=7), 1)\n",
"search_date(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2), 0)\n",
"\n",
"#check timedelta handling\n",
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date=specific_datetime, time_delta=timedelta(days=7)))\n",
"assert len(rec) == 1\n",
"#end is exclusive\n",
Expand Down
16 changes: 13 additions & 3 deletions timescale_vector/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,15 +351,25 @@ def build_query(self, params: List) -> Tuple[str, List]:
operator = self.operators_mapping[operator]
else:
raise ValueError("Invalid clause format")


index = len(params)+1
param_name = f"${index}"

if field == '__uuid_timestamp':
#convert str to timestamp in the database, it's better at it than python
if isinstance(value, str):
where_conditions.append(f"uuid_timestamp(id) {operator} ({param_name}::text)::timestamptz")
else:
where_conditions.append(f"uuid_timestamp(id) {operator} {param_name}")
params.append(value)
continue

field_cast = ''
if isinstance(value, int):
field_cast = '::int'
elif isinstance(value, float):
field_cast = '::numeric'

index = len(params)+1
param_name = f"${index}"
where_conditions.append(f"(metadata->>'{field}'){field_cast} {operator} {param_name}")
params.append(value)

Expand Down

0 comments on commit 9d6f580

Please sign in to comment.