diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 011659e0a6..88870be860 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -342,6 +342,8 @@ jobs: - uses: actions/checkout@v3 - name: Test building docs + env: + VALIDATE_NOTEBOOK_EXECUTION: 0 # TODO: Enable once kernel setup is fixed run: | cd docs && ./ci.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ba16a4ea..21a869af46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,29 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Dev +### Added +* GFQL temporal predicates and type system for date/time comparisons + * Support for datetime, date, and time comparisons with operators: `gt`, `lt`, `ge`, `le`, `eq`, `ne`, `between`, `is_in` + * Proper timezone handling for datetime comparisons + * Type-safe temporal value handling with TypeGuard annotations + * Temporal value classes: `DateTimeValue`, `DateValue`, `TimeValue` for explicit temporal types + * Wire protocol support for JSON serialization of temporal predicates + * Comprehensive documentation: datetime filtering guide, wire protocol reference, and examples notebook +* Documentation build improvements: + * Added notebook CI testing infrastructure for validating example notebooks + * Structure validation ensures notebooks have required fields (execution_count) + * Optional execution validation to verify notebooks run without errors + * Currently validates temporal_predicates.ipynb and layout_tree.ipynb + ### Breaking * Plottable is now a Protocol * py.typed added, type checking active on PyGraphistry! * transform() and transform_umap() now require some parameters to be keyword-only +### Fixed +* Fixed PlotterBase.py docstring formatting for spanner and kusto methods +* Fixed temporal_predicates.ipynb notebook structure (removed outputs from markdown cells) + ## [0.38.0 - 2025-06-17] ### Changed diff --git a/demos/gfql/temporal_predicates.ipynb b/demos/gfql/temporal_predicates.ipynb new file mode 100644 index 0000000000..877306dd09 --- /dev/null +++ b/demos/gfql/temporal_predicates.ipynb @@ -0,0 +1,450 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# GFQL DateTime Filtering Examples\n\nThis notebook shows how to filter graph data by dates and times using GFQL predicates." + }, + { + "cell_type": "markdown", + "source": "## Table of Contents\n\n**Key Temporal Filtering Concepts:**\n\n1. **Basic DateTime Filtering** - Filter by specific dates and times\n2. **Date-Only Filtering** - Ignore time components \n3. **Time-of-Day Filtering** - Filter by time patterns\n4. **Complex Temporal Queries** - Combine with other predicates\n5. **Temporal Value Classes** - Explicit temporal objects\n6. **Timezone-Aware Filtering** - Handle timezone conversions\n7. **Chain Operations** - Multi-hop temporal queries\n8. **Wire Protocol Dicts** - JSON-compatible configuration\n\n**Quick Reference:**\n- `gt()`, `lt()`, `ge()`, `le()` - Greater/less than comparisons\n- `between()` - Range queries\n- `is_in()` - Match specific values\n- `DateTimeValue`, `DateValue`, `TimeValue` - Explicit temporal types", + "metadata": {} + }, + { + "cell_type": "code", + "metadata": {}, + "outputs": [], + "source": "# Standard Python datetime imports\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime, date, time, timedelta\nimport pytz\n\n# Graphistry imports\nimport graphistry\nfrom graphistry import n, e_forward, e_reverse, e_undirected\n\n# Temporal predicates\nfrom graphistry.compute import (\n gt, lt, ge, le, eq, ne, between, is_in,\n DateTimeValue, DateValue, TimeValue\n)", + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup: Create Sample Data\n", + "\n", + "Let's create a sample dataset representing a transaction network with temporal data." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate sample transaction data\n", + "np.random.seed(42)\n", + "\n", + "# Create nodes (accounts)\n", + "n_accounts = 100\n", + "accounts_df = pd.DataFrame({\n", + " 'account_id': [f'ACC_{i:04d}' for i in range(n_accounts)],\n", + " 'account_type': np.random.choice(['checking', 'savings', 'business'], n_accounts),\n", + " 'created_date': pd.date_range('2020-01-01', periods=n_accounts, freq='W'),\n", + " 'last_active': pd.date_range('2023-01-01', periods=n_accounts, freq='D') + \n", + " pd.to_timedelta(np.random.randint(0, 365, n_accounts), unit='D')\n", + "})\n", + "\n", + "# Create edges (transactions)\n", + "n_transactions = 500\n", + "transactions_df = pd.DataFrame({\n", + " 'transaction_id': [f'TXN_{i:06d}' for i in range(n_transactions)],\n", + " 'source': np.random.choice(accounts_df['account_id'], n_transactions),\n", + " 'target': np.random.choice(accounts_df['account_id'], n_transactions),\n", + " 'amount': np.random.exponential(100, n_transactions).round(2),\n", + " 'timestamp': pd.date_range('2023-01-01', periods=n_transactions, freq='H') + \n", + " pd.to_timedelta(np.random.randint(0, 8760, n_transactions), unit='H'),\n", + " 'transaction_time': [time(np.random.randint(0, 24), np.random.randint(0, 60)) \n", + " for _ in range(n_transactions)],\n", + " 'transaction_type': np.random.choice(['transfer', 'payment', 'deposit'], n_transactions)\n", + "})\n", + "\n", + "print(f\"Created {len(accounts_df)} accounts and {len(transactions_df)} transactions\")\n", + "print(f\"\\nTransaction date range: {transactions_df['timestamp'].min()} to {transactions_df['timestamp'].max()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Create graphistry instance\n", + "g = graphistry.edges(transactions_df, 'source', 'target').nodes(accounts_df, 'account_id')\n", + "print(f\"Graph: {len(g._nodes)} nodes, {len(g._edges)} edges\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Basic DateTime Filtering\n", + "\n", + "Filter transactions based on datetime values using edge predicates." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# Filter transactions after a specific date\n", + "# First, filter the edges directly\n", + "cutoff_date = datetime(2023, 7, 1)\n", + "recent_edges = g._edges[gt(pd.Timestamp(cutoff_date))(g._edges['timestamp'])]\n", + "recent_g = g.edges(recent_edges)\n", + "\n", + "print(f\"Transactions after {cutoff_date}: {len(recent_g._edges)}\")\n", + "recent_g._edges[['transaction_id', 'timestamp', 'amount']].head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Alternative: Use chain with edge operations\n", + "# Start from all nodes, then follow edges with temporal filter\n", + "recent_chain = g.chain([\n", + " n(), # Start with all nodes\n", + " e_forward({\n", + " \"timestamp\": gt(pd.Timestamp(cutoff_date))\n", + " })\n", + "])\n", + "\n", + "print(f\"Transactions after {cutoff_date} (chain): {len(recent_chain._edges)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Filter transactions in a specific month\n", + "march_edges = g._edges[\n", + " between(\n", + " datetime(2023, 3, 1),\n", + " datetime(2023, 3, 31, 23, 59, 59)\n", + " )(g._edges['timestamp'])\n", + "]\n", + "march_g = g.edges(march_edges)\n", + "\n", + "print(f\"Transactions in March 2023: {len(march_g._edges)}\")\n", + "march_g._edges[['transaction_id', 'timestamp', 'amount']].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Date-Only Filtering\n", + "\n", + "Filter nodes based on dates, ignoring time components." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Filter accounts created after a specific date\n", + "new_accounts = g.chain([\n", + " n(filter_dict={\n", + " \"created_date\": ge(date(2021, 1, 1))\n", + " })\n", + "])\n", + "\n", + "print(f\"Accounts created after 2021: {len(new_accounts._nodes)}\")\n", + "new_accounts._nodes[['account_id', 'created_date', 'account_type']].head()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Find accounts active in the last 90 days\n", + "ninety_days_ago = datetime.now().date() - timedelta(days=90)\n", + "active_accounts = g.chain([\n", + " n(filter_dict={\n", + " \"last_active\": gt(pd.Timestamp(ninety_days_ago))\n", + " })\n", + "])\n", + "\n", + "print(f\"Recently active accounts: {len(active_accounts._nodes)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Time-of-Day Filtering\n", + "\n", + "Filter transactions based on time of day." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# Find transactions during business hours (9 AM - 5 PM)\n", + "business_hours_edges = g._edges[\n", + " between(\n", + " time(9, 0, 0),\n", + " time(17, 0, 0)\n", + " )(g._edges['transaction_time'])\n", + "]\n", + "business_hours_g = g.edges(business_hours_edges)\n", + "\n", + "print(f\"Business hour transactions: {len(business_hours_g._edges)}\")\n", + "print(f\"Percentage of total: {len(business_hours_g._edges) / len(g._edges) * 100:.1f}%\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Find transactions at specific times (e.g., on the hour)\n", + "on_the_hour_times = [time(h, 0, 0) for h in range(24)]\n", + "on_hour_edges = g._edges[\n", + " is_in(on_the_hour_times)(g._edges['transaction_time'])\n", + "]\n", + "on_hour_g = g.edges(on_hour_edges)\n", + "\n", + "print(f\"Transactions on the hour: {len(on_hour_g._edges)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Complex Temporal Queries\n", + "\n", + "Combine temporal predicates with other filters for complex queries." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# Find large transactions (>$500) in Q4 2023\n", + "q4_mask = between(\n", + " datetime(2023, 10, 1),\n", + " datetime(2023, 12, 31, 23, 59, 59)\n", + ")(g._edges['timestamp'])\n", + "large_mask = gt(500)(g._edges['amount'])\n", + "\n", + "q4_large_edges = g._edges[q4_mask & large_mask]\n", + "q4_large_g = g.edges(q4_large_edges)\n", + "\n", + "print(f\"Large Q4 2023 transactions: {len(q4_large_g._edges)}\")\n", + "if len(q4_large_g._edges) > 0:\n", + " print(f\"Total value: ${q4_large_g._edges['amount'].sum():,.2f}\")\n", + " print(f\"Average: ${q4_large_g._edges['amount'].mean():,.2f}\")" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "outputs": [], + "source": "# Multi-hop query: Find accounts that received money recently\n# and then sent money to business accounts\nthirty_days_ago = datetime.now() - timedelta(days=30)\n\n# First, find recent transactions\nrecent_edges = g._edges[gt(pd.Timestamp(thirty_days_ago))(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\n# Use chain to find money flow pattern\nmoney_flow = recent_g.chain([\n # Start with any node\n n(),\n # Follow incoming edges (as destination)\n e_reverse(),\n # Go to source nodes\n n(),\n # Follow outgoing edges\n e_forward(),\n # To business accounts\n n(filter_dict={\"account_type\": \"business\"})\n])\n\nprint(f\"Money flow pattern found: {len(money_flow._nodes)} business accounts\")", + "execution_count": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Using Temporal Value Classes\n", + "\n", + "Use explicit temporal value classes for more control." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# Create temporal values with specific properties\n", + "dt_value = DateTimeValue(\"2023-06-15T14:30:00\", \"UTC\")\n", + "date_value = DateValue(\"2023-06-15\")\n", + "time_value = TimeValue(\"14:30:00\")\n", + "\n", + "# Use in predicates\n", + "specific_edges = g._edges[gt(dt_value)(g._edges['timestamp'])]\n", + "specific_g = g.edges(specific_edges)\n", + "\n", + "print(f\"Transactions after {dt_value.value}: {len(specific_g._edges)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Timezone-Aware Filtering\n", + "\n", + "Handle timezone-aware datetime comparisons." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# Add timezone info to our data for this example\n", + "transactions_df_tz = transactions_df.copy()\n", + "transactions_df_tz['timestamp_utc'] = pd.to_datetime(transactions_df_tz['timestamp']).dt.tz_localize('UTC')\n", + "transactions_df_tz['timestamp_eastern'] = transactions_df_tz['timestamp_utc'].dt.tz_convert('US/Eastern')\n", + "\n", + "g_tz = graphistry.edges(transactions_df_tz, 'source', 'target')\n", + "\n", + "# Filter using Eastern time\n", + "eastern = pytz.timezone('US/Eastern')\n", + "eastern_cutoff = eastern.localize(datetime(2023, 7, 1, 9, 0, 0)) # 9 AM Eastern\n", + "\n", + "eastern_morning_edges = g_tz._edges[\n", + " gt(pd.Timestamp(eastern_cutoff))(g_tz._edges['timestamp_eastern'])\n", + "]\n", + "eastern_morning_g = g_tz.edges(eastern_morning_edges)\n", + "\n", + "print(f\"Transactions after 9 AM Eastern on July 1, 2023: {len(eastern_morning_g._edges)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Chain Operations with Temporal Edge Filters\n", + "\n", + "Demonstrate using temporal predicates in chain operations with proper edge filtering." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# Find paths through recent high-value transactions\n", + "recent_high_value = g.chain([\n", + " # Start from all nodes\n", + " n(),\n", + " # Follow edges with temporal and amount filters\n", + " e_forward({\n", + " \"timestamp\": gt(datetime.now() - timedelta(days=7)),\n", + " \"amount\": gt(200)\n", + " }),\n", + " # Reach destination nodes\n", + " n()\n", + "])\n", + "\n", + "print(f\"Recent high-value transaction paths:\")\n", + "print(f\" Nodes: {len(recent_high_value._nodes)}\")\n", + "print(f\" Edges: {len(recent_high_value._edges)}\")" + ] + }, + { + "cell_type": "code", + "source": "# Wire protocol dicts in is_in predicates\n# Useful for checking against multiple specific timestamps\n\nimportant_dates = [\n {\"type\": \"datetime\", \"value\": \"2023-01-01T00:00:00\", \"timezone\": \"UTC\"}, # New Year\n {\"type\": \"datetime\", \"value\": \"2023-07-04T00:00:00\", \"timezone\": \"UTC\"}, # July 4th\n {\"type\": \"datetime\", \"value\": \"2023-12-25T00:00:00\", \"timezone\": \"UTC\"}, # Christmas\n]\n\n# Note: This checks for exact timestamp matches\n# For date matching, you'd need to extract the date portion\nholiday_pred = is_in(important_dates)\n\n# For demonstration, let's check if any transactions happened exactly at midnight on these days\n# (In real data, you'd probably want to check date ranges instead)\nprint(f\"Checking for transactions at midnight on holidays...\")\nprint(f\"(This is likely 0 unless transactions were specifically created at midnight)\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "source": "## Summary\n\nThis notebook demonstrated:\n\n1. **DateTime filtering** with `gt`, `lt`, `between` predicates on edges\n2. **Date-only filtering** for day-level granularity on nodes\n3. **Time-of-day filtering** for patterns like business hours\n4. **Complex queries** combining temporal and non-temporal predicates\n5. **Multi-hop queries** with temporal constraints using chain operations\n6. **Temporal value classes** for explicit control\n7. **Timezone-aware** filtering\n8. **Wire protocol dictionaries** for JSON-compatible predicate configuration\n9. **Proper chain syntax** with edge filters in `e_forward()` and node filters in `n()`\n\nKey takeaways:\n- Temporal predicates work seamlessly with pandas datetime types\n- Wire protocol dicts enable configuration-driven filtering: `gt({\"type\": \"datetime\", \"value\": \"2023-01-01T00:00:00\", \"timezone\": \"UTC\"})`\n- Timezone awareness is built-in for accurate cross-timezone comparisons\n- Complex temporal patterns can be expressed through chain operations\n\nTemporal predicates in GFQL provide a powerful way to analyze time-series aspects of graph data, enabling complex temporal queries while maintaining the expressiveness of graph traversals.", + "metadata": {} + }, + { + "cell_type": "code", + "source": "# Build predicates programmatically with wire protocol dicts\ndef create_date_filter(year, month, day, comparison=\"gt\"):\n \"\"\"Create a date filter using wire protocol format\"\"\"\n date_dict = {\n \"type\": \"date\",\n \"value\": f\"{year:04d}-{month:02d}-{day:02d}\"\n }\n \n if comparison == \"gt\":\n return gt(date_dict)\n elif comparison == \"lt\":\n return lt(date_dict)\n elif comparison == \"ge\":\n return ge(date_dict)\n elif comparison == \"le\":\n return le(date_dict)\n else:\n raise ValueError(f\"Unknown comparison: {comparison}\")\n\n# Use the programmatic filter\nfilter_2023 = create_date_filter(2023, 1, 1, \"ge\")\naccounts_2023 = g.chain([\n n(filter_dict={\n \"created_date\": filter_2023\n })\n])\n\nprint(f\"Accounts created in 2023 or later: {len(accounts_2023._nodes)}\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": "# Example: Load predicate configuration from JSON\nimport json\n\n# Simulate loading from a JSON config file\nconfig_json = '''\n{\n \"filters\": {\n \"recent_transactions\": {\n \"timestamp\": {\n \"type\": \"gt\",\n \"value\": {\n \"type\": \"datetime\",\n \"value\": \"2023-10-01T00:00:00\",\n \"timezone\": \"UTC\"\n }\n }\n },\n \"business_hours\": {\n \"transaction_time\": {\n \"type\": \"between\",\n \"start\": {\"type\": \"time\", \"value\": \"09:00:00\"},\n \"end\": {\"type\": \"time\", \"value\": \"17:00:00\"}\n }\n }\n }\n}\n'''\n\nconfig = json.loads(config_json)\n\n# Use the wire protocol dict directly\nrecent_filter = config[\"filters\"][\"recent_transactions\"][\"timestamp\"][\"value\"]\nrecent_edges = g._edges[gt(recent_filter)(g._edges['timestamp'])]\nrecent_g = g.edges(recent_edges)\n\nprint(f\"Recent transactions (from JSON config): {len(recent_g._edges)}\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": "# Wire protocol dictionaries work directly in Python\n# These are equivalent:\npred1 = gt(pd.Timestamp(\"2023-07-01\"))\npred2 = gt({\"type\": \"datetime\", \"value\": \"2023-07-01T00:00:00\", \"timezone\": \"UTC\"})\n\n# Test they produce the same results\nresult1 = pred1(g._edges['timestamp'])\nresult2 = pred2(g._edges['timestamp'])\nprint(f\"Results are identical: {result1.equals(result2)}\")\nprint(f\"Transactions after July 1, 2023: {result1.sum()}\")", + "metadata": {}, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "source": "## 8. Using Wire Protocol Dictionaries\n\nYou can pass wire protocol dictionaries directly to temporal predicates. This is useful for:\n- Loading predicate configurations from JSON files\n- Building predicates programmatically\n- Sharing predicate definitions between systems", + "metadata": {} + }, + { + "cell_type": "markdown", + "source": "## What's Next?\n\n- **[Datetime Filtering Guide](../../gfql/datetime_filtering.html)** - Full temporal predicate reference\n- **[Wire Protocol Reference](../../gfql/wire_protocol_examples.html)** - JSON serialization examples\n- **[GFQL Documentation](../../gfql/index.html)** - Complete GFQL reference", + "metadata": {} + }, + { + "cell_type": "code", + "metadata": {}, + "outputs": [], + "source": "# Complex multi-hop with temporal constraints\n# Find 2-hop paths through recent transactions\ntwo_hop_recent = g.chain([\n # Start from business accounts\n n(filter_dict={\"account_type\": \"business\"}),\n # First hop: recent outgoing transactions\n e_forward({\n \"timestamp\": gt(datetime.now() - timedelta(days=30))\n }, name=\"hop1\"),\n # Intermediate nodes\n n(),\n # Second hop: any transaction\n e_forward(name=\"hop2\"),\n # Final nodes\n n()\n])\n\nprint(f\"2-hop paths from business accounts through recent transactions:\")\nprint(f\" Total edges: {len(two_hop_recent._edges)}\")\nprint(f\" Hop 1 edges: {two_hop_recent._edges['hop1'].sum()}\")\nprint(f\" Hop 2 edges: {two_hop_recent._edges['hop2'].sum()}\")", + "execution_count": 16 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "This notebook demonstrated:\n", + "\n", + "1. **DateTime filtering** with `gt`, `lt`, `between` predicates on edges\n", + "2. **Date-only filtering** for day-level granularity on nodes\n", + "3. **Time-of-day filtering** for patterns like business hours\n", + "4. **Complex queries** combining temporal and non-temporal predicates\n", + "5. **Multi-hop queries** with temporal constraints using chain operations\n", + "6. **Temporal value classes** for explicit control\n", + "7. **Timezone-aware** filtering\n", + "8. **Proper chain syntax** with edge filters in `e_forward()` and node filters in `n()`\n", + "\n", + "Temporal predicates in GFQL provide a powerful way to analyze time-series aspects of graph data, enabling complex temporal queries while maintaining the expressiveness of graph traversals." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/docs/ci.sh b/docs/ci.sh index f79907e01c..e66d93ad67 100755 --- a/docs/ci.sh +++ b/docs/ci.sh @@ -6,5 +6,8 @@ DOCS_FORMAT=${DOCS_FORMAT:-all} # Default to building all formats if not specif ( cd docker \ && docker compose build \ - && docker compose run --rm -e DOCS_FORMAT=$DOCS_FORMAT sphinx + && docker compose run --rm \ + -e DOCS_FORMAT=$DOCS_FORMAT \ + -e VALIDATE_NOTEBOOK_EXECUTION=${VALIDATE_NOTEBOOK_EXECUTION:-0} \ + sphinx ) diff --git a/docs/docker/Dockerfile b/docs/docker/Dockerfile index 6132b9d19a..bf205bbb28 100644 --- a/docs/docker/Dockerfile +++ b/docs/docker/Dockerfile @@ -16,9 +16,10 @@ RUN --mount=type=cache,target=/var/cache/apt \ # Step 2: Copy project files needed for pip install (setup.py, setup.cfg, versioneer.py, README.md) COPY ../../setup.py ../../setup.cfg ../../versioneer.py ../../README.md ./ -# Step 3: Install the project dependencies with pip, including docs extras +# Step 3: Install the project dependencies with pip, including docs extras and register kernel RUN --mount=type=cache,target=/root/.cache/pip \ - python3 -m pip install -e .[docs] + python3 -m pip install -e .[docs] && \ + python3 -m ipykernel install --sys-prefix --name python3 # Step 4: Copy the graphistry source files into the container COPY ../../graphistry /graphistry diff --git a/docs/docker/build-docs.sh b/docs/docker/build-docs.sh index 6925a93557..5676ff3bda 100755 --- a/docs/docker/build-docs.sh +++ b/docs/docker/build-docs.sh @@ -1,6 +1,51 @@ -#!/bin/sh +#!/bin/bash set -ex +# Validate notebooks before building docs +NOTEBOOKS_TO_VALIDATE=( + "/docs/source/demos/gfql/temporal_predicates.ipynb" + "/docs/source/demos/more_examples/graphistry_features/layout_tree.ipynb" +) + +for notebook in "${NOTEBOOKS_TO_VALIDATE[@]}"; do +if [ -f "$notebook" ]; then + echo "Validating $(basename $notebook) structure..." + python3 -c " +import json +import sys + +notebook_path = '$notebook' +with open(notebook_path, 'r') as f: + nb = json.load(f) + +# Check for missing execution_count in code cells +errors = [] +for i, cell in enumerate(nb['cells']): + if cell.get('cell_type') == 'code' and 'execution_count' not in cell: + errors.append(f'Cell {i} missing execution_count') + +if errors: + print('Notebook validation errors:') + for err in errors: + print(f' {err}') + sys.exit(1) +else: + print('Notebook structure validation passed') +" + + # Optionally execute notebook to verify it runs without errors + if [ "${VALIDATE_NOTEBOOK_EXECUTION:-0}" = "1" ]; then + echo "Executing $(basename $notebook) to verify it runs..." + python3 -m nbconvert --to notebook --execute \ + --ExecutePreprocessor.timeout=600 \ + --output /tmp/executed_notebook.ipynb \ + "$notebook" && \ + rm -f /tmp/executed_notebook.ipynb && \ + echo "$(basename $notebook) execution completed successfully" + fi +fi +done + build_html() { sphinx-build -b html -d /docs/doctrees . /docs/_build/html } diff --git a/docs/source/api/gfql/predicates.rst b/docs/source/api/gfql/predicates.rst index fedacd84df..039d793edb 100644 --- a/docs/source/api/gfql/predicates.rst +++ b/docs/source/api/gfql/predicates.rst @@ -53,3 +53,12 @@ Temporal :undoc-members: :show-inheritance: :noindex: + +Temporal Values +--------------- + +.. automodule:: graphistry.compute.ast_temporal + :members: + :undoc-members: + :show-inheritance: + :noindex: diff --git a/docs/source/gfql/datetime_filtering.md b/docs/source/gfql/datetime_filtering.md new file mode 100644 index 0000000000..7416282a33 --- /dev/null +++ b/docs/source/gfql/datetime_filtering.md @@ -0,0 +1,369 @@ +# Working with Dates and Times + +GFQL predicates support filtering by datetime, date, and time values. This guide covers common patterns and gotchas when working with temporal data. + +## Required Imports + +```python +# Core imports +import graphistry +from graphistry import n, e_forward, e_reverse, e_undirected + +# Temporal predicates +from graphistry.compute import ( + gt, lt, ge, le, eq, ne, between, is_in, + DateTimeValue, DateValue, TimeValue +) + +# Standard datetime types +import pandas as pd +from datetime import datetime, date, time, timedelta +import pytz # For timezone support +``` + +## Supported Types and Standards + +### Supported Python Types + +- [`pd.Timestamp`](https://pandas.pydata.org/docs/reference/api/pandas.Timestamp.html) - Pandas timestamp +- [`datetime`](https://docs.python.org/3/library/datetime.html#datetime.datetime) - Python datetime +- [`date`](https://docs.python.org/3/library/datetime.html#datetime.date) - Date only (no time) +- [`time`](https://docs.python.org/3/library/datetime.html#datetime.time) - Time only (no date) +- Wire protocol dicts - For ISO strings and JSON compatibility + +```python +# Use datetime objects +gt(pd.Timestamp("2023-01-01 12:00:00")) +between(datetime(2023, 1, 1), datetime(2023, 12, 31)) + +# Wire protocol dicts accept ISO strings +gt({"type": "datetime", "value": "2023-01-01T00:00:00", "timezone": "UTC"}) + +# Raw strings raise ValueError +gt("2023-01-01") # ValueError +``` + +### Creating from Strings + +```python +# Timestamps +pd.Timestamp("2023-01-01T12:00:00Z") # UTC +pd.Timestamp("2023-01-01 12:00:00") # Naive + +# Date/Time objects +date.fromisoformat("2023-01-01") # date(2023, 1, 1) +time.fromisoformat("14:30:00") # time(14, 30, 0) +``` + +### Standards +- [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) datetime strings: `"2023-01-01T12:00:00Z"` +- [IANA timezone](https://www.iana.org/time-zones) names: `"US/Eastern"`, `"UTC"` + +### Wire Protocol Types +For JSON serialization and cross-system compatibility: +- **DateTimeWire**: `{"type": "datetime", "value": "ISO-8601-string", "timezone": "IANA-timezone"}` +- **DateWire**: `{"type": "date", "value": "YYYY-MM-DD"}` +- **TimeWire**: `{"type": "time", "value": "HH:MM:SS[.ffffff]"}` + +Note: The `timezone` field is optional for DateTimeWire and defaults to "UTC" if omitted. + +## Basic Usage + +### Datetime Filtering + +Filter nodes or edges based on datetime values: + +```python +import pandas as pd +from datetime import datetime +from graphistry import n +from graphistry.compute import gt, lt, between + +# Filter nodes created after a specific datetime +recent_nodes = g.chain([ + n(filter_dict={"created_at": gt(pd.Timestamp("2023-01-01 12:00:00"))}) +]) + +# Filter edges within a date range +date_range_edges = g.chain([ + n(edge_match={"timestamp": between( + datetime(2023, 1, 1), + datetime(2023, 12, 31) + )}) +]) +``` + +### Date-Only Filtering + +For date comparisons (ignoring time): + +```python +from datetime import date +from graphistry.compute import eq, ge + +# Filter nodes by exact date +specific_date = g.chain([ + n(filter_dict={"event_date": eq(date(2023, 6, 15))}) +]) + +# Filter nodes on or after a date +after_date = g.chain([ + n(filter_dict={"start_date": ge(date(2023, 1, 1))}) +]) +``` + +### Time-Only Filtering + +Filter based on time of day: + +```python +from datetime import time +from graphistry.compute import is_in, between + +# Filter events at specific times +morning_events = g.chain([ + n(filter_dict={"event_time": is_in([ + time(9, 0, 0), + time(9, 30, 0), + time(10, 0, 0) + ])}) +]) + +# Filter events in time range +business_hours = g.chain([ + n(filter_dict={"timestamp": between( + time(9, 0, 0), + time(17, 0, 0) + )}) +]) +``` + +## Timezone Support + +```python +import pytz + +# Timezone-aware filtering +eastern = pytz.timezone('US/Eastern') +tz_aware_filter = g.chain([ + n(filter_dict={ + "timestamp": gt(pd.Timestamp("2023-01-01 12:00:00", tz=eastern)) + }) +]) +``` + +Comparisons automatically handle timezone conversions. + +## Advanced Usage + +### Mixed Temporal and Non-Temporal Predicates + +Combine temporal predicates with other filters: + +```python +from graphistry.compute import gt, lt, eq + +# Complex filter with multiple conditions +complex_filter = g.chain([ + n(filter_dict={ + "created_at": gt(datetime(2023, 1, 1)), + "status": eq("active"), + "priority": gt(5) + }) +]) +``` + +### Using Wire Protocol Dictionaries Directly + +You can pass wire protocol dictionaries directly to predicates, which is useful for programmatic predicate creation or when working with JSON configurations: + +```python +# Pass wire protocol dictionaries directly +filter_with_dict = g.chain([ + n(filter_dict={"timestamp": gt({ + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "UTC" + })}) +]) + +# Works with all temporal predicates +date_range_filter = g.chain([ + n(filter_dict={"event_date": between( + {"type": "date", "value": "2023-01-01"}, + {"type": "date", "value": "2023-12-31"} + )}) +]) + +# And with is_in for multiple values +time_filter = g.chain([ + n(filter_dict={"event_time": is_in([ + {"type": "time", "value": "09:00:00"}, + {"type": "time", "value": "12:00:00"}, + {"type": "time", "value": "17:00:00"} + ])}) +]) +``` + +This is the same format used by the wire protocol, making it easy to: +- Store predicate configurations in JSON files +- Build predicates programmatically from external data sources +- Share predicate definitions between Python and other systems + +### Temporal Predicates in Multi-Hop Queries + +Use temporal filters in complex graph traversals: + +```python +# Find all transactions after a date, then their related accounts +recent_transactions = g.chain([ + n(filter_dict={"type": eq("transaction"), + "date": gt(date(2023, 6, 1))}), + n(edge_match={"relationship": eq("involves")}), + n(filter_dict={"type": eq("account")}) +]) +``` + +## Temporal Value Classes + +PyGraphistry provides three temporal value classes for precise control: + +### DateTimeValue + +For full datetime with optional timezone: + +```python +from graphistry.compute import DateTimeValue, gt + +# Create datetime value with timezone +dt_value = DateTimeValue("2023-01-01T12:00:00", "US/Eastern") + +# Use in predicate +filter_dt = g.chain([ + n(filter_dict={"timestamp": gt(dt_value)}) +]) +``` + +### DateValue + +For date-only comparisons: + +```python +from graphistry.compute import DateValue, between + +# Create date values +start = DateValue("2023-01-01") +end = DateValue("2023-12-31") + +# Use in between predicate +year_filter = g.chain([ + n(filter_dict={"event_date": between(start, end)}) +]) +``` + +### TimeValue + +For time-of-day comparisons: + +```python +from graphistry.compute import TimeValue, is_in + +# Create time values +morning = TimeValue("09:00:00") +noon = TimeValue("12:00:00") + +# Filter by specific times +time_filter = g.chain([ + n(filter_dict={"daily_event": is_in([morning, noon])}) +]) +``` + +## Best Practices + +1. **Use Explicit Types**: Always use `pd.Timestamp`, `datetime`, `date`, or `time` objects instead of strings to avoid ambiguity. + +2. **Timezone Awareness**: When working with timestamps across timezones, always specify timezones explicitly. + +3. **Performance**: Temporal comparisons are optimized for pandas DataFrames. For large datasets, ensure your datetime columns are properly typed. + +4. **JSON Serialization**: When serializing queries, temporal values are automatically converted to tagged dictionaries that preserve type and timezone information. + +## Unsupported Features + +### Duration/Interval Support +Currently, PyGraphistry does not support duration or interval types (e.g., ISO 8601 durations like "P1D" or "PT2H"). For duration-based queries: + +```python +# Instead of duration literals, calculate explicit timestamps +from datetime import datetime, timedelta + +# Find events within last 7 days +now = datetime.now() +week_ago = now - timedelta(days=7) +recent_events = g.chain([ + n(filter_dict={"timestamp": gt(pd.Timestamp(week_ago))}) +]) + +# For recurring intervals, use multiple conditions +business_days = g.chain([ + n(filter_dict={ + "timestamp": between( + pd.Timestamp("2023-01-01"), + pd.Timestamp("2023-12-31") + ) + }) +]) +``` + +## Common Patterns + +### Filter Recent Data + +```python +from datetime import datetime, timedelta + +# Get data from last 30 days +thirty_days_ago = datetime.now() - timedelta(days=30) +recent_data = g.chain([ + n(filter_dict={"timestamp": gt(pd.Timestamp(thirty_days_ago))}) +]) +``` + +### Business Hours Filtering + +```python +# Filter events during business hours +business_hours = g.chain([ + n(filter_dict={ + "timestamp": between(time(9, 0, 0), time(17, 0, 0)) + }) +]) +``` + +### Quarterly Data Analysis + +```python +# Q1 2023 data +q1_2023 = g.chain([ + n(filter_dict={ + "date": between( + date(2023, 1, 1), + date(2023, 3, 31) + ) + }) +]) +``` + +## Error Handling + +```python +# Strings raise ValueError - always use datetime objects +gt("2023-01-01") # ❌ ValueError +gt(pd.Timestamp("2023-01-01")) # ✓ Correct +``` + +## See Also + +- [GFQL Predicates API Reference](../api/gfql/predicates.rst) +- [GFQL Chain Operations](../api/gfql/chain.rst) +- [Wire Protocol Reference](wire_protocol_examples.md) \ No newline at end of file diff --git a/docs/source/gfql/index.rst b/docs/source/gfql/index.rst index bb6e3b485a..fd21b5f2ba 100644 --- a/docs/source/gfql/index.rst +++ b/docs/source/gfql/index.rst @@ -19,3 +19,5 @@ See also: combo quick predicates/quick + datetime_filtering + wire_protocol_examples diff --git a/docs/source/gfql/overview.rst b/docs/source/gfql/overview.rst index a08a2ae0b4..bbedaf9414 100644 --- a/docs/source/gfql/overview.rst +++ b/docs/source/gfql/overview.rst @@ -94,6 +94,30 @@ Example: Find nodes up to 2 hops away from node `"a"` and label each hop. first_hop_edges = g_2_hops._edges[ g_2_hops._edges["hop1"] == True ] print('Number of first-hop edges:', len(first_hop_edges)) +**Filter by Date/Time** + +Example: Find recent transactions using temporal predicates. + +.. code-block:: python + + from graphistry import n, e_forward + from graphistry.compute import gt, between + from datetime import datetime, date, time + import pandas as pd + + # Find transactions after a specific date + recent = g.chain([ + n(edge_match={"timestamp": gt(pd.Timestamp("2023-01-01"))}) + ]) + + # Find transactions in a date range during business hours + business_hours_txns = g.chain([ + n(edge_match={ + "date": between(date(2023, 6, 1), date(2023, 6, 30)), + "time": between(time(9, 0), time(17, 0)) + }) + ]) + **Query for Transaction Nodes Between Risky Nodes** Example: Find transaction nodes between two kinds of risky nodes. diff --git a/docs/source/gfql/predicates/quick.rst b/docs/source/gfql/predicates/quick.rst index 0fe7f7f09e..7eeda5a7b2 100644 --- a/docs/source/gfql/predicates/quick.rst +++ b/docs/source/gfql/predicates/quick.rst @@ -40,6 +40,15 @@ The following table lists the available operators, their descriptions, and examp - Between ``lower`` and ``upper`` (inclusive). - ``n({ "age": between(18, 65) })`` +.. note:: + All numeric comparison operators (``gt``, ``lt``, ``ge``, ``le``, ``eq``, ``ne``, ``between``) also support temporal values: + + - **DateTime**: ``n({ "created_at": gt(pd.Timestamp("2023-01-01 12:00:00")) })`` + - **Date**: ``n({ "event_date": eq(date(2023, 6, 15)) })`` + - **Time**: ``n({ "daily_time": between(time(9, 0), time(17, 0)) })`` + + See :doc:`/gfql/datetime_filtering` for datetime filtering examples. + **Categorical Operators** .. list-table:: diff --git a/docs/source/gfql/wire_protocol_examples.md b/docs/source/gfql/wire_protocol_examples.md new file mode 100644 index 0000000000..40f507b097 --- /dev/null +++ b/docs/source/gfql/wire_protocol_examples.md @@ -0,0 +1,516 @@ +# Temporal Predicates Wire Protocol Reference + +This document provides a comprehensive reference for how temporal predicates serialize to JSON in the GFQL wire protocol. The wire protocol enables interoperability between Python and other systems. + +## Overview + +The wire protocol uses tagged dictionaries to preserve type information during JSON serialization. This enables: +- Cross-language compatibility +- Configuration-driven predicate creation +- Network transport of queries +- Storage of predicate definitions + +**Key Concept**: Wire protocol dictionaries can be used directly in the Python API: + +```python +# These are equivalent: +pred1 = gt(100) +pred2 = gt(pd.Timestamp("2023-01-01")) +pred3 = gt({"type": "datetime", "value": "2023-01-01T00:00:00", "timezone": "UTC"}) +``` + +## 1. DateTime Comparisons + +### Python API +```python +import pandas as pd +from datetime import datetime +from graphistry import n +from graphistry.compute import gt, between + +# Using pandas Timestamp +filter1 = n(filter_dict={ + "created_at": gt(pd.Timestamp("2023-01-01 12:00:00")) +}) + +# Using Python datetime +filter2 = n(edge_match={ + "timestamp": between( + datetime(2023, 1, 1), + datetime(2023, 12, 31, 23, 59, 59) + ) +}) +``` + +### Wire Protocol (JSON) +```json +// GT with datetime +{ + "type": "ASTNode", + "filter_dict": { + "created_at": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-01-01T12:00:00", + "timezone": "UTC" + } + } + } +} + +// Between with datetime range +{ + "type": "ASTNode", + "edge_match": { + "timestamp": { + "type": "Between", + "lower": { + "type": "datetime", + "value": "2023-01-01T00:00:00", + "timezone": "UTC" + }, + "upper": { + "type": "datetime", + "value": "2023-12-31T23:59:59", + "timezone": "UTC" + }, + "inclusive": true + } + } +} +``` + +### Round-trip Example +```python +# Create predicate +from graphistry.compute import gt +pred = gt(pd.Timestamp("2023-01-01 12:00:00")) + +# Serialize to JSON +json_data = pred.to_json() +print(json_data) +# Output: { +# 'type': 'GT', +# 'val': { +# 'type': 'datetime', +# 'value': '2023-01-01T12:00:00', +# 'timezone': 'UTC' +# } +# } + +# Deserialize from JSON +from graphistry.compute.predicates.numeric import GT +pred2 = GT.from_json(json_data) +# pred2 is functionally equivalent to pred +``` + +## 2. Date-Only Comparisons + +### Python API +```python +from datetime import date +from graphistry.compute import eq, ge + +# Date equality +filter1 = n(filter_dict={ + "event_date": eq(date(2023, 6, 15)) +}) + +# Date range check +filter2 = n(filter_dict={ + "start_date": ge(date(2023, 1, 1)) +}) +``` + +### Wire Protocol (JSON) +```json +// Date equality +{ + "type": "ASTNode", + "filter_dict": { + "event_date": { + "type": "EQ", + "val": { + "type": "date", + "value": "2023-06-15" + } + } + } +} + +// Date greater than or equal +{ + "type": "ASTNode", + "filter_dict": { + "start_date": { + "type": "GE", + "val": { + "type": "date", + "value": "2023-01-01" + } + } + } +} +``` + +## 3. Time-Only Comparisons + +### Python API +```python +from datetime import time +from graphistry.compute import is_in, between + +# Specific times +filter1 = n(filter_dict={ + "event_time": is_in([ + time(9, 0, 0), + time(12, 0, 0), + time(17, 0, 0) + ]) +}) + +# Time range +filter2 = n(edge_match={ + "daily_schedule": between( + time(9, 0, 0), + time(17, 30, 0) + ) +}) +``` + +### Wire Protocol (JSON) +```json +// IsIn with times +{ + "type": "ASTNode", + "filter_dict": { + "event_time": { + "type": "IsIn", + "options": [ + {"type": "time", "value": "09:00:00"}, + {"type": "time", "value": "12:00:00"}, + {"type": "time", "value": "17:00:00"} + ] + } + } +} + +// Time range +{ + "type": "ASTNode", + "edge_match": { + "daily_schedule": { + "type": "Between", + "lower": {"type": "time", "value": "09:00:00"}, + "upper": {"type": "time", "value": "17:30:00"}, + "inclusive": true + } + } +} +``` + +## 4. Timezone-Aware DateTime + +### Python API +```python +import pytz +from graphistry.compute import DateTimeValue, gt + +# Using timezone-aware timestamp +eastern = pytz.timezone('US/Eastern') +filter1 = n(filter_dict={ + "timestamp": gt( + pd.Timestamp("2023-01-01 09:00:00", tz=eastern) + ) +}) + +# Using DateTimeValue with explicit timezone +dt_val = DateTimeValue("2023-01-01T09:00:00", "US/Eastern") +filter2 = n(filter_dict={ + "timestamp": gt(dt_val) +}) +``` + +### Wire Protocol (JSON) +```json +// Timezone-aware datetime +{ + "type": "ASTNode", + "filter_dict": { + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-01-01T09:00:00", + "timezone": "US/Eastern" + } + } + } +} +``` + +## 5. Complex Chain with Temporal Predicates + +### Python API +```python +from graphistry import n, e_forward +from graphistry.compute import gt, eq, between +from datetime import datetime, timedelta + +# Multi-hop query with temporal filters +chain = g.chain([ + # Recent transactions + n(edge_match={ + "timestamp": gt(datetime.now() - timedelta(days=7)), + "amount": gt(1000) + }), + # To active accounts + n(filter_dict={ + "status": eq("active"), + "last_login": between( + datetime.now() - timedelta(days=30), + datetime.now() + ) + }), + # Outgoing transfers + e_forward(edge_match={ + "type": eq("transfer"), + "timestamp": gt(datetime.now() - timedelta(days=1)) + }) +]) +``` + +### Wire Protocol (JSON) +```json +{ + "type": "Chain", + "queries": [ + { + "type": "ASTNode", + "edge_match": { + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-12-18T10:30:00", + "timezone": "UTC" + } + }, + "amount": { + "type": "GT", + "val": 1000 + } + } + }, + { + "type": "ASTNode", + "filter_dict": { + "status": { + "type": "EQ", + "val": "active" + }, + "last_login": { + "type": "Between", + "lower": { + "type": "datetime", + "value": "2023-11-25T10:30:00", + "timezone": "UTC" + }, + "upper": { + "type": "datetime", + "value": "2023-12-25T10:30:00", + "timezone": "UTC" + }, + "inclusive": true + } + } + }, + { + "type": "ASTEdge", + "direction": "forward", + "edge_match": { + "type": { + "type": "EQ", + "val": "transfer" + }, + "timestamp": { + "type": "GT", + "val": { + "type": "datetime", + "value": "2023-12-24T10:30:00", + "timezone": "UTC" + } + } + } + } + ] +} +``` + +## 6. Temporal Value Classes Direct Usage + +### Python API +```python +from graphistry.compute import ( + DateTimeValue, DateValue, TimeValue, + temporal_value_from_json, gt +) + +# Create temporal values +dt_val = DateTimeValue("2023-06-15T14:30:00", "Europe/London") +date_val = DateValue("2023-06-15") +time_val = TimeValue("14:30:00") + +# Use in predicates +filter1 = n(filter_dict={"timestamp": gt(dt_val)}) +filter2 = n(filter_dict={"event_date": eq(date_val)}) +filter3 = n(filter_dict={"daily_time": eq(time_val)}) + +# Create from JSON +json_dt = { + "type": "datetime", + "value": "2023-06-15T14:30:00", + "timezone": "Europe/London" +} +dt_from_json = temporal_value_from_json(json_dt) +``` + +### Wire Protocol (JSON) +```json +// DateTimeValue serialization +{ + "type": "datetime", + "value": "2023-06-15T14:30:00", + "timezone": "Europe/London" +} + +// DateValue serialization +{ + "type": "date", + "value": "2023-06-15" +} + +// TimeValue serialization +{ + "type": "time", + "value": "14:30:00" +} +``` + +## 7. Full Round-Trip Example + +```python +# 1. Create a complex query with temporal predicates +from graphistry import n, Chain +from graphistry.compute import gt, between, is_in +from datetime import datetime, date, time +import pandas as pd + +query = Chain([ + n(filter_dict={ + "created": gt(pd.Timestamp("2023-01-01")), + "event_date": between(date(2023, 6, 1), date(2023, 6, 30)), + "event_time": is_in([time(9, 0), time(12, 0), time(17, 0)]) + }) +]) + +# 2. Serialize to JSON +json_query = query.to_json() +print(json_query) + +# 3. Send over wire (simulated) +import json +wire_data = json.dumps(json_query) +received_data = json.loads(wire_data) + +# 4. Deserialize on receiving end +from graphistry.compute.ast import Chain +reconstructed_query = Chain.from_json(received_data) + +# 5. Apply to graph data +result = g.chain(reconstructed_query.queries) +``` + +## Wire Protocol Structure + +### Temporal Value Types + +All temporal values in the wire protocol follow this pattern: + +```typescript +// DateTime with timezone +interface DateTimeWire { + type: "datetime"; + value: string; // ISO 8601 format + timezone?: string; // IANA timezone (default: "UTC") +} + +// Date only +interface DateWire { + type: "date"; + value: string; // YYYY-MM-DD format +} + +// Time only +interface TimeWire { + type: "time"; + value: string; // HH:MM:SS[.ffffff] format +} +``` + +### Predicate Structure + +Predicates containing temporal values serialize as: + +```typescript +interface TemporalPredicate { + type: "GT" | "LT" | "GE" | "LE" | "EQ" | "NE"; + val: DateTimeWire | DateWire | TimeWire; +} + +interface BetweenPredicate { + type: "Between"; + lower: DateTimeWire | DateWire | TimeWire; + upper: DateTimeWire | DateWire | TimeWire; + inclusive: boolean; +} + +interface IsInPredicate { + type: "IsIn"; + options: Array; +} +``` + +## Key Points + +1. **Type Safety**: Raw strings are rejected in the Python API to avoid ambiguity +2. **Automatic Conversion**: Python datetime objects are automatically converted to appropriate temporal values +3. **Timezone Preservation**: Timezone information is preserved through serialization +4. **Tagged Format**: JSON uses tagged dictionaries to preserve type information +5. **Direct Usage**: Wire protocol dicts can be passed directly to Python predicates + +## Error Handling + +```python +# Raw strings raise ValueError +try: + filter_raw = n(filter_dict={"date": gt("2023-01-01")}) +except ValueError as e: + print(e) + # Output: Raw string '2023-01-01' is ambiguous. Use: + # - gt(pd.Timestamp('2023-01-01')) for datetime + # - gt({'type': 'datetime', 'value': '2023-01-01T00:00:00'}) for explicit type + +# Valid approaches +filter1 = n(filter_dict={"date": gt(pd.Timestamp("2023-01-01"))}) +filter2 = n(filter_dict={"date": gt(datetime(2023, 1, 1))}) +filter3 = n(filter_dict={"date": gt({"type": "datetime", "value": "2023-01-01T00:00:00", "timezone": "UTC"})}) +``` + +## Performance Considerations + +- Temporal predicates leverage pandas' optimized datetime operations +- Timezone conversions are handled efficiently +- For large datasets, ensure datetime columns are properly typed (not object dtype) +- Use `pd.Timestamp` for best performance when creating many predicates programmatically \ No newline at end of file diff --git a/docs/source/notebooks/gfql.rst b/docs/source/notebooks/gfql.rst index dab38cf40e..d03accb479 100644 --- a/docs/source/notebooks/gfql.rst +++ b/docs/source/notebooks/gfql.rst @@ -7,6 +7,7 @@ GFQL Graph queries :titlesonly: Intro to graph queries with hop and chain <../demos/more_examples/graphistry_features/hop_and_chain_graph_pattern_mining.ipynb> + DateTime Filtering Examples <../demos/gfql/temporal_predicates.ipynb> GPU Benchmarking <../demos/gfql/benchmark_hops_cpu_gpu.ipynb> GFQL Remote mode <../demos/gfql/gfql_remote.ipynb> Python Remote mode <../demos/gfql/python_remote.ipynb> diff --git a/graphistry/compute/__init__.py b/graphistry/compute/__init__.py index 1f65fd8359..7c9f36b1d0 100644 --- a/graphistry/compute/__init__.py +++ b/graphistry/compute/__init__.py @@ -18,7 +18,14 @@ is_year_end, IsYearEnd, is_leap_year, IsLeapYear ) -from .predicates.numeric import ( +from .ast_temporal import ( + TemporalValue, + DateTimeValue, + DateValue, + TimeValue, + temporal_value_from_json +) +from .predicates.comparison import ( gt, GT, lt, LT, ge, GE, diff --git a/graphistry/compute/ast_temporal.py b/graphistry/compute/ast_temporal.py new file mode 100644 index 0000000000..cb69dfdf0b --- /dev/null +++ b/graphistry/compute/ast_temporal.py @@ -0,0 +1,155 @@ +from abc import ABC, abstractmethod +from datetime import datetime, date, time +from typing import Dict, Any, Union, TYPE_CHECKING +import pandas as pd +from dateutil import parser as date_parser # type: ignore[import] +import pytz # type: ignore[import] + +from graphistry.utils.json import JSONVal + +if TYPE_CHECKING: + from ..models.gfql.types.temporal import DateTimeWire, DateWire, TimeWire, TemporalWire + + +class TemporalValue(ABC): + """Base class for temporal values with tagging support""" + + @abstractmethod + def to_json(self) -> 'TemporalWire': + """Serialize to JSON-compatible dictionary""" + pass + + @abstractmethod + def as_pandas_value(self) -> Any: + """Convert to pandas-compatible value for comparison""" + pass + + +class DateTimeValue(TemporalValue): + """Tagged datetime value with timezone support""" + + def __init__(self, value: str, timezone: str = "UTC"): + self.value = value + self.timezone = timezone + self._parsed = self._parse_iso8601(value, timezone) + + @classmethod + def from_pandas_timestamp(cls, ts: pd.Timestamp) -> 'DateTimeValue': + """Create from pandas Timestamp""" + tz = str(ts.tz) if ts.tz else "UTC" + value = ts.isoformat() + return cls(value, tz) + + @classmethod + def from_datetime(cls, dt: datetime) -> 'DateTimeValue': + """Create from Python datetime""" + tz = str(dt.tzinfo) if dt.tzinfo else "UTC" + value = dt.isoformat() + return cls(value, tz) + + def _parse_iso8601(self, value: str, timezone: str) -> pd.Timestamp: + """Parse ISO8601 datetime string with timezone""" + # Parse the datetime + dt = date_parser.isoparse(value) + + # Handle timezone + if dt.tzinfo is None: + # Naive datetime - localize to specified timezone + tz = pytz.timezone(timezone) + dt = tz.localize(dt) + else: + # Already has timezone - convert to specified timezone + tz = pytz.timezone(timezone) + dt = dt.astimezone(tz) + + return pd.Timestamp(dt) + + def to_json(self) -> 'DateTimeWire': + """Return dict for tagged temporal value""" + from ..models.gfql.types.temporal import DateTimeWire + result: DateTimeWire = { + "type": "datetime", + "value": self.value, + "timezone": self.timezone + } + return result + + def as_pandas_value(self) -> pd.Timestamp: + return self._parsed + + +class DateValue(TemporalValue): + """Tagged date value""" + + def __init__(self, value: str): + self.value = value + self._parsed = self._parse_date(value) + + @classmethod + def from_date(cls, d: date) -> 'DateValue': + """Create from Python date""" + return cls(d.isoformat()) + + def _parse_date(self, value: str) -> date: + """Parse date string in ISO format (YYYY-MM-DD)""" + return date_parser.isoparse(value).date() + + def to_json(self) -> 'DateWire': + """Return dict for tagged temporal value""" + from ..models.gfql.types.temporal import DateWire + result: DateWire = { + "type": "date", + "value": self.value + } + return result + + def as_pandas_value(self) -> pd.Timestamp: + # Convert date to pandas Timestamp at midnight + return pd.Timestamp(self._parsed) + + +class TimeValue(TemporalValue): + """Tagged time value""" + + def __init__(self, value: str): + self.value = value + self._parsed = self._parse_time(value) + + @classmethod + def from_time(cls, t: time) -> 'TimeValue': + """Create from Python time""" + return cls(t.isoformat()) + + def _parse_time(self, value: str) -> time: + """Parse time string in ISO format (HH:MM:SS)""" + # Handle time-only strings + if 'T' not in value and ' ' not in value: + # Pure time string like "14:30:00" + return datetime.strptime(value, "%H:%M:%S").time() + else: + # Extract time from full datetime + return date_parser.isoparse(value).time() + + def to_json(self) -> 'TimeWire': + """Return dict for tagged temporal value""" + from ..models.gfql.types.temporal import TimeWire + result: TimeWire = { + "type": "time", + "value": self.value + } + return result + + def as_pandas_value(self) -> time: + return self._parsed + + +def temporal_value_from_json(d: Dict[str, Any]) -> TemporalValue: + """Factory function to create temporal value from JSON dict""" + if d["type"] == "datetime": + return DateTimeValue(d["value"], d.get("timezone", "UTC")) + elif d["type"] == "date": + return DateValue(d["value"]) + elif d["type"] == "time": + return TimeValue(d["value"]) + else: + raise ValueError(f"Unknown temporal value type: {d['type']}") diff --git a/graphistry/compute/predicates/comparison.py b/graphistry/compute/predicates/comparison.py new file mode 100644 index 0000000000..d21a8bfad3 --- /dev/null +++ b/graphistry/compute/predicates/comparison.py @@ -0,0 +1,340 @@ +from typing import Union, TYPE_CHECKING, Literal, Dict, cast, overload +import pandas as pd +import numpy as np +from datetime import datetime, date, time + +from .ASTPredicate import ASTPredicate +from ..ast_temporal import TemporalValue, DateTimeValue, DateValue, TimeValue +from ...models.gfql.coercions.temporal import to_ast +from ...models.gfql.types.guards import is_native_numeric, is_any_temporal, is_string +from graphistry.models.gfql.types.predicates import ComparisonInput, BetweenBoundInput +from ...utils.json import JSONVal +from graphistry.compute.typing import SeriesT + +if TYPE_CHECKING: + # Note: pd.Series[T] syntax not supported in Python 3.8 + pass + + +class ComparisonPredicate(ASTPredicate): + """Base class for comparison predicates that support both numeric and temporal values""" + + def __init__(self, val: ComparisonInput) -> None: + self.val = self._normalize_value(val) + + def _normalize_value(self, val: ComparisonInput) -> Union[int, float, np.number, TemporalValue]: + """Convert various input types to internal representation""" + # Comparison predicates need: + # - Numerics as-is + # - Temporals as AST objects (for timezone handling) + # - Strings rejected (ambiguous) + if is_native_numeric(val): + return val + elif is_any_temporal(val): + # to_ast always returns TemporalValue for valid temporal inputs + temporal_val = to_ast(val) + return temporal_val + elif is_string(val): + raise ValueError( + f"Raw string '{val}' is ambiguous. Use:\n" + f" - pd.Timestamp('{val}') for datetime\n" + f" - {{'type': 'datetime', 'value': '{val}'}} for explicit type" + ) + else: + raise TypeError(f"Unsupported type for {self.__class__.__name__}: {type(val)}") + + def _prepare_temporal_series(self, s: SeriesT, temporal_val: TemporalValue) -> SeriesT: + """Prepare series for temporal comparison by extracting/converting as needed""" + if isinstance(temporal_val, DateTimeValue): + # Normalize series to target timezone for comparison + if hasattr(s, 'dt') and hasattr(s.dt, 'tz_localize'): + if s.dt.tz is None: + return s.dt.tz_localize('UTC').dt.tz_convert(temporal_val.timezone) + else: + return s.dt.tz_convert(temporal_val.timezone) + return s + + elif isinstance(temporal_val, DateValue): + # Extract date from datetime series if needed + if hasattr(s, 'dt'): + return s.dt.date + return s + + elif isinstance(temporal_val, TimeValue): + # Extract time from datetime series if needed + if hasattr(s, 'dt'): + return s.dt.time + return s + + raise TypeError(f"Unknown temporal value type: {type(temporal_val)}") + + def _get_temporal_comparison_value(self, temporal_val: TemporalValue) -> Union[pd.Timestamp, date, time]: + """Get the appropriate comparison value from a TemporalValue""" + if isinstance(temporal_val, DateTimeValue): + return temporal_val.as_pandas_value() + elif isinstance(temporal_val, (DateValue, TimeValue)): + return temporal_val._parsed + raise TypeError(f"Unknown temporal value type: {type(temporal_val)}") + + + def validate(self) -> None: + """Validate both numeric and temporal values""" + if isinstance(self.val, (int, float)): + pass # Numeric values are always valid + elif isinstance(self.val, TemporalValue): + pass # Temporal values validated on construction + else: + raise TypeError(f"Invalid value type: {type(self.val)}") + + def to_json(self, validate=True) -> dict: + """Serialize maintaining backward compatibility""" + if validate: + self.validate() + + result: Dict[str, JSONVal] = {"type": self.__class__.__name__} + + if isinstance(self.val, TemporalValue): + # to_json() returns a dict, not a string + val_dict = self.val.to_json() + result["val"] = cast(JSONVal, val_dict) + else: + result["val"] = cast(JSONVal, self.val) + + return result + + +# For backward compatibility - keep but deprecated +class NumericASTPredicate(ComparisonPredicate): + """Deprecated: Use ComparisonPredicate instead""" + pass + +### + +class GT(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Greater than comparison""" + if isinstance(self.val, (int, float)): + return s > self.val + elif isinstance(self.val, TemporalValue): + prepared_s = self._prepare_temporal_series(s, self.val) + comparison_val = self._get_temporal_comparison_value(self.val) + return prepared_s > comparison_val + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def gt(val: ComparisonInput) -> GT: + """ + Return whether a given value is greater than a threshold + """ + return GT(val) + +class LT(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Less than comparison""" + if isinstance(self.val, (int, float)): + return s < self.val + elif isinstance(self.val, TemporalValue): + prepared_s = self._prepare_temporal_series(s, self.val) + comparison_val = self._get_temporal_comparison_value(self.val) + return prepared_s < comparison_val + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def lt(val: ComparisonInput) -> LT: + """ + Return whether a given value is less than a threshold + """ + return LT(val) + +class GE(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Greater than or equal comparison""" + if isinstance(self.val, (int, float)): + return s >= self.val + elif isinstance(self.val, TemporalValue): + prepared_s = self._prepare_temporal_series(s, self.val) + comparison_val = self._get_temporal_comparison_value(self.val) + return prepared_s >= comparison_val + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def ge(val: ComparisonInput) -> GE: + """ + Return whether a given value is greater than or equal to a threshold + """ + return GE(val) + +class LE(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Less than or equal comparison""" + if isinstance(self.val, (int, float)): + return s <= self.val + elif isinstance(self.val, TemporalValue): + prepared_s = self._prepare_temporal_series(s, self.val) + comparison_val = self._get_temporal_comparison_value(self.val) + return prepared_s <= comparison_val + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def le(val: ComparisonInput) -> LE: + """ + Return whether a given value is less than or equal to a threshold + """ + return LE(val) + +class EQ(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Equal comparison""" + if isinstance(self.val, (int, float)): + return s == self.val + elif isinstance(self.val, TemporalValue): + prepared_s = self._prepare_temporal_series(s, self.val) + comparison_val = self._get_temporal_comparison_value(self.val) + return prepared_s == comparison_val + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def eq(val: ComparisonInput) -> EQ: + """ + Return whether a given value is equal to a threshold + """ + return EQ(val) + +class NE(ComparisonPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + """Not equal comparison""" + if isinstance(self.val, (int, float)): + return s != self.val + elif isinstance(self.val, TemporalValue): + prepared_s = self._prepare_temporal_series(s, self.val) + comparison_val = self._get_temporal_comparison_value(self.val) + return prepared_s != comparison_val + else: + raise TypeError(f"Unexpected value type: {type(self.val)}") + +def ne(val: ComparisonInput) -> NE: + """ + Return whether a given value is not equal to a threshold + """ + return NE(val) + +class Between(ASTPredicate): + def __init__(self, lower: BetweenBoundInput, + upper: BetweenBoundInput, + inclusive: bool = True) -> None: + # Store original inputs for creating sub-predicates + self.lower_input = lower + self.upper_input = upper + # Store normalized values for type checking + self.lower = self._normalize_value(lower) + self.upper = self._normalize_value(upper) + self.inclusive = inclusive + + def _normalize_value(self, val: BetweenBoundInput) -> Union[int, float, np.number, TemporalValue]: + """Convert various input types to internal representation""" + # Same normalization as ComparisonPredicate + if is_native_numeric(val): + return val + elif is_any_temporal(val): + # to_ast always returns TemporalValue for valid temporal inputs + temporal_val = to_ast(val) + return temporal_val + elif is_string(val): + raise ValueError( + f"Raw string '{val}' is ambiguous. Use:\n" + f" - pd.Timestamp('{val}') for datetime\n" + f" - {{'type': 'datetime', 'value': '{val}'}} for explicit type" + ) + else: + raise TypeError(f"Unsupported type for {self.__class__.__name__}: {type(val)}") + + def __call__(self, s: SeriesT) -> SeriesT: + # Check if both bounds are same type + lower_is_numeric = isinstance(self.lower, (int, float)) + upper_is_numeric = isinstance(self.upper, (int, float)) + lower_is_temporal = isinstance(self.lower, TemporalValue) + upper_is_temporal = isinstance(self.upper, TemporalValue) + + if lower_is_numeric and upper_is_numeric: + # Numeric comparison + if self.inclusive: + return (s >= self.lower) & (s <= self.upper) + else: + return (s > self.lower) & (s < self.upper) + + elif lower_is_temporal and upper_is_temporal: + # Temporal comparison + # Create comparison predicates using original inputs + ge_pred = GE(self.lower_input) + le_pred = LE(self.upper_input) + gt_pred = GT(self.lower_input) + lt_pred = LT(self.upper_input) + + if self.inclusive: + return ge_pred(s) & le_pred(s) + else: + return gt_pred(s) & lt_pred(s) + + else: + raise TypeError("Between requires both bounds to be same type (numeric or temporal)") + + def validate(self) -> None: + # Check types match + lower_is_numeric = isinstance(self.lower, (int, float)) + upper_is_numeric = isinstance(self.upper, (int, float)) + lower_is_temporal = isinstance(self.lower, TemporalValue) + upper_is_temporal = isinstance(self.upper, TemporalValue) + + if not ((lower_is_numeric and upper_is_numeric) or (lower_is_temporal and upper_is_temporal)): + raise TypeError("Between requires both bounds to be same type") + + assert isinstance(self.inclusive, bool) + + def to_json(self, validate=True) -> dict: + """Serialize maintaining backward compatibility""" + if validate: + self.validate() + + result: Dict[str, JSONVal] = {"type": self.__class__.__name__, "inclusive": self.inclusive} + + # Serialize lower/upper based on type + if isinstance(self.lower, TemporalValue): + result["lower"] = cast(JSONVal, self.lower.to_json()) + else: + result["lower"] = cast(JSONVal, self.lower) + + if isinstance(self.upper, TemporalValue): + result["upper"] = cast(JSONVal, self.upper.to_json()) + else: + result["upper"] = cast(JSONVal, self.upper) + + return result + +def between(lower: BetweenBoundInput, + upper: BetweenBoundInput, + inclusive: bool = True) -> Between: + """ + Return whether a given value is between a lower and upper threshold + """ + return Between(lower, upper, inclusive) + +class IsNA(ASTPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + return s.isna() + +def isna() -> IsNA: + """ + Return whether a given value is NA + """ + return IsNA() + + +class NotNA(ASTPredicate): + def __call__(self, s: SeriesT) -> SeriesT: + return s.notna() + +def notna() -> NotNA: + """ + Return whether a given value is not NA + """ + return NotNA() diff --git a/graphistry/compute/predicates/is_in.py b/graphistry/compute/predicates/is_in.py index 6c2959a7c4..76132a72e6 100644 --- a/graphistry/compute/predicates/is_in.py +++ b/graphistry/compute/predicates/is_in.py @@ -1,21 +1,151 @@ -from typing import Any, List +from typing import Any, List, Union +from .types import NormalizedIsInElement, NormalizedScalar, NormalizedNumeric import pandas as pd +import numpy as np +from datetime import datetime, date, time from graphistry.utils.json import assert_json_serializable from .ASTPredicate import ASTPredicate +# Temporal value classes imported for type checking only +from ...models.gfql.coercions.temporal import to_native +from ...models.gfql.types.guards import is_basic_scalar, is_any_temporal +from graphistry.models.gfql.types.predicates import IsInElementInput +from graphistry.models.gfql.types.temporal import TemporalWire, DateTimeWire, DateWire, TimeWire from graphistry.compute.typing import SeriesT class IsIn(ASTPredicate): - def __init__(self, options: List[Any]) -> None: - self.options = options - + def __init__(self, options: List[IsInElementInput]) -> None: + self.options = self._normalize_options(options) + + def _normalize_options( + self, options: List[IsInElementInput] + ) -> List['NormalizedIsInElement']: + """Normalize options list to handle temporal values""" + normalized = [] + has_temporal = False + has_numeric = False + + for val in options: + norm_val = self._normalize_value(val) + normalized.append(norm_val) + + # Track types for validation + if isinstance(norm_val, (pd.Timestamp, date, time)): + has_temporal = True + elif isinstance(norm_val, (int, float, np.number)): + has_numeric = True + + # Validate no mixing of temporal and numeric types + if has_temporal and has_numeric: + raise ValueError( + "Cannot mix temporal and numeric values in is_in. " + "Found both temporal and numeric values." + ) + + return normalized + + def _normalize_value( + self, val: IsInElementInput + ) -> 'NormalizedIsInElement': + """Convert various input types to internal representation""" + # IsIn predicate needs: + # - Basic scalars (including strings) as-is + # - Temporals as native/pandas types (for .isin() method) + # - Everything else passes through + if is_basic_scalar(val): + return val + elif is_any_temporal(val): + return to_native(val) + else: + # At this point, val should only be wire format dicts that aren't temporal + # Since is_any_temporal catches TemporalValue, we can safely cast + return val # type: ignore[return-value] + def __call__(self, s: SeriesT) -> SeriesT: + # Check if we have any temporal values in options + has_temporal = any( + isinstance(opt, (pd.Timestamp, date, time)) + for opt in self.options + ) + + if has_temporal and hasattr(s, 'dt'): + # For datetime series with time-only values in options, + # we need special handling + time_opts = [opt for opt in self.options if isinstance(opt, time)] + other_opts = [ + opt for opt in self.options if not isinstance(opt, time) + ] + + if time_opts: + time_matches = s.dt.time.isin(time_opts) + if other_opts: + # Also check other values + other_matches = s.isin(other_opts) + return time_matches | other_matches + else: + return time_matches + return s.isin(self.options) - + def validate(self) -> None: assert isinstance(self.options, list) - assert_json_serializable(self.options) + # Check normalized options are JSON serializable + # (temporal values are converted to pandas types which are serializable) + try: + # Create a test list with JSON-compatible versions + json_test: List[Any] = [] + for opt in self.options: + if isinstance(opt, pd.Timestamp): + json_test.append(opt.isoformat()) + elif isinstance(opt, (date, time)): + json_test.append(str(opt)) + elif isinstance(opt, dict): + # Handle wire format temporal types + json_test.append(opt) + else: + # Handle numeric, string, None types + json_test.append(opt) + assert_json_serializable(json_test) + except Exception as e: + raise ValueError(f"Options not JSON serializable: {e}") + + def to_json(self, validate=True) -> dict: + """Override to handle temporal values in options""" + if validate: + self.validate() + + # Convert temporal values back to tagged dicts + json_options: List[Any] = [] + for opt in self.options: + if isinstance(opt, pd.Timestamp): + # Convert back to tagged dict + datetime_wire: DateTimeWire = { + "type": "datetime", + "value": opt.isoformat(), + "timezone": str(opt.tz) if opt.tz else "UTC" + } + json_options.append(datetime_wire) + elif isinstance(opt, date) and not isinstance(opt, datetime): + date_wire: DateWire = { + "type": "date", + "value": opt.isoformat() + } + json_options.append(date_wire) + elif isinstance(opt, time): + time_wire: TimeWire = { + "type": "time", + "value": opt.isoformat() + } + json_options.append(time_wire) + else: + json_options.append(opt) + + return { + 'type': self.__class__.__name__, + 'options': json_options + } + -def is_in(options: List[Any]) -> IsIn: +def is_in(options: List[IsInElementInput]) -> IsIn: return IsIn(options) diff --git a/graphistry/compute/predicates/types.py b/graphistry/compute/predicates/types.py new file mode 100644 index 0000000000..0eb91f7026 --- /dev/null +++ b/graphistry/compute/predicates/types.py @@ -0,0 +1,23 @@ +"""Type definitions for predicates""" +from typing import Union, Literal +import pandas as pd +import numpy as np +from datetime import datetime, date, time + +# Import temporal wire types +from ...models.gfql.types.temporal import DateTimeWire, DateWire, TimeWire + +# Normalized types after processing inputs +NormalizedNumeric = Union[int, float, np.number] +NormalizedTemporal = Union[pd.Timestamp, datetime, date, time, DateTimeWire, DateWire, TimeWire] +NormalizedScalar = Union[str, bool, None] + +# For is_in after normalization - includes all possible return types from _normalize_value +NormalizedIsInElement = Union[ + NormalizedScalar, + NormalizedNumeric, + NormalizedTemporal +] + +# Comparison operators +ComparisonOp = Literal["gt", "lt", "ge", "le", "eq", "ne"] diff --git a/graphistry/models/gfql/coercions/numeric.py b/graphistry/models/gfql/coercions/numeric.py new file mode 100644 index 0000000000..304c0a394a --- /dev/null +++ b/graphistry/models/gfql/coercions/numeric.py @@ -0,0 +1,32 @@ +""" +Pure numeric type transformations + +Currently numeric types don't need much transformation, +but this provides a consistent interface. +""" + +from typing import Union +import numpy as np + +from ..types.numeric import NativeNumeric + + +def to_native(val: NativeNumeric) -> NativeNumeric: + """Numeric values are already native, just pass through""" + return val + + +def to_ast(val: NativeNumeric) -> NativeNumeric: + """Numeric values don't have special AST representation, pass through""" + return val + + +def to_wire(val: NativeNumeric) -> Union[int, float]: + """Convert numeric to JSON-compatible type""" + if isinstance(val, np.number): + # Convert numpy types to Python native + if isinstance(val, (np.integer, np.int_)): + return int(val) + else: + return float(val) + return val diff --git a/graphistry/models/gfql/coercions/temporal.py b/graphistry/models/gfql/coercions/temporal.py new file mode 100644 index 0000000000..386f875361 --- /dev/null +++ b/graphistry/models/gfql/coercions/temporal.py @@ -0,0 +1,97 @@ +""" +Pure temporal type transformations + +Functions to convert between Native, Wire, and AST representations. +""" + +from typing import Union +from datetime import datetime, date, time +import pandas as pd + +from ....compute.ast_temporal import ( + TemporalValue, DateTimeValue, DateValue, TimeValue +) +from ..types.temporal import NativeTemporal, TemporalWire + + +# ============= To AST Transforms ============= + + +def to_ast( + val: Union[NativeTemporal, TemporalWire, TemporalValue] +) -> TemporalValue: + """Convert any temporal representation to AST (TemporalValue)""" + # Already AST + if isinstance(val, TemporalValue): + return val + + # From native + elif isinstance(val, pd.Timestamp): + return DateTimeValue.from_pandas_timestamp(val) + elif isinstance(val, datetime): + return DateTimeValue.from_datetime(val) + elif isinstance(val, date): + return DateValue.from_date(val) + elif isinstance(val, time): + return TimeValue.from_time(val) + + # From wire + elif isinstance(val, dict) and "type" in val: + if val["type"] == "datetime": + timezone = val.get("timezone", "UTC") + assert isinstance(timezone, str) + return DateTimeValue(val["value"], timezone) + elif val["type"] == "date": + return DateValue(val["value"]) + elif val["type"] == "time": + return TimeValue(val["value"]) + else: + raise ValueError(f"Unknown temporal wire type: {val['type']}") + + else: + raise TypeError(f"Cannot convert {type(val)} to AST temporal") + + +# ============= To Native Transforms ============= + + +def to_native( + val: Union[NativeTemporal, TemporalWire, TemporalValue] +) -> Union[pd.Timestamp, time]: + """Convert any temporal representation to native Python/Pandas type""" + # Already native pandas + if isinstance(val, pd.Timestamp): + return val + + # Native Python to pandas + elif isinstance(val, (datetime, date)): + return pd.Timestamp(val) + elif isinstance(val, time): + return val # time stays as time + + # From AST + elif isinstance(val, TemporalValue): + return val.as_pandas_value() + + # From wire (via AST) + elif isinstance(val, dict) and "type" in val: + ast_val = to_ast(val) + return ast_val.as_pandas_value() + + else: + raise TypeError(f"Cannot convert {type(val)} to native temporal") + + +# ============= To Wire Transforms ============= + + +def to_wire(val: Union[NativeTemporal, TemporalValue]) -> TemporalWire: + """Convert temporal to wire format (for JSON serialization)""" + # From AST + if isinstance(val, TemporalValue): + return val.to_json() + + # From native (via AST) + else: + ast_val = to_ast(val) + return ast_val.to_json() diff --git a/graphistry/models/gfql/types/guards.py b/graphistry/models/gfql/types/guards.py new file mode 100644 index 0000000000..77f8aa3db8 --- /dev/null +++ b/graphistry/models/gfql/types/guards.py @@ -0,0 +1,91 @@ +""" +Type detection utilities + +Clear functions to check what kind of type a value is. +""" + +from typing import Any, TYPE_CHECKING, Union + +# Python 3.10+ has TypeGuard in typing module, older versions need typing_extensions +if TYPE_CHECKING: + import sys + if sys.version_info >= (3, 10): + from typing import TypeGuard + else: + from typing_extensions import TypeGuard +from datetime import datetime, date, time +import pandas as pd +import numpy as np + +from ....compute.ast_temporal import TemporalValue +from .temporal import NativeTemporal, TemporalWire +from .numeric import NativeNumeric + + +# ============= Temporal Detection ============= + +def is_native_temporal(val: Any) -> "TypeGuard[NativeTemporal]": + """Check if value is a native Python/Pandas temporal type""" + return isinstance(val, (pd.Timestamp, datetime, date, time)) + + +def is_ast_temporal(val: Any) -> "TypeGuard[TemporalValue]": + """Check if value is an AST TemporalValue""" + return isinstance(val, TemporalValue) + + +def is_wire_temporal(val: Any) -> "TypeGuard[TemporalWire]": + """Check if value is a wire format temporal (tagged dict)""" + return ( + isinstance(val, dict) + and "type" in val + and val["type"] in ["datetime", "date", "time"] + ) + + +def is_any_temporal( + val: Any +) -> "TypeGuard[Union[NativeTemporal, TemporalValue, TemporalWire]]": + """Check if value is any kind of temporal (native, AST, or wire)""" + return ( + is_native_temporal(val) or is_ast_temporal(val) or is_wire_temporal(val) + ) + + +# ============= Numeric Detection ============= + +def is_native_numeric(val: Any) -> "TypeGuard[NativeNumeric]": + """Check if value is a native numeric type""" + return isinstance(val, (int, float, np.number)) + + +def is_any_numeric(val: Any) -> bool: + """Check if value is any kind of numeric (currently same as native)""" + return is_native_numeric(val) + + +# ============= Other Detection ============= + +def is_string(val: Any) -> "TypeGuard[str]": + """Check if value is a string""" + return isinstance(val, str) + + +def is_none(val: Any) -> bool: + """Check if value is None""" + return val is None + + +def is_basic_scalar(val: Any) -> "TypeGuard[Union[int, float, str, np.number, None]]": + """Check if value is a basic scalar (int, float, str, None)""" + return isinstance(val, (int, float, str, np.number, type(None))) + + +def is_dict(val: Any) -> bool: + """Check if value is a dictionary""" + return isinstance(val, dict) + + +def is_tagged_dict(val: Any) -> bool: + """Check if value is a tagged dictionary (has 'type' field)""" + return isinstance(val, dict) and "type" in val diff --git a/graphistry/models/gfql/types/numeric.py b/graphistry/models/gfql/types/numeric.py new file mode 100644 index 0000000000..b56d5fa076 --- /dev/null +++ b/graphistry/models/gfql/types/numeric.py @@ -0,0 +1,33 @@ +""" +Type definitions for numeric values in GFQL + +Numeric data representations: +1. Native types - Host language numeric types (int, float, numpy types) +2. AST types - For numeric predicates, native types are used directly +3. Wire types - JSON number format (no special encoding needed) +""" + +from typing import Union +import numpy as np + + +# ============= Native Numeric Types ============= +# Host language numeric types + +NativeInt = int +NativeFloat = float +NativeNumeric = Union[int, float, np.number] + + +# ============= Wire Types (JSON) ============= +# For numeric values, JSON numbers map directly to native types +# No special wire format needed - JSON handles int/float natively + +WireNumeric = Union[int, float] + + +# ============= AST Types ============= +# Numeric predicates use native types directly in the AST +# No special wrapper needed unlike temporal values + +ASTNumeric = NativeNumeric diff --git a/graphistry/models/gfql/types/predicates.py b/graphistry/models/gfql/types/predicates.py new file mode 100644 index 0000000000..61f34949c3 --- /dev/null +++ b/graphistry/models/gfql/types/predicates.py @@ -0,0 +1,45 @@ +""" +Type definitions for GFQL predicate inputs + +Defines what types users can pass to predicates. +Implementation details (like what gets stored internally) +belong in compute/predicates. +""" + +from typing import Union, TYPE_CHECKING +import numpy as np + +from .temporal import NativeTemporal, TemporalWire +from .numeric import NativeNumeric + +if TYPE_CHECKING: + from graphistry.compute.ast_temporal import TemporalValue + + +# ============= Basic Types ============= +# Simple scalar types + +BasicScalar = Union[int, float, str, np.number, None] + + +# ============= Predicate Input Types ============= +# What users can provide to each predicate type + +# Comparison predicates (GT, LT, GE, LE, EQ, NE) - strict, no strings +ComparisonInput = Union[ + NativeNumeric, # Python int, float, np.number + NativeTemporal, # Python datetime, date, time, pd.Timestamp + TemporalWire, # Wire format: {"type": "datetime", ...} + "TemporalValue", # AST temporal values +] + +# IsIn predicate - permissive, allows strings and arbitrary values +IsInElementInput = Union[ + BasicScalar, # Includes strings and None + NativeTemporal, # Python datetime types + TemporalWire, # Wire format temporal + "TemporalValue", # AST temporal values +] + +# Between predicate - each bound follows comparison rules +BetweenBoundInput = ComparisonInput diff --git a/graphistry/models/gfql/types/temporal.py b/graphistry/models/gfql/types/temporal.py new file mode 100644 index 0000000000..5b3df8cfe0 --- /dev/null +++ b/graphistry/models/gfql/types/temporal.py @@ -0,0 +1,53 @@ +""" +Type definitions for temporal values in GFQL + +Temporal data has three representations: +1. Native types - Host language types (Python datetime, pandas.Timestamp, etc.) +2. AST types - Domain model objects used in the query AST (TemporalValue classes) +3. Wire types - JSON serialization format for client/server communication + +Conversion functions follow the pattern: to/from_native, to/from_wire, to/from_ast +""" + +from typing import Union, TypedDict, Literal +from datetime import datetime, date, time +import pandas as pd + + +# ============= Native Temporal Types ============= +# Host language types that users work with directly + +NativeDateTime = Union[pd.Timestamp, datetime] +NativeDate = date # Note: NOT a Union - important for overload ordering +NativeTime = time +NativeTemporal = Union[NativeDateTime, NativeDate, NativeTime] + + +# ============= Wire Types (JSON) ============= +# Tagged dictionaries for JSON serialization/deserialization + +class DateTimeWire(TypedDict, total=False): + """Wire format for datetime with timezone""" + type: Literal["datetime"] + value: str # ISO 8601 datetime string + timezone: str # Optional IANA timezone (default: UTC) + + +class DateWire(TypedDict): + """Wire format for date only""" + type: Literal["date"] + value: str # ISO 8601 date (YYYY-MM-DD) + + +class TimeWire(TypedDict): + """Wire format for time only""" + type: Literal["time"] + value: str # ISO 8601 time (HH:MM:SS[.ffffff]) + + +# Union of all temporal wire types +TemporalWire = Union[DateTimeWire, DateWire, TimeWire] + + +# Note: AST types (TemporalValue, DateTimeValue, etc.) are defined +# in graphistry.compute.ast_temporal to avoid circular imports diff --git a/graphistry/tests/compute/predicates/test_temporal_values.py b/graphistry/tests/compute/predicates/test_temporal_values.py new file mode 100644 index 0000000000..62dfd2de3b --- /dev/null +++ b/graphistry/tests/compute/predicates/test_temporal_values.py @@ -0,0 +1,103 @@ +import pytest +import pandas as pd +from datetime import datetime, date, time +import pytz + +from graphistry.compute.ast_temporal import ( + DateTimeValue, DateValue, TimeValue, temporal_value_from_json +) + + +class TestDateTimeValue: + def test_parse_iso8601_with_timezone(self): + dt = DateTimeValue("2024-01-01T12:00:00+00:00", "UTC") + assert dt.value == "2024-01-01T12:00:00+00:00" + assert dt.timezone == "UTC" + assert isinstance(dt.as_pandas_value(), pd.Timestamp) + assert dt.as_pandas_value().hour == 12 + + def test_parse_iso8601_naive(self): + dt = DateTimeValue("2024-01-01T12:00:00", "UTC") + assert dt.timezone == "UTC" + assert dt.as_pandas_value().hour == 12 + assert dt.as_pandas_value().tz.zone == "UTC" + + def test_timezone_conversion(self): + # Create datetime in UTC + dt_utc = DateTimeValue("2024-01-01T12:00:00+00:00", "UTC") + # Create same instant in EST (UTC-5) + dt_est = DateTimeValue("2024-01-01T12:00:00+00:00", "US/Eastern") + + # Should be same instant but displayed in EST + assert dt_est.as_pandas_value().hour == 7 # 12 UTC = 7 EST + assert dt_utc.as_pandas_value().timestamp() == dt_est.as_pandas_value().timestamp() + + def test_to_json(self): + dt = DateTimeValue("2024-01-01T12:00:00Z", "UTC") + json_data = dt.to_json() + assert json_data == { + "type": "datetime", + "value": "2024-01-01T12:00:00Z", + "timezone": "UTC" + } + + +class TestDateValue: + def test_parse_date(self): + d = DateValue("2024-01-01") + assert d.value == "2024-01-01" + assert d._parsed == date(2024, 1, 1) + assert isinstance(d.as_pandas_value(), pd.Timestamp) + assert d.as_pandas_value().date() == date(2024, 1, 1) + + def test_to_json(self): + d = DateValue("2024-01-01") + json_data = d.to_json() + assert json_data == { + "type": "date", + "value": "2024-01-01" + } + + +class TestTimeValue: + def test_parse_time(self): + t = TimeValue("14:30:00") + assert t.value == "14:30:00" + assert t._parsed == time(14, 30, 0) + assert isinstance(t.as_pandas_value(), time) + assert t.as_pandas_value().hour == 14 + assert t.as_pandas_value().minute == 30 + + def test_to_json(self): + t = TimeValue("14:30:00") + json_data = t.to_json() + assert json_data == { + "type": "time", + "value": "14:30:00" + } + + +class TestTemporalValueFromJson: + def test_datetime_from_json(self): + json_data = {"type": "datetime", "value": "2024-01-01T12:00:00Z", "timezone": "UTC"} + dt = temporal_value_from_json(json_data) + assert isinstance(dt, DateTimeValue) + assert dt.value == "2024-01-01T12:00:00Z" + assert dt.timezone == "UTC" + + def test_date_from_json(self): + json_data = {"type": "date", "value": "2024-01-01"} + d = temporal_value_from_json(json_data) + assert isinstance(d, DateValue) + assert d.value == "2024-01-01" + + def test_time_from_json(self): + json_data = {"type": "time", "value": "14:30:00"} + t = temporal_value_from_json(json_data) + assert isinstance(t, TimeValue) + assert t.value == "14:30:00" + + def test_invalid_type(self): + json_data = {"type": "invalid", "value": "something"} + with pytest.raises(ValueError, match="Unknown temporal value type"): + temporal_value_from_json(json_data) diff --git a/setup.py b/setup.py index 75b060efcf..8d05e1c74a 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ def unique_flatten_dict(d): 'docs': [ 'docutils==0.21.2', 'ipython==8.28', + 'ipykernel==6.29.5', # For notebook execution validation 'Jinja2==3.1.4', 'myst-parser==4.0.0', 'nbsphinx==0.9.5',