From 32293f4b55a71ed0ee3745e085c15bd574d1f05e Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Thu, 30 Jul 2026 00:24:06 -0700 Subject: [PATCH 01/21] feat(tables): typed predicate filter grammar, cursor pagination, and the v2 table surface (#6067) --- apps/docs/openapi-v2-tables.json | 381 ++++++++++++ apps/docs/openapi.json | 2 +- .../api/table/[tableId]/cancel-runs/route.ts | 14 +- .../api/table/[tableId]/columns/run/route.ts | 23 +- .../[tableId]/delete-async/route.test.ts | 21 + .../api/table/[tableId]/delete-async/route.ts | 20 +- .../app/api/table/[tableId]/export/route.ts | 4 +- .../api/table/[tableId]/query/route.test.ts | 202 ++++++ .../app/api/table/[tableId]/query/route.ts | 134 ++++ .../table/[tableId]/rows/find/route.test.ts | 22 +- .../api/table/[tableId]/rows/find/route.ts | 23 +- .../api/table/[tableId]/rows/route.test.ts | 160 ++++- .../sim/app/api/table/[tableId]/rows/route.ts | 118 +++- apps/sim/app/api/table/row-wire.ts | 24 +- apps/sim/app/api/table/utils.test.ts | 50 +- apps/sim/app/api/table/utils.ts | 48 +- apps/sim/app/api/v1/middleware.ts | 13 +- .../app/api/v1/tables/[tableId]/rows/route.ts | 6 +- .../v2/tables/[tableId]/query/route.test.ts | 255 ++++++++ .../api/v2/tables/[tableId]/query/route.ts | 151 +++++ apps/sim/app/api/v2/tables/route.test.ts | 131 ++++ apps/sim/app/api/v2/tables/route.ts | 78 +++ .../components/table-filter/table-filter.tsx | 15 +- .../components/table-grid/table-grid.tsx | 10 +- .../tables/[tableId]/hooks/use-table.test.ts | 2 +- .../tables/[tableId]/hooks/use-table.ts | 8 +- .../[workspaceId]/tables/[tableId]/table.tsx | 35 +- .../[workspaceId]/tables/[tableId]/types.ts | 6 +- .../components/sort-builder/sort-builder.tsx | 4 +- apps/sim/blocks/blocks/table.ts | 2 +- apps/sim/blocks/blocks/table_v2.test.ts | 119 ++++ apps/sim/blocks/blocks/table_v2.ts | 584 ++++++++++++++++++ apps/sim/blocks/registry-maps.minimal.ts | 2 + apps/sim/blocks/registry-maps.ts | 2 + .../lib/copy/copy-resources.ts | 17 + apps/sim/hooks/queries/tables.ts | 32 +- apps/sim/lib/api/client/request.test.ts | 30 +- apps/sim/lib/api/client/request.ts | 20 +- .../api/contracts/tables-predicate.test.ts | 261 ++++++++ apps/sim/lib/api/contracts/tables.ts | 240 ++++++- apps/sim/lib/api/contracts/v2/tables/index.ts | 115 ++++ .../lib/copilot/generated/tool-catalog-v1.ts | 21 +- .../tools/handlers/function-execute.ts | 9 +- .../tools/server/table/user-table.test.ts | 139 ++++- .../copilot/tools/server/table/user-table.ts | 97 ++- apps/sim/lib/core/config/env.ts | 3 +- .../sim/lib/core/config/feature-flags.test.ts | 27 + apps/sim/lib/core/config/feature-flags.ts | 8 + apps/sim/lib/table/__tests__/paging.test.ts | 33 - .../lib/table/__tests__/select-values.test.ts | 128 ++++ .../service-filter-threading.test.ts | 319 +++++++++- apps/sim/lib/table/__tests__/sql.test.ts | 375 ++++++++++- .../lib/table/__tests__/validation.test.ts | 6 +- apps/sim/lib/table/column-keys.ts | 29 + apps/sim/lib/table/constants.ts | 52 +- apps/sim/lib/table/delete-runner.ts | 5 + apps/sim/lib/table/errors.ts | 28 + apps/sim/lib/table/export-runner.ts | 4 +- apps/sim/lib/table/index.ts | 1 + apps/sim/lib/table/jobs/service.ts | 25 +- apps/sim/lib/table/planner.ts | 55 +- .../__tests__/converters.test.ts | 302 ++++++++- .../query-builder/__tests__/validate.test.ts | 215 +++++++ apps/sim/lib/table/query-builder/constants.ts | 10 +- .../sim/lib/table/query-builder/converters.ts | 236 +++++++ .../table/query-builder/use-query-builder.ts | 4 +- apps/sim/lib/table/query-builder/validate.ts | 225 +++++++ .../lib/table/rows/__tests__/cursor.test.ts | 96 +++ apps/sim/lib/table/rows/cursor.test.ts | 78 +++ apps/sim/lib/table/rows/cursor.ts | 171 +++++ apps/sim/lib/table/rows/paging.ts | 23 - apps/sim/lib/table/rows/service.ts | 379 +++++++++--- apps/sim/lib/table/select-values.test.ts | 52 +- apps/sim/lib/table/select-values.ts | 81 ++- apps/sim/lib/table/snapshot-cache.ts | 4 +- apps/sim/lib/table/sql.ts | 549 ++++++++++++---- apps/sim/lib/table/types.ts | 61 +- apps/sim/lib/table/validation.ts | 86 ++- apps/sim/lib/table/views/service.test.ts | 42 +- apps/sim/lib/table/views/service.ts | 58 +- apps/sim/lib/workspaces/utils.ts | 15 + apps/sim/tools/normalize.test.ts | 40 ++ apps/sim/tools/normalize.ts | 38 +- apps/sim/tools/registry.minimal.ts | 29 + apps/sim/tools/registry.ts | 2 + apps/sim/tools/table/index.ts | 1 + apps/sim/tools/table/query_rows_v2.ts | 106 ++++ apps/sim/tools/table/types.ts | 22 + scripts/check-api-validation-contracts.ts | 4 +- 89 files changed, 7116 insertions(+), 561 deletions(-) create mode 100644 apps/docs/openapi-v2-tables.json create mode 100644 apps/sim/app/api/table/[tableId]/query/route.test.ts create mode 100644 apps/sim/app/api/table/[tableId]/query/route.ts create mode 100644 apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts create mode 100644 apps/sim/app/api/v2/tables/[tableId]/query/route.ts create mode 100644 apps/sim/app/api/v2/tables/route.test.ts create mode 100644 apps/sim/app/api/v2/tables/route.ts create mode 100644 apps/sim/blocks/blocks/table_v2.test.ts create mode 100644 apps/sim/blocks/blocks/table_v2.ts create mode 100644 apps/sim/lib/api/contracts/tables-predicate.test.ts create mode 100644 apps/sim/lib/api/contracts/v2/tables/index.ts delete mode 100644 apps/sim/lib/table/__tests__/paging.test.ts create mode 100644 apps/sim/lib/table/__tests__/select-values.test.ts create mode 100644 apps/sim/lib/table/errors.ts create mode 100644 apps/sim/lib/table/query-builder/__tests__/validate.test.ts create mode 100644 apps/sim/lib/table/query-builder/validate.ts create mode 100644 apps/sim/lib/table/rows/__tests__/cursor.test.ts create mode 100644 apps/sim/lib/table/rows/cursor.test.ts create mode 100644 apps/sim/lib/table/rows/cursor.ts delete mode 100644 apps/sim/lib/table/rows/paging.ts create mode 100644 apps/sim/tools/normalize.test.ts create mode 100644 apps/sim/tools/table/query_rows_v2.ts diff --git a/apps/docs/openapi-v2-tables.json b/apps/docs/openapi-v2-tables.json new file mode 100644 index 00000000000..15757b19f1e --- /dev/null +++ b/apps/docs/openapi-v2-tables.json @@ -0,0 +1,381 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Sim Tables API v2", + "version": "2.0.0-preview", + "description": "Read access to Sim tables with the typed predicate filter grammar and opaque cursor pagination. This surface is feature-gated (`tables-v2-api`): when the flag is off for the caller, every endpoint returns 404 as if it does not exist. Filters are predicate trees — `{\"all\": [...]}` (AND) or `{\"any\": [...]}` (OR) groups whose members are `{field, op, value}` conditions or nested groups. Built-in columns `id`, `createdAt`, and `updatedAt` (camelCase) are filterable and sortable alongside user columns." + }, + "servers": [{ "url": "https://www.sim.ai" }], + "security": [{ "apiKey": [] }], + "paths": { + "/api/v2/tables": { + "get": { + "operationId": "v2ListTables", + "summary": "List Tables", + "description": "List every table in a workspace with its column schema and row count.", + "tags": ["Tables v2"], + "parameters": [ + { + "name": "workspaceId", + "in": "query", + "required": true, + "schema": { "type": "string", "minLength": 1 } + } + ], + "responses": { + "200": { + "description": "Tables in the workspace. Served with `Cache-Control: private, no-store`.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["success", "data"], + "properties": { + "success": { "const": true }, + "data": { + "type": "object", + "required": ["tables", "totalCount"], + "properties": { + "tables": { + "type": "array", + "items": { "$ref": "#/components/schemas/TableSummary" } + }, + "totalCount": { "type": "integer" } + } + } + } + } + } + } + }, + "400": { "$ref": "#/components/responses/ValidationError" }, + "401": { "$ref": "#/components/responses/Unauthorized" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFoundOrGated" }, + "429": { "$ref": "#/components/responses/RateLimited" } + } + } + }, + "/api/v2/tables/{tableId}/query": { + "post": { + "operationId": "v2QueryTableRows", + "summary": "Query Rows", + "description": "Query rows with a typed predicate filter, an ordered sort spec, and opaque cursor pagination. Row `data` is keyed by column NAME; `select` cells return option names, and filter operands on select columns accept option names (resolved case-insensitively).\n\n**Pagination contract:** page by passing the previous response's `nextCursor` back as `cursor`, and stop only when it is `null` — a page may return fewer than `limit` rows and still have more behind it, so page fullness is never a termination signal. A cursor is bound to the exact query shape it was minted under: keyset cursors to the default row order, offset cursors (sorted views) to that sort. Replaying one under a different `sort` returns 400 `CURSOR_SORT_CONFLICT`. `totalCount` is computed on the first page only (requests with a `cursor` return `totalCount: null`).", + "tags": ["Tables v2"], + "parameters": [ + { + "name": "tableId", + "in": "path", + "required": true, + "schema": { "type": "string", "minLength": 1 } + } + ], + "requestBody": { + "required": true, + "description": "Bodies over 1 MB are rejected with 413.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["workspaceId"], + "properties": { + "workspaceId": { "type": "string", "minLength": 1 }, + "predicate": { "$ref": "#/components/schemas/Predicate" }, + "sort": { + "type": "array", + "maxItems": 16, + "description": "Ordered sort spec, highest priority first.", + "items": { + "type": "object", + "required": ["field", "direction"], + "properties": { + "field": { "type": "string" }, + "direction": { "enum": ["asc", "desc"] } + } + } + }, + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 1000, + "default": 100, + "description": "Omitted → 100. `1..1000` → page size. `0` → the ENTIRE matching result in one response; fails with 400 `TABLE_QUERY_RESULT_TOO_LARGE` if it exceeds the 5 MB row-data budget (narrow the predicate or page instead)." + }, + "cursor": { + "type": "string", + "description": "Opaque token from a previous response's `nextCursor`. Pass back verbatim. Mutually exclusive with `sort`." + } + } + }, + "examples": { + "filtered": { + "summary": "Multi-select membership + negated pattern", + "value": { + "workspaceId": "ws_123", + "predicate": { + "all": [ + { "field": "Color", "op": "contains", "value": "Purple" }, + { "field": "name", "op": "nlike", "value": "G*" } + ] + }, + "limit": 100 + } + }, + "builtinColumns": { + "summary": "Built-in column range (UTC, timezone-independent)", + "value": { + "workspaceId": "ws_123", + "predicate": { + "all": [ + { "field": "createdAt", "op": "gte", "value": "2026-07-24T03:00:00.000Z" }, + { "field": "createdAt", "op": "lte", "value": "2026-07-25T02:59:59.999Z" } + ] + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "A page of rows. Served with `Cache-Control: private, no-store`.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["success", "data"], + "properties": { + "success": { "const": true }, + "data": { + "type": "object", + "required": ["rows", "rowCount", "nextCursor"], + "properties": { + "rows": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "data", "createdAt", "updatedAt"], + "properties": { + "id": { "type": "string" }, + "data": { + "type": "object", + "description": "Column-NAME-keyed cell values.", + "additionalProperties": true + }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + } + }, + "rowCount": { "type": "integer", "description": "Rows in THIS page." }, + "totalCount": { + "type": ["integer", "null"], + "description": "Rows matching the predicate across all pages. First page only; null when a cursor was supplied." + }, + "limit": { "type": ["integer", "null"] }, + "nextCursor": { + "type": ["string", "null"], + "description": "Non-null ⇒ more rows exist. The ONLY termination signal is null." + } + } + } + } + } + } + } + }, + "400": { + "description": "Validation failure. Machine-readable `code` values include `INVALID_FILTER` (unknown column, operator/type mismatch, malformed tree), `INVALID_ORDER`, `INVALID_CURSOR`, `CURSOR_SORT_CONFLICT`, and `TABLE_QUERY_RESULT_TOO_LARGE` (unbounded result exceeded the 5 MB budget).", + "content": { + "application/json": { + "schema": { "$ref": "#/components/schemas/ErrorBody" } + } + } + }, + "401": { "$ref": "#/components/responses/Unauthorized" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFoundOrGated" }, + "413": { + "description": "Request body exceeded the 1 MB cap.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "429": { "$ref": "#/components/responses/RateLimited" } + } + } + } + }, + "components": { + "securitySchemes": { + "apiKey": { "type": "apiKey", "in": "header", "name": "X-API-Key" } + }, + "schemas": { + "Predicate": { + "description": "A predicate tree: exactly one of `all` (every member must match) or `any` (at least one must). Members are conditions or nested groups; nesting expresses mixed AND/OR logic. Groups must be non-empty (1–100 members), trees at most 10 levels deep and 500 nodes total. Nodes are STRICT: unknown keys, or a node carrying both a group key and condition keys, are rejected rather than ignored.", + "oneOf": [ + { + "type": "object", + "required": ["all"], + "additionalProperties": false, + "properties": { + "all": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { "$ref": "#/components/schemas/PredicateNode" } + } + } + }, + { + "type": "object", + "required": ["any"], + "additionalProperties": false, + "properties": { + "any": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { "$ref": "#/components/schemas/PredicateNode" } + } + } + } + ] + }, + "PredicateNode": { + "oneOf": [ + { "$ref": "#/components/schemas/Predicate" }, + { "$ref": "#/components/schemas/Condition" } + ] + }, + "Condition": { + "type": "object", + "required": ["field", "op"], + "additionalProperties": false, + "properties": { + "field": { + "type": "string", + "maxLength": 128, + "description": "Column name, or a built-in: `id`, `createdAt`, `updatedAt` (camelCase — snake_case is treated as a user column and matches nothing)." + }, + "op": { + "enum": [ + "eq", + "ne", + "gt", + "gte", + "lt", + "lte", + "in", + "nin", + "contains", + "ncontains", + "startsWith", + "endsWith", + "like", + "ilike", + "nlike", + "nilike", + "isEmpty", + "isNotEmpty", + "isNull", + "isNotNull" + ], + "description": "`eq`/`ne`/`in`/`nin` are case-sensitive equality/membership. `contains`/`ncontains`/`startsWith`/`endsWith` are case-insensitive text matches — except on a multi-select column, where `contains`/`ncontains` mean set membership by option name. `like`/`nlike` are case-sensitive and `ilike`/`nilike` case-insensitive patterns with `*` as the only wildcard (literal `%`/`_` match themselves). `isEmpty`/`isNotEmpty` treat null and empty string as empty; `isNull`/`isNotNull` are strict null checks. The four `is*` operators take no `value`. Negated text matches retain rows where the cell is absent. `in`/`nin` require a non-empty array of at most 1000 values; other value-taking operators reject arrays. Select columns accept only equality/membership operators appropriate to their cardinality (single: eq/ne/in/nin; multi: contains/ncontains; both: the `is*` checks)." + }, + "value": { + "description": "Operand. Omit for the `is*` operators. Ranges on `number` columns require numbers, on `date` columns ISO strings (compared as UTC, independent of any session timezone); ranges on `boolean`/`json` columns are rejected." + } + } + }, + "TableSummary": { + "type": "object", + "required": ["id", "name", "schema", "rowCount", "maxRows", "createdAt", "updatedAt"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "schema": { + "type": "object", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "type"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "type": { "enum": ["string", "number", "boolean", "date", "json", "select"] }, + "required": { "type": "boolean" }, + "unique": { "type": "boolean" }, + "options": { + "type": "array", + "description": "Declared choices on a `select` column.", + "items": { + "type": "object", + "properties": { "id": { "type": "string" }, "name": { "type": "string" } } + } + }, + "multiple": { "type": "boolean" } + } + } + } + } + }, + "rowCount": { "type": "integer" }, + "maxRows": { "type": "integer" }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + }, + "ErrorBody": { + "type": "object", + "required": ["error"], + "properties": { + "error": { + "type": "string", + "description": "Human-readable message naming the failing field/operator." + }, + "code": { + "type": "string", + "description": "Machine-readable code, present on domain validation failures." + } + } + } + }, + "responses": { + "ValidationError": { + "description": "Malformed request.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "Unauthorized": { + "description": "Missing or invalid API key.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "Forbidden": { + "description": "The key's workspace scope does not cover this workspace, or the caller lacks read access.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "NotFoundOrGated": { + "description": "Table not found — or the `tables-v2-api` feature flag is off for this caller, in which case the entire surface answers 404. The gate is evaluated after authorization, so a 404 never distinguishes rollout cohort from missing resource for callers without access.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + }, + "RateLimited": { + "description": "Rate limit exceeded for this key.", + "content": { + "application/json": { "schema": { "$ref": "#/components/schemas/ErrorBody" } } + } + } + } + } +} diff --git a/apps/docs/openapi.json b/apps/docs/openapi.json index 1844cc7a1e2..4967b8c8186 100644 --- a/apps/docs/openapi.json +++ b/apps/docs/openapi.json @@ -3405,7 +3405,7 @@ "name": "sort", "in": "query", "required": false, - "description": "JSON-encoded sort object. Example: {\"created_at\": \"desc\"}.", + "description": "JSON-encoded sort object. Example: {\"createdAt\": \"desc\"}. Built-in columns are camelCase: id, createdAt, updatedAt.", "schema": { "type": "string" } diff --git a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts index ce656d6be50..acace4561c5 100644 --- a/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts +++ b/apps/sim/app/api/table/[tableId]/cancel-runs/route.ts @@ -5,6 +5,8 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { TableQueryValidationError } from '@/lib/table/errors' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { cancelWorkflowGroupRuns } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -32,7 +34,10 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(cancelTableRunsContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, scope, rowId, filter, excludeRowIds } = parsed.data.body + const { workspaceId, scope, rowId, filter: wireFilter, excludeRowIds } = parsed.data.body + // Dual-grammar wire: a predicate downgrades losslessly-or-throws to the + // legacy Filter the runners/persisted payloads still compile. + const filter = toLegacyFilter(wireFilter) const result = await checkAccess(tableId, authResult.userId, 'write') if (!result.ok) return accessError(result, requestId, tableId) @@ -42,7 +47,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const filterError = tableFilterError(filter, table.schema.columns) + const filterError = tableFilterError(wireFilter, table.schema.columns) if (filterError) return filterError const cancelled = await cancelWorkflowGroupRuns(tableId, scope === 'row' ? rowId : undefined, { @@ -57,6 +62,11 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ success: true, data: { cancelled } }) } catch (error) { + // A predicate that Zod accepts but the downgrade rejects (hybrid node, + // eq-with-array, valueless op) is caller error, not a server fault. + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } logger.error(`[${requestId}] cancel-runs failed:`, error) return NextResponse.json({ error: 'Failed to cancel runs' }, { status: 500 }) } diff --git a/apps/sim/app/api/table/[tableId]/columns/run/route.ts b/apps/sim/app/api/table/[tableId]/columns/run/route.ts index 00856ae4a1a..824ce73ddb4 100644 --- a/apps/sim/app/api/table/[tableId]/columns/run/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/run/route.ts @@ -5,6 +5,8 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { TableQueryValidationError } from '@/lib/table/errors' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import { runWorkflowColumn } from '@/lib/table/workflow-columns' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -25,13 +27,23 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(runColumnContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, groupIds, runMode, rowIds, filter, excludeRowIds, limit } = - parsed.data.body + const { + workspaceId, + groupIds, + runMode, + rowIds, + filter: wireFilter, + excludeRowIds, + limit, + } = parsed.data.body + // Dual-grammar wire: downgrade a predicate to the legacy Filter the + // dispatcher and scheduled runs still compile. + const filter = toLegacyFilter(wireFilter) const access = await checkAccess(tableId, auth.userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) // Validate the filter up front (the dispatcher reuses it) so a bad field fails fast. - const filterError = tableFilterError(filter, access.table.schema.columns) + const filterError = tableFilterError(wireFilter, access.table.schema.columns) if (filterError) return filterError const { dispatchId } = await runWorkflowColumn({ @@ -49,6 +61,11 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ success: true, data: { dispatchId } }) } catch (error) { + // A predicate that Zod accepts but the downgrade rejects (hybrid node, + // eq-with-array, valueless op) is caller error, not a server fault. + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } if (error instanceof Error && error.message === 'Invalid workspace ID') { return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts index 38b24e06fb0..cffb5dc8810 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.test.ts @@ -208,4 +208,25 @@ describe('POST /api/table/[tableId]/delete-async', () => { expect(mockReleaseJobClaim).toHaveBeenCalledWith('tbl_1', 'job-id-xyz') expect(mockRunTableDelete).not.toHaveBeenCalled() }) + + /** + * PR #6067 review finding (greptile P1 / bugbot High): a hybrid filter — group + * key AND leaf keys on one node — passes the dual-grammar union via the + * non-stripping legacy branch, and the downgrade used to convert group-first, + * silently dropping the leaf and WIDENING an async select-all delete. + */ + it('rejects a hybrid group+leaf filter with 400 instead of widening the delete', async () => { + const response = await makeRequest({ + workspaceId: 'workspace-1', + filter: { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + }, + }) + expect(response.status).toBe(400) + const body = await response.json() + expect(body.error).toMatch(/not both/) + }) }) diff --git a/apps/sim/app/api/table/[tableId]/delete-async/route.ts b/apps/sim/app/api/table/[tableId]/delete-async/route.ts index 236eb556240..83382ea2ddd 100644 --- a/apps/sim/app/api/table/[tableId]/delete-async/route.ts +++ b/apps/sim/app/api/table/[tableId]/delete-async/route.ts @@ -8,9 +8,12 @@ import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { runDetached } from '@/lib/core/utils/background' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Filter } from '@/lib/table' import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner' +import { TableQueryValidationError } from '@/lib/table/errors' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete } from '@/lib/table/mutation-locks' +import { toLegacyFilter } from '@/lib/table/query-builder/converters' import type { TableDeleteJobPayload } from '@/lib/table/types' import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' @@ -43,7 +46,20 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(deleteTableRowsAsyncContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, filter, excludeRowIds, estimatedCount } = parsed.data.body + const { workspaceId, filter: wireFilter, excludeRowIds, estimatedCount } = parsed.data.body + // Dual-grammar wire: a predicate downgrades losslessly-or-throws to the + // legacy Filter the runners/persisted payloads still compile. A shape the + // union accepted but the downgrade rejects (hybrid node, eq-with-array) is + // caller error — 400, never the generic 500. + let filter: Filter | undefined + try { + filter = toLegacyFilter(wireFilter) + } catch (error) { + if (error instanceof TableQueryValidationError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + throw error + } const access = await checkAccess(tableId, userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) @@ -62,7 +78,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro assertRowDelete(table) // Validate the filter up front so the caller gets immediate feedback (the worker reuses it). - const filterError = tableFilterError(filter, table.schema.columns) + const filterError = tableFilterError(wireFilter, table.schema.columns) if (filterError) return filterError // Rows inserted after this instant are spared (created_at <= cutoff in the worker). diff --git a/apps/sim/app/api/table/[tableId]/export/route.ts b/apps/sim/app/api/table/[tableId]/export/route.ts index 6d717bdcdc3..58df047c629 100644 --- a/apps/sim/app/api/table/[tableId]/export/route.ts +++ b/apps/sim/app/api/table/[tableId]/export/route.ts @@ -111,7 +111,9 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou } } - if (result.rows.length < EXPORT_BATCH_SIZE) break + // A page can be cut by the byte budget before reaching EXPORT_BATCH_SIZE, + // so a short page does NOT mean the export is done — only a null cursor does. + if (!result.nextCursor) break offset += result.rows.length } diff --git a/apps/sim/app/api/table/[tableId]/query/route.test.ts b/apps/sim/app/api/table/[tableId]/query/route.test.ts new file mode 100644 index 00000000000..fb727cd53f3 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/query/route.test.ts @@ -0,0 +1,202 @@ +/** + * @vitest-environment node + * + * v2 query route: predicate parsing, unconditional name→id translation + * (session auth included — the string grammar is name-keyed for every caller), + * cursor validation, and the response envelope. + */ +import { hybridAuthMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockCheckAccess, mockQueryRows, mockGate } = vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockQueryRows: vi.fn(), + mockGate: vi.fn(), +})) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'Access denied' }, { status: result.status }), + tablesV2GateError: mockGate, + } +}) + +vi.mock('@/lib/table', async () => { + // row-wire pulls the column-keys helpers through this barrel. + const columnKeys = await import('@/lib/table/column-keys') + return { ...columnKeys } +}) + +vi.mock('@/lib/table/rows/service', () => ({ + queryRows: mockQueryRows, +})) + +import { encodeCursor } from '@/lib/table/rows/cursor' +import { POST } from '@/app/api/table/[tableId]/query/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { id: 'col_aaa', name: 'name', type: 'string' }, + { id: 'col_bbb', name: 'wins', type: 'number' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + } +} + +function authAs(authType: 'session' | 'internal_jwt') { + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType, + }) +} + +function callQuery(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/query', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +const EMPTY_RESULT = { + rows: [], + rowCount: 0, + totalCount: 0, + limit: 0, + offset: 0, + nextCursor: null, +} + +describe('POST /api/table/[tableId]/query', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockQueryRows.mockResolvedValue(EMPTY_RESULT) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + authAs('session') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(404) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + authAs('session') + mockCheckAccess.mockResolvedValue({ ok: false, status: 403 }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() + }) + + it('translates predicate/sort column names to storage ids for SESSION auth too', async () => { + authAs('session') + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { + all: [ + { field: 'name', op: 'eq', value: 'John' }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }, + sort: [{ field: 'wins', direction: 'desc' }], + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ + all: [ + { field: 'col_aaa', op: 'eq', value: 'John' }, + { field: 'col_bbb', op: 'gte', value: 10 }, + ], + }) + expect(options.sort).toEqual({ col_bbb: 'desc' }) + expect(options.withExecutions).toBe(false) + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + authAs('internal_jwt') + const cursor = encodeCursor({ + lastRow: { id: 'row_1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + const res = await callQuery({ + workspaceId: 'workspace-1', + sort: [{ field: 'wins', direction: 'desc' }], + cursor, + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toMatch(/not valid for a sorted query/) + expect(body.code).toBe('CURSOR_SORT_CONFLICT') + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('returns 400 (not 500) for a cursor that decodes to a JSON primitive', async () => { + authAs('internal_jwt') + const res = await callQuery({ + workspaceId: 'workspace-1', + cursor: Buffer.from('42').toString('base64url'), + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toBe('Invalid cursor') + expect(body.code).toBe('INVALID_CURSOR') + }) + + it('returns 400 for a predicate referencing an unknown column', async () => { + authAs('internal_jwt') + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'nope', op: 'eq', value: 1 }] }, + }) + + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + }) + + it('passes nextCursor through the response envelope and skips the count on later pages', async () => { + authAs('internal_jwt') + mockQueryRows.mockResolvedValue({ ...EMPTY_RESULT, nextCursor: 'tok' }) + const cursor = encodeCursor({ + lastRow: { id: 'row_1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + + const res = await callQuery({ workspaceId: 'workspace-1', cursor }) + + expect(res.status).toBe(200) + const body = await res.json() + expect(body.data.nextCursor).toBe('tok') + const options = mockQueryRows.mock.calls[0][1] + expect(options.includeTotal).toBe(false) + expect(options.after).toEqual({ orderKey: 'a1', id: 'row_1' }) + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts new file mode 100644 index 00000000000..34a162200d1 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -0,0 +1,134 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { rowQueryContract, TABLE_QUERY_MAX_BODY_BYTES } from '@/lib/api/contracts/tables' +import { parseRequest } from '@/lib/api/server' +import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' +import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Sort, TableSchema } from '@/lib/table' +import { buildIdByName, sortSpecNamesToIds } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' +import { queryRows } from '@/lib/table/rows/service' +import { predicateToStorage } from '@/lib/table/select-values' +import { rowWireTranslators } from '@/app/api/table/row-wire' +import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' + +const logger = createLogger('TableRowQueryAPI') + +interface RowQueryRouteParams { + params: Promise<{ tableId: string }> +} + +/** + * POST /api/table/[tableId]/query — v2 row query. Typed `predicate`/`sort` + * objects (validated server-side) + opaque cursor pagination (no offset on the + * wire). Shares the same engine as the legacy GET /rows route via `queryRows`. + */ +export const POST = withRouteHandler(async (request: NextRequest, context: RowQueryRouteParams) => { + const requestId = generateRequestId() + + try { + const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) + } + + const parsed = await parseRequest(rowQueryContract, request, context, { + maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + const { params, body } = parsed.data + const { tableId } = params + + const accessResult = await checkAccess(tableId, authResult.userId, 'read') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + const { table } = accessResult + + if (body.workspaceId !== table.workspaceId) { + logger.warn( + `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${body.workspaceId}, Actual: ${table.workspaceId}` + ) + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } + + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(authResult.userId, body.workspaceId) + if (gateError) return gateError + + const schema = table.schema as TableSchema + const wire = rowWireTranslators(authResult.authType, schema) + const cursor = body.cursor ? decodeCursor(body.cursor) : undefined + + // Predicate/sort fields are column-NAME-keyed by construction (the caller + // authors names), so validate against the schema then translate names → + // storage ids unconditionally — unlike row data, this is not authType-dependent. + const idByName = buildIdByName(schema) + let predicate = body.predicate + if (predicate) { + validatePredicate(predicate, schema.columns) + predicate = predicateToStorage(predicate, schema) + } + let sortSpec = body.sort + if (sortSpec?.length) { + validateSortSpec(sortSpec, schema.columns) + sortSpec = sortSpecNamesToIds(sortSpec, idByName) + } + const sort: Sort | undefined = sortSpec?.length + ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) + : undefined + + // Cursor↔sort binding: keyset cursors are default-order only; an offset + // cursor must be replayed under the exact sort it was minted with. + if (cursor) assertCursorSortBinding(cursor, sort) + + const result = await queryRows( + table, + { + predicate, + sort, + limit: body.limit, + after: cursor?.after, + offset: cursor?.offset, + // Only the first page (no inbound cursor) pays for the total count. + includeTotal: !body.cursor, + // Executions are grid UI state; the v2 surface returns row data only + // and the byte budget deliberately measures just `data`. + withExecutions: false, + }, + requestId + ) + + return NextResponse.json({ + success: true, + data: { + rows: result.rows.map((r) => ({ + id: r.id, + data: wire.dataOut(r.data), + executions: r.executions, + position: r.position, + orderKey: r.orderKey ?? undefined, + createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), + updatedAt: r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), + })), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }) + } catch (error) { + if (isZodError(error)) return validationErrorResponse(error) + if (error instanceof TableQueryValidationError) { + return NextResponse.json( + { error: error.message, ...(error.code ? { code: error.code } : {}) }, + { status: 400 } + ) + } + logger.error(`[${requestId}] Error querying rows (v2):`, error) + return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts index 139427066cb..4b892a39340 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.test.ts @@ -6,14 +6,16 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table' -const { mockCheckAccess, mockFindRowMatches } = vi.hoisted(() => ({ +const { mockCheckAccess, mockFindRowMatches, mockTableFilterError } = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockFindRowMatches: vi.fn(), + mockTableFilterError: vi.fn(), })) vi.mock('@/app/api/table/utils', async () => { const { NextResponse } = await import('next/server') return { + tableFilterError: mockTableFilterError, checkAccess: mockCheckAccess, accessError: (result: { status: number }) => NextResponse.json( @@ -67,6 +69,7 @@ describe('GET /api/table/[tableId]/rows/find', () => { authType: 'session', }) mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockTableFilterError.mockReturnValue(null) mockFindRowMatches.mockResolvedValue({ matches: [{ ordinal: 4, rowId: 'row_4', column: 'name' }], truncated: false, @@ -120,8 +123,23 @@ describe('GET /api/table/[tableId]/rows/find', () => { ) }) + it('short-circuits with the filter gate response before searching', async () => { + const { NextResponse } = await import('next/server') + mockTableFilterError.mockReturnValueOnce( + NextResponse.json({ error: 'Unknown filter column "nope"' }, { status: 400 }) + ) + const res = await callGet({ + workspaceId: 'workspace-1', + q: 'foo', + filter: JSON.stringify({ all: [{ field: 'nope', op: 'eq', value: 1 }] }), + }) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + expect(mockFindRowMatches).not.toHaveBeenCalled() + }) + it('maps a TableQueryValidationError to 400', async () => { - const { TableQueryValidationError } = await import('@/lib/table/sql') + const { TableQueryValidationError } = await import('@/lib/table/errors') mockFindRowMatches.mockRejectedValueOnce(new TableQueryValidationError('Invalid field name')) const res = await callGet({ workspaceId: 'workspace-1', q: 'foo' }) expect(res.status).toBe(400) diff --git a/apps/sim/app/api/table/[tableId]/rows/find/route.ts b/apps/sim/app/api/table/[tableId]/rows/find/route.ts index 77a476ee145..f72c86633f6 100644 --- a/apps/sim/app/api/table/[tableId]/rows/find/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/find/route.ts @@ -5,10 +5,11 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server/validation import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Sort } from '@/lib/table' +import type { Filter, Sort, SortSpec, TablePredicate } from '@/lib/table' +import { TableQueryValidationError } from '@/lib/table/errors' +import { toLegacyFilter, toLegacySort } from '@/lib/table/query-builder/converters' import { findRowMatches } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' -import { accessError, checkAccess } from '@/app/api/table/utils' +import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils' const logger = createLogger('TableRowsFindAPI') @@ -58,9 +59,23 @@ export const GET = withRouteHandler( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + // Same gate as the bulk/async routes: the predicate branch rejects unknown + // storage keys (a typo'd field must 400, not return zero matches), the + // legacy branch keeps its compile check. + const filterError = tableFilterError( + validated.filter as Filter | TablePredicate | undefined, + table.schema.columns + ) + if (filterError) return filterError + const { matches, truncated } = await findRowMatches( table, - { q: validated.q, filter: validated.filter, sort: validated.sort }, + { + q: validated.q, + // Dual-grammar wire: findRowMatches compiles the legacy pair. + filter: toLegacyFilter(validated.filter as Filter | TablePredicate | undefined), + sort: toLegacySort(validated.sort as Sort | SortSpec | undefined), + }, requestId ) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index 8c5fbab9276..b6162110b57 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -6,11 +6,20 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { TableDefinition } from '@/lib/table' -const { mockCheckAccess, mockInsertRow, mockValidateRowData, mockQueryRows } = vi.hoisted(() => ({ +const { + mockCheckAccess, + mockInsertRow, + mockValidateRowData, + mockQueryRows, + mockUpdateRowsByFilter, + mockDeleteRowsByFilter, +} = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockInsertRow: vi.fn(), mockValidateRowData: vi.fn(), mockQueryRows: vi.fn(), + mockUpdateRowsByFilter: vi.fn(), + mockDeleteRowsByFilter: vi.fn(), })) vi.mock('@/app/api/table/utils', async () => { @@ -31,9 +40,9 @@ vi.mock('@/lib/table', async () => { insertRow: mockInsertRow, batchInsertRows: vi.fn(), batchUpdateRows: vi.fn(), - deleteRowsByFilter: vi.fn(), + deleteRowsByFilter: mockDeleteRowsByFilter, deleteRowsByIds: vi.fn(), - updateRowsByFilter: vi.fn(), + updateRowsByFilter: mockUpdateRowsByFilter, validateBatchRows: vi.fn(), validateRowData: mockValidateRowData, validateRowSize: vi.fn(() => ({ valid: true })), @@ -48,7 +57,7 @@ vi.mock('@/lib/table/sql', () => ({ TableQueryValidationError: class TableQueryValidationError extends Error {}, })) -import { GET, POST } from '@/app/api/table/[tableId]/rows/route' +import { DELETE, GET, POST, PUT } from '@/app/api/table/[tableId]/rows/route' function buildTable(): TableDefinition { return { @@ -217,4 +226,147 @@ describe('GET /api/table/[tableId]/rows', () => { const body = await res.json() expect(body.data.rows[0].data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) }) + + /** + * The grid now speaks the v2 grammar on this route: a predicate-shaped filter + * takes the NATIVE predicate path into queryRows (not a downgrade), and an + * ordered sort spec compiles to the record the engine's sort builder takes. + */ + it('routes a predicate filter + spec sort natively for session callers', async () => { + authAs('session') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }), + sort: JSON.stringify([{ field: 'col_bbb', direction: 'desc' }]), + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) + expect(options.filter).toBeUndefined() + expect(options.sort).toEqual({ col_bbb: 'desc' }) + }) + + it('translates a name-keyed predicate for internal-JWT callers', async () => { + authAs('internal_jwt') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }), + sort: JSON.stringify([{ field: 'Age', direction: 'asc' }]), + }) + + expect(res.status).toBe(200) + const options = mockQueryRows.mock.calls[0][1] + expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) + expect(options.sort).toEqual({ col_bbb: 'asc' }) + }) + + /** + * Storage validation runs post-translation, mirroring bulk PUT/DELETE: a + * typo'd field must 400, not compile to a clause that matches nothing and + * read back as a plausible empty page. + */ + it('400s a predicate naming an unknown column instead of returning an empty page', async () => { + authAs('session') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ all: [{ field: 'col_nope', op: 'eq', value: 'Ada' }] }), + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.error).toMatch(/Unknown filter column "col_nope"/) + expect(mockQueryRows).not.toHaveBeenCalled() + }) +}) + +describe('PUT/DELETE /api/table/[tableId]/rows — predicate filters', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockUpdateRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) + mockDeleteRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) + }) + + function callPut(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return PUT(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + } + + function callDelete(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return DELETE(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + } + + /** + * Keying follows the caller (PR #6067 review): the grid authors ID-keyed + * predicates and the session wire is identity, so ids pass through — and a + * NAME under session auth is just an unknown storage key, rejected like any + * other typo rather than half-translated. + */ + it('PUT passes an id-keyed predicate through untouched under SESSION auth', async () => { + authAs('session') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }, + data: { col_aaa: 'Grace' }, + }) + + expect(res.status).toBe(200) + const args = mockUpdateRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) + }) + + it('PUT rejects an unknown storage key under SESSION auth with 400', async () => { + authAs('session') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, + data: { col_aaa: 'Grace' }, + }) + expect(res.status).toBe(400) + expect(mockUpdateRowsByFilter).not.toHaveBeenCalled() + }) + + it('PUT translates a name-keyed predicate for INTERNAL_JWT callers', async () => { + authAs('internal_jwt') + const res = await callPut({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, + data: { Name: 'Grace' }, + }) + + expect(res.status).toBe(200) + const args = mockUpdateRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) + }) + + it('DELETE accepts the predicate and rejects an unknown column with 400', async () => { + authAs('internal_jwt') + const ok = await callDelete({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Age', op: 'gte', value: 30 }] }, + }) + expect(ok.status).toBe(200) + const args = mockDeleteRowsByFilter.mock.calls[0][1] + expect(args.filter).toEqual({ $and: [{ col_bbb: { $gte: 30 } }] }) + + const bad = await callDelete({ + workspaceId: 'workspace-1', + filter: { all: [{ field: 'Nope', op: 'eq', value: 1 }] }, + }) + expect(bad.status).toBe(400) + expect((await bad.json()).error).toMatch(/Unknown filter column/) + }) }) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 5c73dd8dff1..748c7138b50 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -9,11 +9,11 @@ import { updateRowsByFilterBodySchema, } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' -import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' +import { isZodError, parseJsonBody, validationErrorResponse } from '@/lib/api/server/validation' import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Filter, RowData, Sort, TableRowsCursor, TableSchema } from '@/lib/table' +import type { Filter, RowData, Sort, SortSpec, TableRowsCursor, TableSchema } from '@/lib/table' import { batchInsertRows, batchUpdateRows, @@ -25,13 +25,56 @@ import { validateRowData, validateRowSize, } from '@/lib/table' +import { TableQueryValidationError } from '@/lib/table/errors' +import { isTablePredicate, predicateToFilter } from '@/lib/table/query-builder/converters' +import { + validatePredicateShape, + validateStoragePredicate, +} from '@/lib/table/query-builder/validate' import { queryRows } from '@/lib/table/rows/service' -import { TableQueryValidationError } from '@/lib/table/sql' -import { rowWireTranslators } from '@/app/api/table/row-wire' +import type { TablePredicate } from '@/lib/table/types' +import { type RowWireTranslators, rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' const logger = createLogger('TableRowsAPI') +/** Dual-grammar sort: an ordered spec (v2) or the legacy record, either keying. */ +function resolveWireSort( + sort: Sort | SortSpec | undefined, + wire: RowWireTranslators +): Sort | undefined { + if (!sort) return undefined + if (!Array.isArray(sort)) return wire.sortIn(sort) + const spec = wire.sortSpecIn(sort) + return spec.length > 0 ? Object.fromEntries(spec.map((s) => [s.field, s.direction])) : undefined +} + +/** + * Resolves a bulk-op filter to a storage-id-keyed legacy `Filter`. The v2 + * predicate tree is column-NAME-keyed by construction (the caller authors + * names), so it validates then translates names → ids unconditionally — unlike + * the legacy object form, whose keying follows the caller's wire dialect + * (`wire.filterIn`: ids from the UI, names from workflow tools). + */ +function resolveBulkFilter( + raw: TablePredicate | Filter, + schema: TableSchema, + wire: RowWireTranslators +): Filter { + if (isTablePredicate(raw)) { + // Shape first (keying-agnostic: hybrid nodes, leaf value rules), then let + // the wire translate — identity for the ID-keyed grid, names→ids for + // workflow tools — and validate the RESULT against storage keys. Post- + // translation, any unresolved field is a typo in the caller's own keying, + // and on a destructive path a typo must 400, not silently match nothing. + validatePredicateShape(raw) + const translated = wire.predicateIn(raw) + validateStoragePredicate(translated, schema.columns) + return predicateToFilter(translated) + } + return wire.filterIn(raw) +} + interface TableRowsRouteParams { params: Promise<{ tableId: string }> } @@ -259,8 +302,22 @@ export const GET = withRouteHandler( const result = await queryRows( table, { - filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined, - sort: validated.sort ? wire.sortIn(validated.sort) : undefined, + ...(validated.filter && isTablePredicate(validated.filter as Filter | TablePredicate) + ? { + predicate: (() => { + // Shape-check first: nothing upstream validates this branch, and + // an unchecked hybrid node would silently widen the result. + validatePredicateShape(validated.filter as TablePredicate) + const translated = wire.predicateIn(validated.filter as TablePredicate) + // Post-translation storage check, mirroring the bulk PUT/DELETE + // paths: a typo'd field must 400, not compile to a clause that + // matches nothing and read back as an empty page. + validateStoragePredicate(translated, (table.schema as TableSchema).columns) + return translated + })(), + } + : { filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined }), + sort: resolveWireSort(validated.sort as Sort | SortSpec | undefined, wire), limit: validated.limit, offset: validated.offset, after: validated.after, @@ -287,6 +344,7 @@ export const GET = withRouteHandler( totalCount: result.totalCount, limit: result.limit, offset: result.offset, + nextCursor: result.nextCursor, }, }) } catch (error) { @@ -316,12 +374,12 @@ export const PUT = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = updateRowsByFilterBodySchema.parse(body) @@ -351,7 +409,11 @@ export const PUT = withRouteHandler( const result = await updateRowsByFilter( table, { - filter: wire.filterIn(validated.filter as Filter), + filter: resolveBulkFilter( + validated.filter as TablePredicate | Filter, + table.schema as TableSchema, + wire + ), data: patchData, limit: validated.limit, actorUserId: authResult.userId, @@ -410,12 +472,12 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = deleteTableRowsBodySchema.parse(body) @@ -457,7 +519,11 @@ export const DELETE = withRouteHandler( const result = await deleteRowsByFilter( table, { - filter: wire.filterIn(validated.filter as Filter), + filter: resolveBulkFilter( + validated.filter as TablePredicate | Filter, + table.schema as TableSchema, + wire + ), limit: validated.limit, }, requestId @@ -504,12 +570,12 @@ export const PATCH = withRouteHandler( return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - let body: unknown - try { - body = await request.json() - } catch { - return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 }) - } + // Bulk write bodies carry caller-supplied row data, so they can legitimately + // be large — but not unbounded. `parseJsonBody` applies the platform cap + // (413 past it) that a raw `request.json()` on this destructive surface skips. + const parsedBody = await parseJsonBody(request) + if (!parsedBody.success) return parsedBody.response + const body = parsedBody.data const validated = batchUpdateTableRowsBodySchema.parse(body) diff --git a/apps/sim/app/api/table/row-wire.ts b/apps/sim/app/api/table/row-wire.ts index d8458e56ec5..f4c0a85d988 100644 --- a/apps/sim/app/api/table/row-wire.ts +++ b/apps/sim/app/api/table/row-wire.ts @@ -1,13 +1,14 @@ import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' -import type { Filter, RowData, Sort, TableSchema } from '@/lib/table' +import type { Filter, RowData, Sort, SortSpec, TablePredicate, TableSchema } from '@/lib/table' import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, filterNamesToIds, rowDataNameToId, sortNamesToIds, + sortSpecNamesToIds, } from '@/lib/table/column-keys' -import { resolveFilterSelectValues } from '@/lib/table/select-values' +import { predicateToStorage, resolveFilterSelectValues } from '@/lib/table/select-values' export interface RowWireTranslators { /** Inbound row data: wire keys → storage column ids. */ @@ -18,6 +19,10 @@ export interface RowWireTranslators { filterIn: (filter: Filter) => Filter /** Inbound sort: wire field refs → storage column ids. */ sortIn: (sort: Sort) => Sort + /** Inbound v2 predicate: wire field refs → storage column ids. */ + predicateIn: (predicate: TablePredicate) => TablePredicate + /** Inbound v2 sort spec: wire field refs → storage column ids. */ + sortSpecIn: (sort: SortSpec) => SortSpec } /** @@ -33,15 +38,26 @@ export function rowWireTranslators( ): RowWireTranslators { if (authType !== AuthType.INTERNAL_JWT) { const identity = (value: T): T => value - return { dataIn: identity, dataOut: identity, filterIn: identity, sortIn: identity } + return { + dataIn: identity, + dataOut: identity, + filterIn: identity, + sortIn: identity, + predicateIn: identity, + sortSpecIn: identity, + } } const idByName = buildIdByName(schema) return { dataOut: namedRowMapper(schema.columns), dataIn: (data) => rowDataNameToId(data, idByName), - // Rekey field refs name → id, then resolve select operand names → ids. + // Rekey field refs name → id, then resolve select operand names → ids. Both + // grammars need that second step: a select cell stores an option id, so a + // filter written with the option NAME matches nothing without it. filterIn: (filter) => resolveFilterSelectValues(filterNamesToIds(filter, idByName), schema.columns), sortIn: (sort) => sortNamesToIds(sort, idByName), + predicateIn: (predicate) => predicateToStorage(predicate, schema), + sortSpecIn: (sort) => sortSpecNamesToIds(sort, idByName), } } diff --git a/apps/sim/app/api/table/utils.test.ts b/apps/sim/app/api/table/utils.test.ts index 2b23d45ffe9..99d0ce0c5a5 100644 --- a/apps/sim/app/api/table/utils.test.ts +++ b/apps/sim/app/api/table/utils.test.ts @@ -3,7 +3,8 @@ */ import { describe, expect, it } from 'vitest' import { TableRowLimitError } from '@/lib/table/billing' -import { rootErrorMessage, rowWriteErrorResponse } from '@/app/api/table/utils' +import type { ColumnDefinition } from '@/lib/table/types' +import { rootErrorMessage, rowWriteErrorResponse, tableFilterError } from '@/app/api/table/utils' /** Mimics drizzle's DrizzleQueryError: message is the failed SQL, real error on `cause`. */ function wrapLikeDrizzle(cause: Error): Error { @@ -53,3 +54,50 @@ describe('rowWriteErrorResponse', () => { expect(rowWriteErrorResponse(wrapLikeDrizzle(new Error('deadlock detected')))).toBeNull() }) }) + +/** + * The async destructive routes (delete-async, cancel-runs, columns/run) + * validate the WIRE filter here. The predicate branch must reject unknown + * storage keys the way the sync bulk routes do — the `toLegacyFilter` + * downgrade compiles a typo'd field into a clause that silently matches + * nothing, turning a scoped delete/run into a no-op. + */ +describe('tableFilterError', () => { + const columns: ColumnDefinition[] = [{ id: 'col_status', name: 'status', type: 'string' }] + + it('returns null for an absent filter and a valid id-keyed predicate', () => { + expect(tableFilterError(undefined, columns)).toBeNull() + expect( + tableFilterError({ all: [{ field: 'col_status', op: 'eq', value: 'x' }] }, columns) + ).toBeNull() + expect(tableFilterError({ all: [{ field: 'createdAt', op: 'isNotNull' }] }, columns)).toBeNull() + }) + + it('400s a predicate naming an unknown storage key', async () => { + const response = tableFilterError( + { all: [{ field: 'statuss', op: 'eq', value: 'x' }] }, + columns + ) + expect(response?.status).toBe(400) + const body = await response?.json() + expect(body.error).toMatch(/Unknown filter column "statuss"/) + }) + + it('400s a structurally invalid predicate (empty group, dual group keys)', () => { + expect(tableFilterError({ all: [] } as never, columns)?.status).toBe(400) + expect( + tableFilterError( + { + all: [{ field: 'col_status', op: 'eq', value: 'a' }], + any: [{ field: 'col_status', op: 'eq', value: 'b' }], + } as never, + columns + )?.status + ).toBe(400) + }) + + it('still validates the legacy grammar through buildFilterClause', () => { + expect(tableFilterError({ col_status: 'x' }, columns)).toBeNull() + expect(tableFilterError({ col_status: { $regex: 'x' } } as never, columns)?.status).toBe(400) + }) +}) diff --git a/apps/sim/app/api/table/utils.ts b/apps/sim/app/api/table/utils.ts index f3c62d56022..835866d829a 100644 --- a/apps/sim/app/api/table/utils.ts +++ b/apps/sim/app/api/table/utils.ts @@ -7,12 +7,35 @@ import { deleteTableColumnBodySchema, updateTableColumnBodySchema, } from '@/lib/api/contracts/tables' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import type { MultipartError } from '@/lib/core/utils/multipart' -import type { ColumnDefinition, Filter, TableDefinition } from '@/lib/table' +import type { ColumnDefinition, Filter, TableDefinition, TablePredicate } from '@/lib/table' import { buildFilterClause, getTableById, TableQueryValidationError } from '@/lib/table' import { USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableLockedError } from '@/lib/table/mutation-locks' +import { isTablePredicate } from '@/lib/table/query-builder/converters' +import { validateStoragePredicate } from '@/lib/table/query-builder/validate' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' +import { getWorkspaceOrganizationId } from '@/lib/workspaces/utils' + +/** + * Gate for the v2 tables HTTP API (`tables-v2-api` flag). Returns a 404 response + * when the flag is off for the caller — the surface behaves as if it doesn't + * exist — or `null` to proceed. Gated by userId + the workspace's org cohort. + * + * **Call this AFTER the authz check, never before.** Ahead of authz it does a + * primary-DB read keyed on a caller-supplied `workspaceId`, and the 404-vs-403 + * split tells an unauthorized caller whether that workspace's org is in the + * rollout cohort. + */ +export async function tablesV2GateError( + userId: string, + workspaceId: string +): Promise { + const orgId = await getWorkspaceOrganizationId(workspaceId) + if (await isFeatureEnabled('tables-v2-api', { userId, orgId })) return null + return NextResponse.json({ error: 'Not found' }, { status: 404 }) +} /** * Maps a {@link TableLockedError} thrown by the service layer to a 423 response @@ -33,17 +56,30 @@ export function tableLockErrorResponse(error: unknown): NextResponse | null { } /** - * Validates a `filter` against the table's column schema, returning a 400 response on a bad field - * (or `null` when the filter is valid or absent). Shared by the routes that accept a filter - * (`delete-async`, `columns/run`) so a bad field fails fast with a clear message. + * Validates a wire `filter` (either grammar) against the table's column schema, + * returning a 400 response on a bad field (or `null` when the filter is valid or + * absent). Shared by the routes that accept a filter (`delete-async`, + * `cancel-runs`, `columns/run`) so a bad field fails fast with a clear message. + * + * Pass the WIRE filter, not the `toLegacyFilter` downgrade: the downgrade + * compiles cleanly through `buildFilterClause` even when a predicate leaf names + * a column that doesn't exist, so validating only the downgraded form lets a + * typo'd field become a clause that silently matches nothing — a no-op where + * the sync bulk routes 400. */ export function tableFilterError( - filter: Filter | undefined, + filter: Filter | TablePredicate | undefined, columns: ColumnDefinition[] ): NextResponse | null { if (!filter) return null try { - buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, columns) + if (isTablePredicate(filter)) { + // These routes speak storage keys (session grid uses column ids; system + // columns keep their names) — same keying the sync bulk routes validate. + validateStoragePredicate(filter, columns) + } else { + buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, columns) + } return null } catch (error) { if (error instanceof TableQueryValidationError) { diff --git a/apps/sim/app/api/v1/middleware.ts b/apps/sim/app/api/v1/middleware.ts index 25357c19d18..bdc82a6613a 100644 --- a/apps/sim/app/api/v1/middleware.ts +++ b/apps/sim/app/api/v1/middleware.ts @@ -18,7 +18,12 @@ import { authenticateV1Request } from '@/app/api/v1/auth' const logger = createLogger('V1Middleware') const rateLimiter = new RateLimiter() -export type V1Endpoint = +/** + * Endpoint labels for public API auth/rate-limit telemetry. Version-neutral: the + * v1 and v2 public surfaces share the same `authenticateV1Request` + `api-endpoint` + * rate bucket, so the label is only a log/metric dimension, not a policy switch. + */ +export type ApiEndpoint = | 'logs' | 'logs-detail' | 'workflows' @@ -39,6 +44,8 @@ export type V1Endpoint = | 'knowledge-detail' | 'knowledge-search' | 'copilot-chat' + | 'v2-tables' + | 'v2-table-rows' export interface RateLimitResult { allowed: boolean @@ -65,7 +72,7 @@ export interface AuthorizedRequest { export async function checkRateLimit( request: NextRequest, - endpoint: V1Endpoint = 'logs' + endpoint: ApiEndpoint = 'logs' ): Promise { try { const auth = await authenticateV1Request(request) @@ -144,7 +151,7 @@ export async function checkRateLimit( */ export async function authenticateRequest( request: NextRequest, - endpoint: V1Endpoint + endpoint: ApiEndpoint ): Promise { const requestId = generateRequestId() const rateLimit = await checkRateLimit(request, endpoint) diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts index cc56cc61183..bef97b4d8f0 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/route.ts @@ -28,9 +28,9 @@ import { rowDataNameToId, sortNamesToIds, } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' import { queryRows } from '@/lib/table/rows/service' import { resolveFilterSelectValues } from '@/lib/table/select-values' -import { TableQueryValidationError } from '@/lib/table/sql' import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils' import { checkRateLimit, @@ -190,6 +190,10 @@ export const GET = withRouteHandler(async (request: NextRequest, context: TableR totalCount: result.totalCount, limit: result.limit, offset: result.offset, + // Non-null when more rows exist; a page may return fewer than `limit` + // rows (byte budget) with more remaining, so page fullness is not a + // termination signal — external pagers should stop on null. + nextCursor: result.nextCursor, }, }) } catch (error) { diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts new file mode 100644 index 00000000000..37d8e774d8b --- /dev/null +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.test.ts @@ -0,0 +1,255 @@ +/** + * @vitest-environment node + * + * Public v2 query POST: typed predicate name→id translation, bounded-default vs + * explicit-unbounded limit, cursor validation, workspace scoping, + * and name-keyed row output. + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockCheckAccess, mockQueryRows, mockCheckRateLimit, mockCheckWorkspaceScope, mockGate } = + vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockQueryRows: vi.fn(), + mockCheckRateLimit: vi.fn(), + mockCheckWorkspaceScope: vi.fn(), + mockGate: vi.fn(), + })) + +vi.mock('@/app/api/v1/middleware', async () => { + const { NextResponse } = await import('next/server') + return { + checkRateLimit: mockCheckRateLimit, + checkWorkspaceScope: mockCheckWorkspaceScope, + createRateLimitResponse: (r: { error?: string }) => + NextResponse.json( + { error: r.error ?? 'Rate limit exceeded' }, + { status: r.error ? 401 : 429 } + ), + } +}) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'Access denied' }, { status: result.status }), + tablesV2GateError: mockGate, + } +}) + +vi.mock('@/lib/table', async () => { + const columnKeys = await import('@/lib/table/column-keys') + return { ...columnKeys } +}) + +vi.mock('@/lib/table/rows/service', () => ({ queryRows: mockQueryRows })) + +import { encodeCursor } from '@/lib/table/rows/cursor' +import { POST } from '@/app/api/v2/tables/[tableId]/query/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { id: 'col_name', name: 'name', type: 'string' }, + { id: 'col_wins', name: 'wins', type: 'number' }, + { id: 'col_status', name: 'status', type: 'string' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + } +} + +const EMPTY_RESULT = { + rows: [], + rowCount: 0, + totalCount: 0, + limit: 100, + offset: 0, + nextCursor: null, +} + +function callQuery(body: Record) { + const req = new NextRequest('http://localhost:3000/api/v2/tables/tbl_1/query', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +describe('POST /api/v2/tables/[tableId]/query', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue({ + allowed: true, + userId: 'user-1', + keyType: 'workspace', + workspaceId: 'workspace-1', + }) + mockCheckWorkspaceScope.mockResolvedValue(null) + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockQueryRows.mockResolvedValue(EMPTY_RESULT) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(404) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + mockCheckAccess.mockResolvedValue({ ok: false, status: 403 }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() + }) + + it('translates a name-keyed predicate to storage ids', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { + all: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }, + }) + expect(res.status).toBe(200) + expect(mockQueryRows.mock.calls[0][1].predicate).toEqual({ + all: [ + { field: 'col_status', op: 'eq', value: 'active' }, + { field: 'col_wins', op: 'gte', value: 10 }, + ], + }) + }) + + it('applies the bounded default limit when omitted', async () => { + await callQuery({ workspaceId: 'workspace-1' }) + expect(mockQueryRows.mock.calls[0][1].limit).toBe(100) + }) + + it('treats limit=0 as the explicit unbounded opt-in', async () => { + await callQuery({ workspaceId: 'workspace-1', limit: 0 }) + expect(mockQueryRows.mock.calls[0][1].limit).toBeUndefined() + }) + + it('rejects a limit above the max', async () => { + const res = await callQuery({ workspaceId: 'workspace-1', limit: 5000 }) + expect(res.status).toBe(400) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects the removed regex ops at the contract boundary', async () => { + // `match`/`imatch` are no longer in FILTER_OPS — a catastrophic-backtracking + // pattern can pin a shared-pool connection, and nothing shipped depends on them. + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'name', op: 'match', value: '^jo' }] }, + }) + expect(res.status).toBe(400) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects a predicate referencing an unknown column', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + predicate: { all: [{ field: 'nope', op: 'eq', value: 1 }] }, + }) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/Unknown filter column/) + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'r1', orderKey: 'a1' }, + keysetValid: true, + nextOffset: 1, + }) + const res = await callQuery({ + workspaceId: 'workspace-1', + sort: [{ field: 'wins', direction: 'desc' }], + cursor, + }) + expect(res.status).toBe(400) + expect((await res.json()).code).toBe('CURSOR_SORT_CONFLICT') + }) + + it('returns 400 INVALID_CURSOR for a malformed cursor', async () => { + const res = await callQuery({ + workspaceId: 'workspace-1', + cursor: Buffer.from('42').toString('base64url'), + }) + expect(res.status).toBe(400) + expect((await res.json()).code).toBe('INVALID_CURSOR') + }) + + it('surfaces a workspace-scope 403 from the middleware', async () => { + const { NextResponse } = await import('next/server') + mockCheckWorkspaceScope.mockResolvedValue( + NextResponse.json({ error: 'not authorized' }, { status: 403 }) + ) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(403) + expect(mockQueryRows).not.toHaveBeenCalled() + }) + + it('rejects a workspace-id mismatch against the table', async () => { + const res = await callQuery({ workspaceId: 'other-ws' }) + expect(res.status).toBe(400) + expect((await res.json()).error).toBe('Invalid workspace ID') + }) + + it('returns name-keyed row data with no storage internals and a private cache header', async () => { + mockQueryRows.mockResolvedValue({ + rows: [ + { + id: 'r1', + data: { col_status: 'active', col_wins: 12 }, + position: 3, + orderKey: 'a5', + executions: {}, + createdAt: new Date('2024-02-02'), + updatedAt: new Date('2024-02-03'), + }, + ], + rowCount: 1, + totalCount: 1, + limit: 100, + offset: 0, + nextCursor: null, + }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.headers.get('Cache-Control')).toBe('private, no-store') + expect((await res.json()).data.rows[0]).toEqual({ + id: 'r1', + data: { status: 'active', wins: 12 }, + createdAt: '2024-02-02T00:00:00.000Z', + updatedAt: '2024-02-03T00:00:00.000Z', + }) + }) + + it('returns the rate-limit response when the limiter denies the request', async () => { + mockCheckRateLimit.mockResolvedValue({ allowed: false }) + const res = await callQuery({ workspaceId: 'workspace-1' }) + expect(res.status).toBe(429) + expect(mockCheckAccess).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/v2/tables/[tableId]/query/route.ts b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts new file mode 100644 index 00000000000..4337c0b402c --- /dev/null +++ b/apps/sim/app/api/v2/tables/[tableId]/query/route.ts @@ -0,0 +1,151 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { TABLE_QUERY_MAX_BODY_BYTES } from '@/lib/api/contracts/tables' +import { V2_DEFAULT_ROW_LIMIT, v2QueryRowsContract } from '@/lib/api/contracts/v2/tables' +import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import type { Sort, TablePredicate, TableSchema } from '@/lib/table' +import { buildIdByName, sortSpecNamesToIds } from '@/lib/table' +import { namedRowMapper } from '@/lib/table/cell-format' +import { TableQueryValidationError } from '@/lib/table/errors' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' +import { queryRows } from '@/lib/table/rows/service' +import { predicateToStorage } from '@/lib/table/select-values' +import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' +import { + checkRateLimit, + checkWorkspaceScope, + createRateLimitResponse, +} from '@/app/api/v1/middleware' + +const logger = createLogger('V2TableQueryAPI') + +export const dynamic = 'force-dynamic' +export const revalidate = 0 + +/** Filters may carry user data; keep query responses out of shared caches. */ +const PRIVATE_NO_STORE = { 'Cache-Control': 'private, no-store' } as const + +interface QueryRouteParams { + params: Promise<{ tableId: string }> +} + +/** + * POST /api/v2/tables/[tableId]/query — public row query. Typed `predicate`/`sort` + * objects + opaque cursor pagination. Default page {@link V2_DEFAULT_ROW_LIMIT}; + * `limit=0` = unbounded (whole result or 400). + */ +export const POST = withRouteHandler(async (request: NextRequest, context: QueryRouteParams) => { + const requestId = generateRequestId() + + try { + const rateLimit = await checkRateLimit(request, 'v2-table-rows') + if (!rateLimit.allowed) return createRateLimitResponse(rateLimit) + + const userId = rateLimit.userId! + const parsed = await parseRequest(v2QueryRowsContract, request, context, { + maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + + const { tableId } = parsed.data.params + const { workspaceId, sort, cursor: cursorToken, limit } = parsed.data.body + + const scopeError = await checkWorkspaceScope(rateLimit, workspaceId) + if (scopeError) return scopeError + + const accessResult = await checkAccess(tableId, userId, 'read') + if (!accessResult.ok) return accessError(accessResult, requestId, tableId) + const { table } = accessResult + + if (workspaceId !== table.workspaceId) { + return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) + } + + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + + const schema = table.schema as TableSchema + const cursor = cursorToken ? decodeCursor(cursorToken) : undefined + + const idByName = buildIdByName(schema) + // Fuses the id→name key remap with select-cell value formatting, so a select + // cell surfaces its option NAME rather than the stored option id. + const toNamedRow = namedRowMapper(schema.columns) + let predicate: TablePredicate | undefined = parsed.data.body.predicate + if (predicate) { + validatePredicate(predicate, schema.columns) + predicate = predicateToStorage(predicate, schema) + } + let sortSpec = sort + if (sortSpec?.length) { + validateSortSpec(sortSpec, schema.columns) + sortSpec = sortSpecNamesToIds(sortSpec, idByName) + } + const sortObj: Sort | undefined = sortSpec?.length + ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) + : undefined + + // A cursor is only valid for the query shape it was minted under: keyset + // cursors bind to the default order, offset cursors to their sort. Runs on + // the STORAGE-keyed sort so the fingerprint matches what queryRows stamped. + if (cursor) assertCursorSortBinding(cursor, sortObj) + + // Public default is a bounded page (unlike the internal surface's unbounded + // omit). `limit=0` is the explicit unbounded opt-in. + const effectiveLimit = + limit === undefined ? V2_DEFAULT_ROW_LIMIT : limit === 0 ? undefined : limit + + const result = await queryRows( + table, + { + predicate, + sort: sortObj, + limit: effectiveLimit, + after: cursor?.after, + offset: cursor?.offset, + includeTotal: !cursorToken, + withExecutions: false, + }, + requestId + ) + + return NextResponse.json( + { + success: true, + data: { + rows: result.rows.map((r) => ({ + id: r.id, + data: toNamedRow(r.data), + createdAt: + r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), + updatedAt: + r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), + })), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }, + { headers: PRIVATE_NO_STORE } + ) + } catch (error) { + const validationResponse = validationErrorResponseFromError(error) + if (validationResponse) return validationResponse + + if (error instanceof TableQueryValidationError) { + return NextResponse.json( + { error: error.message, ...(error.code ? { code: error.code } : {}) }, + { status: 400, headers: PRIVATE_NO_STORE } + ) + } + + logger.error(`[${requestId}] Error querying rows (v2 public):`, error) + return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/v2/tables/route.test.ts b/apps/sim/app/api/v2/tables/route.test.ts new file mode 100644 index 00000000000..2880355c8df --- /dev/null +++ b/apps/sim/app/api/v2/tables/route.test.ts @@ -0,0 +1,131 @@ +/** + * @vitest-environment node + * + * Public v2 tables list: auth/scope gating, typed summary output, private cache header. + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table/types' + +const { mockListTables, mockCheckRateLimit, mockValidateWorkspaceAccess, mockGate } = vi.hoisted( + () => ({ + mockListTables: vi.fn(), + mockCheckRateLimit: vi.fn(), + mockValidateWorkspaceAccess: vi.fn(), + mockGate: vi.fn(), + }) +) + +vi.mock('@/app/api/v1/middleware', async () => { + const { NextResponse } = await import('next/server') + return { + checkRateLimit: mockCheckRateLimit, + validateWorkspaceAccess: mockValidateWorkspaceAccess, + createRateLimitResponse: (r: { error?: string }) => + NextResponse.json( + { error: r.error ?? 'Rate limit exceeded' }, + { status: r.error ? 401 : 429 } + ), + } +}) + +vi.mock('@/lib/table', async () => { + const actual = await import('@/lib/table/column-keys') + return { ...actual, listTables: mockListTables } +}) + +vi.mock('@/app/api/table/utils', () => ({ + normalizeColumn: (col: Record) => col, + tablesV2GateError: mockGate, +})) + +import { GET } from '@/app/api/v2/tables/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: 'A table', + schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] }, + metadata: null, + rowCount: 5, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-02'), + } +} + +function callList(query: string) { + const req = new NextRequest(`http://localhost:3000/api/v2/tables?${query}`) + return GET(req) +} + +describe('GET /api/v2/tables', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue({ allowed: true, userId: 'user-1', keyType: 'workspace' }) + mockValidateWorkspaceAccess.mockResolvedValue(null) + mockListTables.mockResolvedValue([buildTable()]) + mockGate.mockResolvedValue(null) + }) + + it('returns 404 when the tables-v2-api flag is off', async () => { + const { NextResponse } = await import('next/server') + mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(404) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { + const { NextResponse } = await import('next/server') + mockValidateWorkspaceAccess.mockResolvedValue( + NextResponse.json({ error: 'Access denied' }, { status: 403 }) + ) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(403) + expect(mockGate).not.toHaveBeenCalled() + }) + + it('returns a typed table summary with a private cache header', async () => { + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(200) + expect(res.headers.get('Cache-Control')).toBe('private, no-store') + const body = await res.json() + expect(body.data.totalCount).toBe(1) + expect(body.data.tables[0]).toMatchObject({ + id: 'tbl_1', + name: 'People', + description: 'A table', + rowCount: 5, + maxRows: 100, + createdAt: '2024-01-01T00:00:00.000Z', + }) + expect(body.data.tables[0].schema.columns[0].name).toBe('name') + }) + + it('400s when workspaceId is missing', async () => { + const res = await callList('') + expect(res.status).toBe(400) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('surfaces an access-denied response from the middleware', async () => { + const { NextResponse } = await import('next/server') + mockValidateWorkspaceAccess.mockResolvedValue( + NextResponse.json({ error: 'Access denied' }, { status: 403 }) + ) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(403) + expect(mockListTables).not.toHaveBeenCalled() + }) + + it('returns the rate-limit response when denied', async () => { + mockCheckRateLimit.mockResolvedValue({ allowed: false }) + const res = await callList('workspaceId=workspace-1') + expect(res.status).toBe(429) + }) +}) diff --git a/apps/sim/app/api/v2/tables/route.ts b/apps/sim/app/api/v2/tables/route.ts new file mode 100644 index 00000000000..be0c59c32b6 --- /dev/null +++ b/apps/sim/app/api/v2/tables/route.ts @@ -0,0 +1,78 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { v2ListTablesContract } from '@/lib/api/contracts/v2/tables' +import { parseRequest, validationErrorResponseFromError } from '@/lib/api/server' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { listTables, type TableSchema } from '@/lib/table' +import { normalizeColumn, tablesV2GateError } from '@/app/api/table/utils' +import { + checkRateLimit, + createRateLimitResponse, + validateWorkspaceAccess, +} from '@/app/api/v1/middleware' + +const logger = createLogger('V2TablesAPI') + +export const dynamic = 'force-dynamic' +export const revalidate = 0 + +/** Filters/ids can appear in query strings; keep list responses out of shared caches. */ +const PRIVATE_NO_STORE = { 'Cache-Control': 'private, no-store' } as const + +/** GET /api/v2/tables — list all tables in a workspace. */ +export const GET = withRouteHandler(async (request: NextRequest) => { + const requestId = generateRequestId() + + try { + const rateLimit = await checkRateLimit(request, 'v2-tables') + if (!rateLimit.allowed) return createRateLimitResponse(rateLimit) + + const userId = rateLimit.userId! + const parsed = await parseRequest(v2ListTablesContract, request, {}) + if (!parsed.success) return parsed.response + + const { workspaceId } = parsed.data.query + + const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) + if (accessError) return accessError + + // After authz: the gate reads the workspace's org off the primary DB, and its + // 404 would otherwise distinguish "not in the rollout cohort" from "no access". + const gateError = await tablesV2GateError(userId, workspaceId) + if (gateError) return gateError + + const tables = await listTables(workspaceId) + + return NextResponse.json( + { + success: true, + data: { + tables: tables.map((t) => { + const schemaData = t.schema as TableSchema + return { + id: t.id, + name: t.name, + description: t.description, + schema: { columns: schemaData.columns.map(normalizeColumn) }, + rowCount: t.rowCount, + maxRows: t.maxRows, + createdAt: + t.createdAt instanceof Date ? t.createdAt.toISOString() : String(t.createdAt), + updatedAt: + t.updatedAt instanceof Date ? t.updatedAt.toISOString() : String(t.updatedAt), + } + }), + totalCount: tables.length, + }, + }, + { headers: PRIVATE_NO_STORE } + ) + } catch (error) { + const validationResponse = validationErrorResponseFromError(error) + if (validationResponse) return validationResponse + + logger.error(`[${requestId}] Error listing tables:`, error) + return NextResponse.json({ error: 'Failed to list tables' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx index a0af627d296..61643e48d4e 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-filter/table-filter.tsx @@ -4,7 +4,7 @@ import { memo, useCallback, useMemo, useRef, useState } from 'react' import { Button, ChipDropdown, ChipInput } from '@sim/emcn' import { Plus, X } from '@sim/emcn/icons' import { generateShortId } from '@sim/utils/id' -import type { ColumnDefinition, Filter, FilterRule } from '@/lib/table' +import type { ColumnDefinition, FilterRule, TablePredicate } from '@/lib/table' import { getColumnId } from '@/lib/table/column-keys' import { COMPARISON_OPERATORS, @@ -12,7 +12,10 @@ import { SINGLE_SELECT_FILTER_OPERATORS, VALUELESS_OPERATORS, } from '@/lib/table/query-builder/constants' -import { filterRulesToFilter, filterToRules } from '@/lib/table/query-builder/converters' +import { + filterRulesToPredicate, + predicateToFilterRules, +} from '@/lib/table/query-builder/converters' const SINGLE_SELECT_COMPARISON_OPERATORS = COMPARISON_OPERATORS.filter((o) => SINGLE_SELECT_FILTER_OPERATORS.has(o.value) @@ -27,14 +30,14 @@ function selectFilterOperators(column: ColumnDefinition | undefined): Set void + filter: TablePredicate | null + onApply: (filter: TablePredicate | null) => void onClose: () => void } export function TableFilter({ columns, filter, onApply, onClose }: TableFilterProps) { const [rules, setRules] = useState(() => { - const fromFilter = filterToRules(filter) + const fromFilter = predicateToFilterRules(filter) return fromFilter.length > 0 ? fromFilter : [createRule(columns)] }) @@ -112,7 +115,7 @@ export function TableFilter({ columns, filter, onApply, onClose }: TableFilterPr const validRules = rulesRef.current.filter( (r) => r.column && (r.value || VALUELESS_OPERATORS.has(r.operator)) ) - onApply(filterRulesToFilter(validRules, columns)) + onApply(filterRulesToPredicate(validRules, columns)) }, [columns, onApply]) const handleClear = useCallback(() => { diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index ed93cdaca87..d7bf866e721 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -13,9 +13,9 @@ import type { RunLimit, RunMode, TableFindMatch } from '@/lib/api/contracts/tabl import { captureEvent } from '@/lib/posthog/client' import type { ColumnDefinition, - Filter, TableLocks, TableMetadata, + TablePredicate, TableRow as TableRowType, WorkflowGroup, } from '@/lib/table' @@ -121,7 +121,7 @@ export interface SelectionSnapshot { allRows: boolean rowCount: number /** Active filter when `allRows` is set — lets a filtered "select all" run only matching rows. */ - filter?: Filter + filter?: TablePredicate /** Deselected rows when `allRows` is set — runs/stops skip them. */ excludeRowIds?: string[] } | null @@ -199,7 +199,7 @@ interface TableGridProps { runMode: RunMode, rowIds?: string[], limit?: RunLimit, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => void /** Fire every runnable column on a single row (per-row gutter Play). */ @@ -209,14 +209,14 @@ interface TableGridProps { onRunRows: ( rowIds: string[] | undefined, runMode: RunMode, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => void /** Stop running workflows on `rowIds`. Per-row gutter Stop also funnels through here. */ onStopRows: (rowIds: string[]) => void /** Select-all Stop: table-wide, or scoped to the active filter when one is set. * `excludeRowIds` (deselected rows) keep running. */ - onStopAllRows: (filter?: Filter, excludeRowIds?: string[]) => void + onStopAllRows: (filter?: TablePredicate, excludeRowIds?: string[]) => void /** Single-row stop for the per-row gutter button. */ onStopRow: (rowId: string) => void /** diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts index 8026fbfa630..0fed31ca065 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.test.ts @@ -218,7 +218,7 @@ describe('useTable – ensureAllRowsLoaded', () => { }) it('encodes queryOptions.filter into the queryKey passed to getQueryData', async () => { - const filter = { column: 'name', operator: 'eq', value: 'Alice' } as never + const filter = { all: [{ field: 'name', op: 'eq' as const, value: 'Alice' }] } mockGetQueryData.mockReturnValue({ pages: makePages([3], 3) }) const { ensureAllRowsLoaded } = makeHook({ filter, sort: null }) await ensureAllRowsLoaded() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts index 9fcf4dd7db5..be473ae71d3 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table.ts @@ -4,13 +4,13 @@ import { useCallback, useMemo } from 'react' import { useQueryClient } from '@tanstack/react-query' import type { ColumnDefinition, - Filter, TableDefinition, + TablePredicate, TableRow, WorkflowGroup, } from '@/lib/table' import { TABLE_LIMITS } from '@/lib/table/constants' -import { pruneFilterForColumns } from '@/lib/table/query-builder/converters' +import { prunePredicateForColumns } from '@/lib/table/query-builder/converters' import type { FlattenOutputsBlockInput } from '@/lib/workflows/blocks/flatten-outputs' import { getBlock } from '@/blocks' import { @@ -51,7 +51,7 @@ export interface UseTableReturn { * select-all run/stop/delete — must scope with THIS, not the raw filter, or * the action targets a predicate the grid isn't displaying. */ - filter: Filter | null + filter: TablePredicate | null isLoadingRows: boolean refetchRows: () => void /** @@ -98,7 +98,7 @@ export function useTable({ workspaceId, tableId, queryOptions }: UseTableParams) // here, above every consumer of the rows query key, so the paged helpers below // can't rebuild the key from the unpruned filter and drift. const filter = useMemo( - () => pruneFilterForColumns(queryOptions.filter ?? null, tableData?.schema?.columns ?? []), + () => prunePredicateForColumns(queryOptions.filter ?? null, tableData?.schema?.columns ?? []), [queryOptions.filter, tableData?.schema?.columns] ) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 0540b5346b5..146c02c103a 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -12,10 +12,10 @@ import type { RunLimit, RunMode, TableViewWire } from '@/lib/api/contracts/table import { captureEvent } from '@/lib/posthog/client' import type { ColumnDefinition, - Filter, - Sort, SortDirection, + SortSpec, TableMetadata, + TablePredicate, TableRow as TableRowType, TableViewConfig, WorkflowGroup, @@ -255,7 +255,7 @@ export function Table({ selectionStats: { hasIncompleteOrFailed: false, hasCompleted: false, hasInFlight: false }, singleWorkflowCell: null, }) - const [filter, setFilter] = useState(null) + const [filter, setFilter] = useState(null) const [filterOpen, setFilterOpen] = useState(false) /** Hidden **column ids**. Lives here (not in the grid) because the filter * panel's Columns section edits it and the active view persists it. */ @@ -271,9 +271,9 @@ export function Table({ const hiddenColumnsRef = useRef(hiddenColumns) hiddenColumnsRef.current = hiddenColumns - /** Resolved single-column sort, or `null` when no column is active. */ - const sortQuery = useMemo( - () => (sortColumn ? { [sortColumn]: sortDirection } : null), + /** Resolved single-column sort as an ordered spec, or `null` when none is active. */ + const sortQuery = useMemo( + () => (sortColumn ? [{ field: sortColumn, direction: sortDirection }] : null), [sortColumn, sortDirection] ) @@ -426,10 +426,10 @@ export function Table({ if (!keep?.filter) setFilter(config?.filter ?? null) if (!keep?.hiddenColumns) setHiddenColumns(config?.hiddenColumns ?? []) if (keep?.sort) return - const sortEntry = config?.sort ? Object.entries(config.sort)[0] : undefined + const sortEntry = config?.sort?.[0] setTableParams({ - sort: sortEntry ? sortEntry[0] : null, - dir: sortEntry ? (sortEntry[1] as SortDirection) : null, + sort: sortEntry ? sortEntry.field : null, + dir: sortEntry ? (sortEntry.direction as SortDirection) : null, }) }, [setTableParams] @@ -651,7 +651,7 @@ export function Table({ const currentViewConfig = useMemo( () => ({ ...(activeView?.config ?? tableData?.metadata), - filter: effectiveFilter, + filter: effectiveFilter ?? null, sort: sortQuery, hiddenColumns: effectiveHiddenColumns, }), @@ -837,7 +837,7 @@ export function Table({ (args: { groupIds: string[] rowIds?: string[] - filter?: Filter + filter?: TablePredicate excludeRowIds?: string[] runMode: RunMode limit?: RunLimit @@ -876,7 +876,7 @@ export function Table({ runMode: RunMode, rowIds?: string[], limit?: RunLimit, - filter?: Filter, + filter?: TablePredicate, excludeRowIds?: string[] ) => { runScope({ @@ -893,7 +893,12 @@ export function Table({ ) const onRunRows = useCallback( - (rowIds: string[] | undefined, runMode: RunMode, filter?: Filter, excludeRowIds?: string[]) => { + ( + rowIds: string[] | undefined, + runMode: RunMode, + filter?: TablePredicate, + excludeRowIds?: string[] + ) => { runScope({ groupIds: tableWorkflowGroups.map((g) => g.id), rowIds, @@ -961,7 +966,7 @@ export function Table({ /** Select-all Stop — filter-scoped when a filter is active; deselected rows keep running. */ const onStopAllRows = useCallback( - (filter?: Filter, excludeRowIds?: string[]) => { + (filter?: TablePredicate, excludeRowIds?: string[]) => { // `sort` scopes the optimistic flip to the active view's cache (filtered stops // only cancel matching rows server-side). cancelRunsMutate({ scope: 'all', filter, sort: queryOptions.sort, excludeRowIds }) @@ -1061,7 +1066,7 @@ export function Table({ [columnOptions, sortColumn, sortDirection, setTableParams] ) - const handleFilterApply = (next: Filter | null) => { + const handleFilterApply = (next: TablePredicate | null) => { setFilter(next) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts index d7c7553c2ce..34618913acf 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/types.ts @@ -1,4 +1,4 @@ -import type { Filter, Sort, TableRow } from '@/lib/table' +import type { SortSpec, TablePredicate, TableRow } from '@/lib/table' /** * Reason the inline editor completed, used to determine navigation after save @@ -9,8 +9,8 @@ export type SaveReason = 'enter' | 'tab' | 'shift-tab' | 'blur' * Query options for filtering and sorting table data */ export interface QueryOptions { - filter: Filter | null - sort: Sort | null + filter: TablePredicate | null + sort: SortSpec | null } /** diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx index d11ec546900..9427c7a8ca5 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sort-builder/sort-builder.tsx @@ -5,7 +5,7 @@ import { Button, type ComboboxOption } from '@sim/emcn' import { generateId } from '@sim/utils/id' import { Plus } from 'lucide-react' import { useTableColumns } from '@/lib/table/hooks' -import { SORT_DIRECTIONS, type SortRule } from '@/lib/table/query-builder/constants' +import { SORT_DIRECTION_OPTIONS, type SortRule } from '@/lib/table/query-builder/constants' import { useCanonicalSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-canonical-sub-block-value' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' import { SortRuleRow } from './components/sort-rule-row' @@ -47,7 +47,7 @@ export function SortBuilder({ }, [propColumns, dynamicColumns]) const directionOptions = useMemo( - () => SORT_DIRECTIONS.map((dir) => ({ value: dir.value, label: dir.label })), + () => SORT_DIRECTION_OPTIONS.map((dir) => ({ value: dir.value, label: dir.label })), [] ) diff --git a/apps/sim/blocks/blocks/table.ts b/apps/sim/blocks/blocks/table.ts index 281bcd077e7..3e2a99f88d7 100644 --- a/apps/sim/blocks/blocks/table.ts +++ b/apps/sim/blocks/blocks/table.ts @@ -183,7 +183,7 @@ export const TableBlock: BlockConfig = { description: 'User-defined data tables', longDescription: 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows.', - docsLink: 'https://docs.simstudio.ai/tools/table', + docsLink: 'https://docs.sim.ai/integrations/table', category: 'blocks', bgColor: '#10B981', icon: TableIcon, diff --git a/apps/sim/blocks/blocks/table_v2.test.ts b/apps/sim/blocks/blocks/table_v2.test.ts new file mode 100644 index 00000000000..1288d912e82 --- /dev/null +++ b/apps/sim/blocks/blocks/table_v2.test.ts @@ -0,0 +1,119 @@ +/** + * @vitest-environment node + * + * Param-transformer behavior for the v2 Table block: filter/order resolution + * across builder vs editor modes, the required query limit, cursor artifact + * handling, and fail-fast limit parsing. + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/triggers', () => ({ + getTrigger: vi.fn(() => ({ subBlocks: [] })), +})) + +import { TableV2Block } from '@/blocks/blocks/table_v2' + +function params(input: Record): Record { + return TableV2Block.tools.config?.params?.(input as never) as Record +} + +describe('table_v2 query_rows transformer', () => { + it('keeps limit optional (byte-budget page) but fails fast on a non-numeric one', () => { + const out = params({ operation: 'query_rows', tableId: 't' }) + expect(out.limit).toBeUndefined() + expect(() => params({ operation: 'query_rows', tableId: 't', limit: 'abc' })).toThrow( + /Invalid Limit/ + ) + }) + + it('treats interpolated "null"/"undefined" cursor artifacts as absent', () => { + const out = params({ operation: 'query_rows', tableId: 't', limit: '10', cursor: 'null' }) + expect(out.cursor).toBeUndefined() + const out2 = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + cursor: 'undefined', + }) + expect(out2.cursor).toBeUndefined() + const out3 = params({ operation: 'query_rows', tableId: 't', limit: '10', cursor: 'tok' }) + expect(out3.cursor).toBe('tok') + }) + + it('compiles builder rules (basic canonical value) to a predicate + sort spec', () => { + const out = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + // filterInput/sortInput carry the ACTIVE canonical mode's value; the visual + // builders (basic) emit rule arrays. + filterInput: [ + { id: '1', logicalOperator: 'and', column: 'wins', operator: 'gte', value: '10' }, + ], + sortInput: [{ id: '1', column: 'wins', direction: 'desc' }], + }) + expect(out.filter).toEqual({ all: [{ field: 'wins', op: 'gte', value: 10 }] }) + expect(out.order).toEqual([{ field: 'wins', direction: 'desc' }]) + }) + + it('parses the editor JSON string (advanced canonical value) into a predicate', () => { + const out = params({ + operation: 'query_rows', + tableId: 't', + limit: '10', + filterInput: '{"all":[{"field":"name","op":"eq","value":"test"}]}', + }) + expect(out.filter).toEqual({ all: [{ field: 'name', op: 'eq', value: 'test' }] }) + }) +}) + +describe('table_v2 bulk transformers', () => { + const editorFilter = '{"all":[{"field":"name","op":"eq","value":"x"}]}' + + it('fails fast on a non-numeric bulk limit instead of widening to every match', () => { + expect(() => + params({ + operation: 'delete_rows_by_filter', + tableId: 't', + filterInput: editorFilter, + limit: 'abc', + }) + ).toThrow(/Invalid Limit/) + }) + + it('keeps the bulk limit optional', () => { + const out = params({ + operation: 'delete_rows_by_filter', + tableId: 't', + filterInput: editorFilter, + }) + expect(out.limit).toBeUndefined() + expect(out.filter).toEqual({ all: [{ field: 'name', op: 'eq', value: 'x' }] }) + }) +}) + +/** + * The Filter/Order fields are canonical pairs: a builder array in Builder mode, + * a JSON string in Editor mode. Clearing the editor field is the ordinary way to + * say "no filter" — it must not surface as a JSON parse error at run time. + */ +describe('table_v2 blank and malformed editor inputs', () => { + const base = { operation: 'query_rows', tableId: 'tbl_1', limit: '10' } + + it('treats a blank / whitespace filter or order as absent', () => { + for (const value of ['', ' ', '\n']) { + expect(params({ ...base, filterInput: value }).filter).toBeUndefined() + expect(params({ ...base, sortInput: value }).order).toBeUndefined() + } + }) + + it('treats an empty builder array as absent', () => { + expect(params({ ...base, filterInput: [] }).filter).toBeUndefined() + expect(params({ ...base, sortInput: [] }).order).toBeUndefined() + }) + + it('still reports genuinely malformed JSON', () => { + expect(() => params({ ...base, filterInput: '{not json}' })).toThrow(/Invalid JSON in Filter/) + expect(() => params({ ...base, sortInput: '{not json}' })).toThrow(/Invalid JSON in Sort/) + }) +}) diff --git a/apps/sim/blocks/blocks/table_v2.ts b/apps/sim/blocks/blocks/table_v2.ts new file mode 100644 index 00000000000..74dffc275a7 --- /dev/null +++ b/apps/sim/blocks/blocks/table_v2.ts @@ -0,0 +1,584 @@ +import { toError } from '@sim/utils/errors' +import { TableIcon } from '@/components/icons' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { filterRulesToPredicate, sortRulesToSortSpec } from '@/lib/table/query-builder/converters' +import type { FilterRule, SortRule, SortSpec, TablePredicate } from '@/lib/table/types' +import type { BlockConfig } from '@/blocks/types' +import type { TableQueryV2Response } from '@/tools/table/types' +import { getTrigger } from '@/triggers' + +/** + * Table v2 — same operations as the v1 Table block, but the filter grammar is a + * typed predicate tree (`{all:[{field:'wins',op:'gte',value:10}]}`), validated + * server-side. Pagination is an opaque cursor (no offset). The filter compiler, + * upsert conflict probe, and unique checks share one case-sensitive containment + * leaf, so upserts can't wedge on a case-mismatched unique value the way they + * could under v1. + */ + +function parseJSON(value: string | unknown, fieldName: string): unknown { + if (typeof value !== 'string') return value + // A blank editor field means "no filter/order", not malformed JSON. Without + // this, clearing the field throws `Unexpected end of JSON input` at run time. + if (value.trim() === '') return undefined + try { + return JSON.parse(value) + } catch (error) { + const errorMsg = toError(error).message + const unquotedValueMatch = value.match( + /:\s*([a-zA-Z][a-zA-Z0-9_\s]*[a-zA-Z0-9]|[a-zA-Z])\s*[,}]/ + ) + let hint = + 'Make sure all property names are in double quotes (e.g., {"name": "value"} not {name: "value"}).' + if (unquotedValueMatch) { + hint = + 'It looks like a string value is not quoted. When using block references in JSON, wrap them in double quotes: {"field": ""} not {"field": }.' + } + throw new Error(`Invalid JSON in ${fieldName}: ${errorMsg}. ${hint}`) + } +} + +interface TableBlockParams { + operation: string + tableId?: string + rowId?: string + data?: string | unknown + rows?: string | unknown + filterInput?: unknown + sortInput?: unknown + limit?: string + cursor?: string + conflictColumn?: string +} + +/** + * Resolves the effective predicate from the `filterInput` canonical param, which + * carries whichever mode's value the ⟷ toggle has active: a FilterRule[] (visual + * builder / basic) or a predicate-JSON string (editor / advanced, agent-authored). + * An array is builder rules; a string is raw JSON. Both compile to the same + * `TablePredicate` the route consumes. + */ +function resolveFilter(params: TableBlockParams): TablePredicate | undefined { + const raw = params.filterInput + if (Array.isArray(raw)) { + return raw.length > 0 ? (filterRulesToPredicate(raw as FilterRule[]) ?? undefined) : undefined + } + const parsed = parseJSON(raw, 'Filter') + return (parsed as TablePredicate | undefined) || undefined +} + +function resolveOrder(params: TableBlockParams): SortSpec | undefined { + const raw = params.sortInput + if (Array.isArray(raw)) { + return raw.length > 0 ? (sortRulesToSortSpec(raw as SortRule[]) ?? undefined) : undefined + } + const parsed = parseJSON(raw, 'Sort') + return (parsed as SortSpec | undefined) || undefined +} + +interface ParsedParams { + tableId?: string + rowId?: string + data?: unknown + rows?: unknown + filter?: TablePredicate + order?: SortSpec + limit?: number + cursor?: string + conflictTarget?: string +} + +/** + * Fail-fast limit parsing for bulk ops: an unparseable limit must never + * silently widen the operation to every matching row. + */ +function parseOptionalLimit(raw: string | undefined): number | undefined { + if (!raw) return undefined + const value = Number.parseInt(raw, 10) + if (Number.isNaN(value)) { + throw new Error(`Invalid Limit "${raw}" — expected a number.`) + } + return value +} + +/** + * `` interpolates the terminating page's `null` as the + * literal string "null" — treat those artifacts as "no cursor". + */ +function resolveCursor(raw: string | undefined): string | undefined { + if (!raw || raw === 'null' || raw === 'undefined') return undefined + return raw +} + +const paramTransformers: Record ParsedParams> = { + insert_row: (params) => ({ + tableId: params.tableId, + data: parseJSON(params.data, 'Row Data'), + }), + + upsert_row: (params) => ({ + tableId: params.tableId, + data: parseJSON(params.data, 'Row Data'), + conflictTarget: params.conflictColumn || undefined, + }), + + batch_insert_rows: (params) => ({ + tableId: params.tableId, + rows: parseJSON(params.rows, 'Rows Data'), + }), + + update_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + data: parseJSON(params.data, 'Row Data'), + }), + + // Bulk write-by-filter takes a predicate object, validated server-side. + update_rows_by_filter: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + data: parseJSON(params.data, 'Row Data'), + limit: parseOptionalLimit(params.limit), + }), + + delete_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + }), + + delete_rows_by_filter: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + limit: parseOptionalLimit(params.limit), + }), + + get_row: (params) => ({ + tableId: params.tableId, + rowId: params.rowId, + }), + + get_schema: (params) => ({ + tableId: params.tableId, + }), + + query_rows: (params) => ({ + tableId: params.tableId, + filter: resolveFilter(params), + order: resolveOrder(params), + // Omitted limit returns the entire result (server fails fast over 5MB); + // with a limit, nextCursor signals when more rows remain. + limit: parseOptionalLimit(params.limit), + cursor: resolveCursor(params.cursor), + }), +} + +export const TableV2Block: BlockConfig = { + type: 'table_v2', + name: 'Table', + description: 'User-defined data tables', + longDescription: + 'Create and manage custom data tables. Store, query, and manipulate structured data within workflows. ' + + 'Query Rows filters with a predicate tree — `{"all":[{"field":"wins","op":"gte","value":10}]}` ' + + '(`all` = AND, `any` = OR; groups nest). Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, ' + + 'nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. Order is a sort ' + + 'spec `[{"field":"wins","direction":"desc"}]`. Query Rows returns every matching row when Limit is omitted ' + + '(fails if the result exceeds 5MB — add a filter or a Limit). With a Limit, responses page: a non-null ' + + 'nextCursor means more rows exist — pass it back as the cursor.', + bestPractices: ` +- To fetch specific rows, use Query Rows with a predicate filter (e.g. {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}) — do NOT read every row and filter downstream with a Condition block. +- Use "Get Row by ID" only when you have the row's id; otherwise filter with a predicate. +- A group is {"all":[...]} (AND) or {"any":[...]} (OR); nest groups as members for mixed logic. +- Example: players who won ≥10 and are active → {"all":[{"field":"wins","op":"gte","value":10},{"field":"status","op":"eq","value":"active"}]}. +- like/ilike use * as the wildcard (e.g. {"field":"name","op":"ilike","value":"*jo*"}). +- Omit Limit to get the entire matching result in one response — the query fails with a clear error if it exceeds 5MB (narrow with a filter or set a Limit). +- With a Limit, pages can end at the Limit or the 5MB byte budget, whichever comes first — pass nextCursor back as the cursor and loop until it is null; never infer completion from page size. +- Columns are scalar (string/number/boolean/date) or opaque json — there are no array columns; for substring use ilike with *x*.`, + docsLink: 'https://docs.sim.ai/integrations/table', + category: 'blocks', + // Unreleased: hidden from every discovery surface until revealed via the hosted + // `block-visibility` AppConfig document or the `PREVIEW_BLOCKS` env allowlist. + // Placed instances always execute. At GA: drop this, add the BlockMeta + docs, + // and mark v1 `table` superseded. + preview: true, + bgColor: '#10B981', + icon: TableIcon, + subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown', + options: [ + { label: 'Query Rows', id: 'query_rows' }, + { label: 'Insert Row', id: 'insert_row' }, + { label: 'Upsert Row', id: 'upsert_row' }, + { label: 'Batch Insert Rows', id: 'batch_insert_rows' }, + { label: 'Update Rows by Filter', id: 'update_rows_by_filter' }, + { label: 'Delete Rows by Filter', id: 'delete_rows_by_filter' }, + { label: 'Update Row by ID', id: 'update_row' }, + { label: 'Delete Row by ID', id: 'delete_row' }, + { label: 'Get Row by ID', id: 'get_row' }, + { label: 'Get Schema', id: 'get_schema' }, + ], + value: () => 'query_rows', + }, + + { + id: 'tableSelector', + title: 'Table', + type: 'table-selector', + canonicalParamId: 'tableId', + mode: 'basic', + placeholder: 'Select a table', + required: true, + }, + { + id: 'manualTableId', + title: 'Table ID', + type: 'short-input', + canonicalParamId: 'tableId', + mode: 'advanced', + placeholder: 'Enter table ID', + required: true, + }, + + { + id: 'rowId', + title: 'Row ID', + type: 'short-input', + placeholder: 'row_xxxxx', + dependsOn: ['tableId'], + condition: { field: 'operation', value: ['get_row', 'update_row', 'delete_row'] }, + required: true, + }, + + { + id: 'data', + title: 'Row Data (JSON)', + type: 'code', + placeholder: '{"column_name": "value"}', + condition: { + field: 'operation', + value: ['insert_row', 'upsert_row', 'update_row', 'update_rows_by_filter'], + }, + required: true, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate row data as a JSON object matching the table's column schema. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON object with field values based on the table's columns. No explanations or markdown. + +IMPORTANT: Reference the table schema visible in the table selector to know which columns exist and their types. + +### EXAMPLES + +Table with columns: email (string), name (string), age (number) +"user with email john@example.com and age 25" +→ {"email": "john@example.com", "name": "John", "age": 25} + +Return ONLY the data JSON:`, + generationType: 'table-schema', + }, + }, + + { + id: 'conflictColumnSelector', + title: 'Conflict Column', + type: 'column-selector', + canonicalParamId: 'conflictColumn', + mode: 'basic', + selectorKey: 'table.columns', + placeholder: 'Select a unique column', + dependsOn: ['tableSelector'], + condition: { field: 'operation', value: 'upsert_row' }, + }, + { + id: 'manualConflictColumn', + title: 'Conflict Column', + type: 'short-input', + canonicalParamId: 'conflictColumn', + mode: 'advanced', + placeholder: 'Enter the column id', + dependsOn: ['tableId'], + condition: { field: 'operation', value: 'upsert_row' }, + }, + + { + id: 'rows', + title: 'Rows Data (Array of JSON)', + type: 'code', + placeholder: '[{"col1": "val1"}, {"col1": "val2"}]', + condition: { field: 'operation', value: 'batch_insert_rows' }, + required: true, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate an array of row data objects matching the table's column schema. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY a valid JSON array of objects. Each object represents one row. No explanations or markdown. +Maximum ${TABLE_LIMITS.MAX_BATCH_INSERT_SIZE} rows per batch. + +Return ONLY the rows array:`, + generationType: 'table-schema', + }, + }, + + // Filter — canonical Builder⟷Editor toggle (the ⟷ swap icon on the field), + // query + bulk update/delete. Basic = visual builder (human-only); Advanced + // = predicate JSON the agent authors directly. Both members are deliberately + // NOT `required` (the canonical group must share required status): the service + // fails closed when no filter resolves for a bulk op ("Filter is required..."). + { + id: 'filterBuilder', + title: 'Filter', + type: 'filter-builder', + canonicalParamId: 'filterInput', + mode: 'basic', + paramVisibility: 'user-only', + condition: { + field: 'operation', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + }, + }, + { + id: 'filter', + title: 'Filter', + type: 'code', + canonicalParamId: 'filterInput', + mode: 'advanced', + placeholder: '{"all":[{"field":"wins","op":"gte","value":10}]}', + condition: { + field: 'operation', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + }, + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `Generate a predicate filter object (JSON) for selecting rows. + +### CONTEXT +{context} + +### INSTRUCTION +Return ONLY the JSON object. No explanations, surrounding quotes, or markdown. + +A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {"field","op","value"} or nested groups. + +### OPERATORS +eq, ne, gt, gte, lt, lte, in, nin (in/nin take an array value), like, ilike (use * as the wildcard), nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. + +### EXAMPLES +"status is active" → {"all":[{"field":"status","op":"eq","value":"active"}]} +"wins at least 10 and active" → {"all":[{"field":"wins","op":"gte","value":10},{"field":"active","op":"eq","value":true}]} +"status active or pending" → {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]} +"name contains jo (any case)" → {"all":[{"field":"name","op":"ilike","value":"*jo*"}]} + +Return ONLY the JSON object:`, + generationType: 'table-schema', + }, + }, + + // Order — canonical Builder⟷Editor toggle, query only. Basic = visual sort + // builder (human-only); Advanced = sort-spec JSON the agent authors. + { + id: 'sortBuilder', + title: 'Order', + type: 'sort-builder', + canonicalParamId: 'sortInput', + mode: 'basic', + paramVisibility: 'user-only', + condition: { field: 'operation', value: 'query_rows' }, + }, + { + id: 'order', + title: 'Order', + type: 'short-input', + canonicalParamId: 'sortInput', + mode: 'advanced', + placeholder: '[{"field":"wins","direction":"desc"}]', + condition: { field: 'operation', value: 'query_rows' }, + }, + { + id: 'limit', + title: 'Limit', + type: 'short-input', + placeholder: 'Leave empty for all rows (fails over 5MB)', + condition: { + field: 'operation', + value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'], + }, + }, + { + id: 'cursor', + title: 'Cursor', + type: 'short-input', + placeholder: 'Paste a nextCursor to fetch the next page', + condition: { field: 'operation', value: 'query_rows' }, + }, + ...getTrigger('table_new_row').subBlocks, + ], + + tools: { + access: [ + 'table_insert_row', + 'table_batch_insert_rows', + 'table_upsert_row', + 'table_update_row', + 'table_update_rows_by_filter', + 'table_delete_row', + 'table_delete_rows_by_filter', + 'table_query_rows_v2', + 'table_get_row', + 'table_get_schema', + ], + config: { + tool: (params) => { + const toolMap: Record = { + insert_row: 'table_insert_row', + batch_insert_rows: 'table_batch_insert_rows', + upsert_row: 'table_upsert_row', + update_row: 'table_update_row', + update_rows_by_filter: 'table_update_rows_by_filter', + delete_row: 'table_delete_row', + delete_rows_by_filter: 'table_delete_rows_by_filter', + query_rows: 'table_query_rows_v2', + get_row: 'table_get_row', + get_schema: 'table_get_schema', + } + return toolMap[params.operation] || 'table_query_rows_v2' + }, + params: (params) => { + const { operation, ...rest } = params + const transformer = paramTransformers[operation] + if (transformer) { + return transformer(rest as TableBlockParams) + } + return rest + }, + }, + }, + + inputs: { + operation: { type: 'string', description: 'Table operation to perform' }, + tableId: { type: 'string', description: 'Table identifier' }, + data: { type: 'json', description: 'Row data for insert/update' }, + rows: { type: 'array', description: 'Array of row data for batch insert' }, + rowId: { type: 'string', description: 'Row identifier for ID-based operations' }, + filterInput: { + type: 'json', + description: + 'Filter — a predicate object {"all":[{"field":"wins","op":"gte","value":10}]} (or visual builder conditions). Used by query and bulk update/delete.', + }, + sortInput: { + type: 'json', + description: + 'Order — a sort spec [{"field":"wins","direction":"desc"}] (or visual sort conditions). Query only.', + }, + limit: { + type: 'number', + description: + 'Page row limit (optional — omit to return the entire result, which fails if it exceeds 5MB); optional cap for bulk update/delete', + }, + cursor: { type: 'string', description: 'Opaque pagination cursor from a prior query response' }, + conflictColumn: { + type: 'string', + description: + 'Unique column to match on for upsert (required if the table has multiple unique columns)', + }, + }, + + outputs: { + success: { type: 'boolean', description: 'Operation success status' }, + row: { + type: 'json', + description: 'Single row data', + condition: { + field: 'operation', + value: ['get_row', 'insert_row', 'upsert_row', 'update_row'], + }, + }, + operation: { + type: 'string', + description: 'Operation performed (insert or update)', + condition: { field: 'operation', value: 'upsert_row' }, + }, + rows: { + type: 'array', + description: 'Array of rows', + condition: { field: 'operation', value: ['query_rows', 'batch_insert_rows'] }, + }, + rowCount: { + type: 'number', + description: 'Rows returned (query) or total rows in the table (get schema)', + condition: { field: 'operation', value: ['query_rows', 'get_schema'] }, + }, + totalCount: { + type: 'number', + description: 'Total rows matching the predicate (first page only)', + condition: { field: 'operation', value: 'query_rows' }, + }, + nextCursor: { + type: 'string', + description: 'Cursor to fetch the next page, or null on the last page', + condition: { field: 'operation', value: 'query_rows' }, + }, + insertedCount: { + type: 'number', + description: 'Number of rows inserted', + condition: { field: 'operation', value: 'batch_insert_rows' }, + }, + updatedCount: { + type: 'number', + description: 'Number of rows updated', + condition: { field: 'operation', value: 'update_rows_by_filter' }, + }, + updatedRowIds: { + type: 'array', + description: 'IDs of updated rows', + condition: { field: 'operation', value: 'update_rows_by_filter' }, + }, + deletedCount: { + type: 'number', + description: 'Number of rows deleted', + condition: { field: 'operation', value: ['delete_row', 'delete_rows_by_filter'] }, + }, + deletedRowIds: { + type: 'array', + description: 'IDs of deleted rows', + condition: { field: 'operation', value: 'delete_rows_by_filter' }, + }, + name: { + type: 'string', + description: 'Table name', + condition: { field: 'operation', value: 'get_schema' }, + }, + columns: { + type: 'array', + description: 'Column definitions (each includes its stable id)', + condition: { field: 'operation', value: 'get_schema' }, + }, + columnCount: { + type: 'number', + description: 'Number of columns', + condition: { field: 'operation', value: 'get_schema' }, + }, + maxRows: { + type: 'number', + description: "Max rows per table for the workspace's plan", + condition: { field: 'operation', value: 'get_schema' }, + }, + message: { type: 'string', description: 'Operation status message' }, + }, + triggers: { + enabled: true, + available: ['table_new_row'], + }, +} diff --git a/apps/sim/blocks/registry-maps.minimal.ts b/apps/sim/blocks/registry-maps.minimal.ts index 29d6a0c3875..39ab88a9ed1 100644 --- a/apps/sim/blocks/registry-maps.minimal.ts +++ b/apps/sim/blocks/registry-maps.minimal.ts @@ -27,6 +27,7 @@ import { SimWorkspaceEventBlock } from '@/blocks/blocks/sim_workspace_event' import { SlackBlock, SlackV2Block } from '@/blocks/blocks/slack' import { StartTriggerBlock } from '@/blocks/blocks/start_trigger' import { TableBlock } from '@/blocks/blocks/table' +import { TableV2Block } from '@/blocks/blocks/table_v2' import { TranslateBlock } from '@/blocks/blocks/translate' import { VariablesBlock } from '@/blocks/blocks/variables' import { WaitBlock } from '@/blocks/blocks/wait' @@ -81,6 +82,7 @@ export const BLOCK_REGISTRY: Record = { slack_v2: SlackV2Block, start_trigger: StartTriggerBlock, table: TableBlock, + table_v2: TableV2Block, translate: TranslateBlock, variables: VariablesBlock, wait: WaitBlock, diff --git a/apps/sim/blocks/registry-maps.ts b/apps/sim/blocks/registry-maps.ts index 70f443bec70..aa4dfac717f 100644 --- a/apps/sim/blocks/registry-maps.ts +++ b/apps/sim/blocks/registry-maps.ts @@ -294,6 +294,7 @@ import { STSBlock, STSBlockMeta } from '@/blocks/blocks/sts' import { SttBlock, SttV2Block } from '@/blocks/blocks/stt' import { SupabaseBlock, SupabaseBlockMeta } from '@/blocks/blocks/supabase' import { TableBlock } from '@/blocks/blocks/table' +import { TableV2Block } from '@/blocks/blocks/table_v2' import { TailscaleBlock, TailscaleBlockMeta } from '@/blocks/blocks/tailscale' import { TavilyBlock, TavilyBlockMeta } from '@/blocks/blocks/tavily' import { TelegramBlock, TelegramBlockMeta } from '@/blocks/blocks/telegram' @@ -613,6 +614,7 @@ export const BLOCK_REGISTRY: Record = { stt_v2: SttV2Block, supabase: SupabaseBlock, table: TableBlock, + table_v2: TableV2Block, tailscale: TailscaleBlock, tavily: TavilyBlock, telegram: TelegramBlock, diff --git a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts index 565d92bc9d8..8dfc98f70ba 100644 --- a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts +++ b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts @@ -27,6 +27,7 @@ import { } from '@/lib/billing/storage' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import type { DbOrTx } from '@/lib/db/types' +import { nKeysBetween } from '@/lib/table/order-key' import type { TableSchema } from '@/lib/table/types' import { deleteFile, @@ -840,6 +841,21 @@ export async function copyForkResourceContent(params: { try { let copied = 0 let afterId: string | null = null + // `order_key` is nullable, and spreading `...row` would inherit NULLs into a + // brand-new tableId that the one-shot backfill script-migration never revisits + // (it snapshots the pending set up front) — leaving rows the keyset pager has to + // special-case forever. Mint keys for the unkeyed ones instead. They sort last in + // the source (NULLS LAST, id tiebreak) and this loop pages by id, so consuming a + // pre-generated run appended after the source's max key preserves visual order. + const [{ maxKey = null, unkeyed = 0 } = {}] = await db + .select({ + maxKey: sql`max(${userTableRows.orderKey})`, + unkeyed: sql`count(*) filter (where ${userTableRows.orderKey} is null)`, + }) + .from(userTableRows) + .where(eq(userTableRows.tableId, table.sourceId)) + const mintedKeys = unkeyed > 0 ? nKeysBetween(maxKey, null, Number(unkeyed)) : [] + let mintedIdx = 0 for (;;) { const where: SQL | undefined = afterId === null @@ -858,6 +874,7 @@ export async function copyForkResourceContent(params: { id: generateId(), tableId: table.childId, workspaceId: childWorkspaceId, + orderKey: row.orderKey ?? mintedKeys[mintedIdx++] ?? null, // Repoint resource-chip URLs in cell data at the child copies (no-op when no maps). data: contentRefMaps ? remapTableRowResourceUrls(row.data, contentRefMaps) : row.data, })) diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index 0f447344bd0..a1f13eba635 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -88,13 +88,13 @@ import { buildUpgradeHref } from '@/lib/billing/upgrade-reasons' import type { CsvHeaderMapping, EnrichmentRunDetail, - Filter, RowData, RowExecutionMetadata, RowExecutions, - Sort, + SortSpec, TableDefinition, TableMetadata, + TablePredicate, TableRow, WorkflowGroup, WorkflowGroupDependencies, @@ -129,8 +129,8 @@ export const TABLE_EXPORT_JOBS_STALE_TIME = 5 * 1000 type TableRowsParams = Omit & TableIdParamsInput & { - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } export type TableRowsResponse = Pick< @@ -416,8 +416,8 @@ interface InfiniteTableRowsParams { workspaceId: string tableId: string pageSize: number - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null enabled?: boolean } @@ -433,8 +433,8 @@ interface FindTableRowsParams { workspaceId: string tableId: string q: string - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } export interface TableFindResult { @@ -1225,7 +1225,7 @@ interface DeleteTableRowsAsyncVariables { filter?: DeleteTableRowsAsyncBody['filter'] /** Active sort — together with `filter` it identifies the exact rows query to optimistically * strip, so we don't clear unrelated cached views (other filters/sorts). */ - sort?: Sort | null + sort?: SortSpec | null /** Rows deselected after "select all" — spared by the job. */ excludeRowIds?: string[] /** Doomed-row estimate shown in the confirm — persisted on the job so server counts can @@ -1259,7 +1259,13 @@ export function useDeleteTableRowsAsync({ workspaceId, tableId }: RowMutationCon // Target the exact infinite-rows query for the view the user is on — not every cached view. const activeKey = tableKeys.infiniteRows( tableId, - tableRowsParamsKey({ pageSize: TABLE_LIMITS.MAX_QUERY_LIMIT, filter: filter ?? null, sort }) + tableRowsParamsKey({ + pageSize: TABLE_LIMITS.MAX_QUERY_LIMIT, + // The wire type is the dual-grammar union; the grid only ever sends its + // own predicate state, so the cache key narrows to that shape. + filter: (filter as TablePredicate | undefined) ?? null, + sort, + }) ) await queryClient.cancelQueries({ queryKey: activeKey }) const previousRows = @@ -1557,10 +1563,10 @@ interface CancelRunsParams { scope: 'all' | 'row' rowId?: string /** Scope-`all` only: cancel just the cells on rows matching this filter (filtered select-all Stop). */ - filter?: Filter + filter?: TablePredicate /** Active sort — with `filter` it identifies the exact rows query whose cells the optimistic * cancel may flip (other cached views contain rows the server won't touch). */ - sort?: Sort | null + sort?: SortSpec | null /** Scope-`all` only: deselected rows whose cells keep running. */ excludeRowIds?: string[] } @@ -2192,7 +2198,7 @@ interface RunColumnVariables { /** "Select all under a filter" — run every row matching this filter (mutually exclusive with * `rowIds`). Optimistic stamping is skipped (like `limit`) since the matching set isn't known * client-side; the dispatcher's real pending stamps drive the UI. */ - filter?: Filter + filter?: TablePredicate /** Select-all scope only: deselected rows — skipped by the dispatcher and the optimistic stamp. */ excludeRowIds?: string[] /** Cap the run to the first `max` eligible rows. Omit for an unbounded run. diff --git a/apps/sim/lib/api/client/request.test.ts b/apps/sim/lib/api/client/request.test.ts index e28ed0837b2..f7a03a53574 100644 --- a/apps/sim/lib/api/client/request.test.ts +++ b/apps/sim/lib/api/client/request.test.ts @@ -54,18 +54,36 @@ describe('requestJson query serialization', () => { expect(calledUrl).not.toContain('[object Object]') }) - it('throws instead of silently corrupting an array-of-objects query param', async () => { - mockFetchReturning({ ok: true }) + it('JSON-encodes an array-of-objects query param instead of corrupting or throwing', async () => { + // A SortSpec ([{field, direction}]) is the everyday case: repeat-append + // would send "[object Object]", so the whole array travels as ONE JSON + // string param, mirroring plain objects; the server contract decodes it. + const fetchMock = mockFetchReturning({ ok: true }) - const badContract = defineRouteContract({ + const contract = defineRouteContract({ method: 'GET', path: '/api/test', query: z.object({ items: z.array(z.object({ a: z.string() })) }), response: { mode: 'json', schema: z.object({ ok: z.boolean() }) }, }) - await expect(requestJson(badContract, { query: { items: [{ a: 'x' }] } })).rejects.toThrow( - /arrays of objects are not URL-safe/ - ) + await requestJson(contract, { query: { items: [{ a: 'x' }] } }) + const url = String(fetchMock.mock.calls[0][0]) + expect(decodeURIComponent(url)).toContain('items=[{"a":"x"}]') + }) + + it('keeps repeat-append for scalar arrays', async () => { + const fetchMock = mockFetchReturning({ ok: true }) + + const contract = defineRouteContract({ + method: 'GET', + path: '/api/test', + query: z.object({ tags: z.array(z.string()) }), + response: { mode: 'json', schema: z.object({ ok: z.boolean() }) }, + }) + + await requestJson(contract, { query: { tags: ['a', 'b'] } }) + const url = String(fetchMock.mock.calls[0][0]) + expect(url).toContain('tags=a&tags=b') }) }) diff --git a/apps/sim/lib/api/client/request.ts b/apps/sim/lib/api/client/request.ts index a68eb96fd13..cf4b5cef479 100644 --- a/apps/sim/lib/api/client/request.ts +++ b/apps/sim/lib/api/client/request.ts @@ -67,19 +67,17 @@ function appendQuery(path: string, query: unknown): string { if (value === undefined || value === null || value === '') continue if (Array.isArray(value)) { + // An array of objects (e.g. a SortSpec) is not repeat-append-able — each + // item would stringify to "[object Object]" and silently corrupt the + // request (the knowledge tagFilters bug). Encode the WHOLE array as one + // JSON string param, mirroring how plain objects are sent below; the + // server-side contract decodes it. Scalar arrays keep repeat-append. + if (value.some((item) => item !== null && typeof item === 'object')) { + searchParams.set(key, JSON.stringify(value)) + continue + } for (const item of value) { if (item === undefined || item === null || item === '') continue - // A non-scalar in a query array would stringify to "[object Object]" and - // silently corrupt the request. Encode such values as a single JSON - // string param and decode them server-side instead. Failing loudly here - // keeps the boundary honest (this is how the knowledge tagFilters bug - // shipped undetected). - if (typeof item === 'object') { - throw new Error( - `Cannot serialize query param "${key}": arrays of objects are not URL-safe — ` + - 'encode the value as a JSON string param and decode it server-side.' - ) - } searchParams.append(key, String(item)) } continue diff --git a/apps/sim/lib/api/contracts/tables-predicate.test.ts b/apps/sim/lib/api/contracts/tables-predicate.test.ts new file mode 100644 index 00000000000..8f45d758822 --- /dev/null +++ b/apps/sim/lib/api/contracts/tables-predicate.test.ts @@ -0,0 +1,261 @@ +/** + * @vitest-environment node + * + * The v2 query/bulk filter wire format is the typed `{ all | any: [...] }` + * predicate tree. The contract validates structure; column-level validation + * (unknown field, json-op) runs server-side in `validate.ts`. + */ +import { describe, expect, it } from 'vitest' +import { + deleteTableRowsBodySchema, + predicateSchema, + rowQueryBodySchema, + tableRowsQuerySchema, + updateRowsByFilterBodySchema, +} from '@/lib/api/contracts/tables' +import { validatePredicate } from '@/lib/table/query-builder/validate' + +describe('rowQueryBodySchema', () => { + it('accepts a predicate/sort object, leaves limit unbounded, has no offset', () => { + const parsed = rowQueryBodySchema.parse({ + workspaceId: 'ws-1', + predicate: { + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'status', op: 'in', value: ['active', 'pending'] }, + ], + }, + sort: [{ field: 'wins', direction: 'desc' }], + cursor: 'abc', + }) + expect(parsed.predicate).toEqual({ + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'status', op: 'in', value: ['active', 'pending'] }, + ], + }) + // Omitted limit stays undefined — the query returns all matching rows. + expect(parsed.limit).toBeUndefined() + expect('offset' in parsed).toBe(false) + }) + + it('accepts a nested any/all predicate', () => { + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { all: [{ field: 'wins', op: 'gte', value: 5 }] }, + ], + }, + }).success + ).toBe(true) + }) + + it('allows omitting the predicate (match all)', () => { + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1' }).success).toBe(true) + }) + + it('rejects an unknown operator and a malformed leaf', () => { + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { all: [{ field: 'wins', op: 'bogus', value: 1 }] }, + }).success + ).toBe(false) + expect( + rowQueryBodySchema.safeParse({ + workspaceId: 'ws-1', + predicate: { all: [{ op: 'eq', value: 1 }] }, + }).success + ).toBe(false) + }) + + it('accepts a large explicit limit (no row cap) but rejects limit < 1', () => { + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1', limit: 100000 }).success).toBe(true) + expect(rowQueryBodySchema.safeParse({ workspaceId: 'ws-1', limit: 0 }).success).toBe(false) + }) +}) + +describe('bulk schemas accept either a predicate tree or the legacy filter object', () => { + it('delete accepts a predicate filter', () => { + expect( + deleteTableRowsBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: { all: [{ field: 'status', op: 'eq', value: 'archived' }] }, + }).success + ).toBe(true) + }) + + it('delete still accepts the legacy object filter (v1 callers)', () => { + expect( + deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: { status: 'archived' } }) + .success + ).toBe(true) + }) + + it('update accepts a predicate filter', () => { + expect( + updateRowsByFilterBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: { all: [{ field: 'wins', op: 'gte', value: 10 }] }, + data: { active: false }, + }).success + ).toBe(true) + }) +}) + +/** + * The predicate tree is parsed by a recursive `z.lazy` union. A few thousand + * nested groups overflow the stack inside `safeParse`, and a `RangeError` + * escaping a parser is a 500 on a public endpoint, not a 400. + */ +describe('predicate depth / size guard', () => { + it('rejects a deeply nested tree with a validation issue, not a RangeError', () => { + let node: unknown = { all: [{ field: 'a', op: 'eq', value: 1 }] } + for (let i = 0; i < 5000; i++) node = { all: [node] } + + const result = predicateSchema.safeParse(node) + + expect(result.success).toBe(false) + expect(JSON.stringify(result.error?.issues)).toMatch(/nesting is too deep/) + }) + + it('rejects a wide-but-shallow tree past the node cap', () => { + const node = { + all: Array.from({ length: 60 }, () => ({ + all: Array.from({ length: 60 }, () => ({ field: 'a', op: 'eq', value: 1 })), + })), + } + + const result = predicateSchema.safeParse(node) + + expect(result.success).toBe(false) + expect(JSON.stringify(result.error?.issues)).toMatch(/too many conditions/) + }) + + it('still accepts a realistic nested predicate', () => { + expect( + predicateSchema.safeParse({ + all: [ + { field: 'status', op: 'eq', value: 'active' }, + { + any: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'name', op: 'contains', value: 'jo' }, + ], + }, + ], + }).success + ).toBe(true) + }) +}) + +/** + * Zod strips unrecognized keys by default, so before the schemas were made + * strict a hybrid node parsed clean against the group branch with its leaf half + * silently deleted — turning "delete archived rows for tenant acme" into + * "delete EVERY row for tenant acme". `validatePredicate`'s hybrid guard could + * not catch it: the keys were gone before it ran. + */ +describe('hybrid group+leaf nodes are rejected, not silently narrowed', () => { + const hybrid = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + } + + it('rejects rather than dropping the leaf half', () => { + const result = predicateSchema.safeParse(hybrid) + expect(result.success).toBe(false) + // The dangerous outcome: parsing succeeds having quietly widened the filter. + expect(result.success ? result.data : null).not.toEqual({ all: hybrid.all }) + }) + + /** + * The bulk schemas union the predicate tree with the legacy `$`-object, and + * that legacy branch accepts any non-empty object — so it absorbs the hybrid + * and the SCHEMA cannot reject it. Crucially the legacy branch does NOT strip, + * so `all` survives, the route's `isTablePredicate` check routes it back to + * `validatePredicate`, and the hybrid guard there rejects it (→ 400 via + * `route.ts:325`). Asserted here so a future change to either layer that + * removes one of them fails loudly. + */ + it('keeps the hybrid intact through the bulk schemas so the runtime guard can see it', () => { + for (const parsed of [ + deleteTableRowsBodySchema.safeParse({ workspaceId: 'ws-1', filter: hybrid }), + updateRowsByFilterBodySchema.safeParse({ + workspaceId: 'ws-1', + filter: hybrid, + data: { active: false }, + }), + ]) { + expect(parsed.success).toBe(true) + // The leaf half must NOT have been silently dropped on the way through. + expect(parsed.success && parsed.data.filter).toMatchObject({ + all: hybrid.all, + field: 'status', + }) + } + }) + + it('and validatePredicate then rejects it', () => { + expect(() => + validatePredicate(hybrid as never, [ + { name: 'tenant_id', type: 'string' }, + { name: 'status', type: 'string' }, + ]) + ).toThrow(/not both/) + }) + + it('rejects an unknown key on a leaf (a typo must not be dropped)', () => { + expect( + predicateSchema.safeParse({ all: [{ field: 'a', op: 'eq', vlaue: 'typo' }] }).success + ).toBe(false) + }) + + it('still accepts well-formed nodes', () => { + expect( + predicateSchema.safeParse({ + all: [{ field: 'a', op: 'eq', value: 1 }, { any: [{ field: 'b', op: 'isNull' }] }], + }).success + ).toBe(true) + }) +}) + +/** + * Wire transport: requestJson serializes structured query params as JSON + * strings; jsonQueryValue decodes them before the union runs. Without it, a + * sorted or filtered grid request 400s at the boundary. + */ +describe('query-string JSON transport (jsonQueryValue)', () => { + it('decodes string-encoded predicate, legacy filter, and sort spec', () => { + const parsed = rowQueryStringSchemaProbe({ + workspaceId: 'ws-1', + filter: JSON.stringify({ all: [{ field: 'a', op: 'eq', value: 1 }] }), + sort: JSON.stringify([{ field: 'a', direction: 'asc' }]), + }) + expect(parsed.filter).toEqual({ all: [{ field: 'a', op: 'eq', value: 1 }] }) + expect(parsed.sort).toEqual([{ field: 'a', direction: 'asc' }]) + + const legacy = rowQueryStringSchemaProbe({ + workspaceId: 'ws-1', + filter: JSON.stringify({ status: { $eq: 'x' } }), + sort: JSON.stringify({ status: 'desc' }), + }) + expect(legacy.filter).toEqual({ status: { $eq: 'x' } }) + expect(legacy.sort).toEqual({ status: 'desc' }) + }) + + it('still rejects a non-JSON garbage string with the real schema error', () => { + expect(() => rowQueryStringSchemaProbe({ workspaceId: 'ws-1', filter: 'not json' })).toThrow() + }) +}) + +function rowQueryStringSchemaProbe(input: Record) { + const result = tableRowsQuerySchema.safeParse(input) + if (!result.success) throw new Error(JSON.stringify(result.error.issues[0])) + return result.data +} diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 2da6ded270c..5bccaa52069 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -11,16 +11,27 @@ import type { CsvHeaderMapping, EnrichmentRunDetail, Filter, + Predicate, + PredicateNode, RowData, Sort, + SortSpec, TableDefinition, TableLocks, TableMetadata, + TablePredicate, TableRow, TableRowsCursor, TableViewConfig, } from '@/lib/table' -import { COLUMN_TYPES, MAX_SELECT_OPTIONS, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' +import { + COLUMN_TYPES, + FILTER_OPS, + MAX_SELECT_OPTIONS, + NAME_PATTERN, + SORT_DIRECTIONS, + TABLE_LIMITS, +} from '@/lib/table/constants' import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import' export const domainObjectSchema = () => z.custom(isRecordLike) @@ -370,6 +381,137 @@ const nonEmptyFilterSchema = domainObjectSchema().refine( const filterSchema = domainObjectSchema() +/* --------------------------- v2 predicate grammar --------------------------- */ + +/** + * Body cap for the row-query routes. A query body is a predicate tree plus a + * cursor; the largest legitimate one — a 100-member predicate with 1000-element + * `in` lists — is comfortably under 1 MB. The 50 MB platform default would let a + * caller buffer two orders of magnitude more before any schema check runs. + */ +export const TABLE_QUERY_MAX_BODY_BYTES = 1024 * 1024 + +/** Max members in one `all`/`any` group — a generous bound against pathological trees. */ +const MAX_PREDICATE_GROUP_SIZE = 100 +/** Max sort keys — more than a few is already a smell. */ +const MAX_SORT_KEYS = 16 +/** Max nesting levels of `all`/`any` groups. Ten is already unreadable. */ +const MAX_PREDICATE_DEPTH = 10 +/** Max nodes in the whole tree, so a wide-but-shallow tree can't amplify either. */ +const MAX_PREDICATE_NODES = 500 + +/** + * Iterative depth/size walk over an unvalidated predicate tree. Runs BEFORE the + * recursive Zod schema: a few thousand nested `{all:[...]}` levels overflow the + * stack inside `safeParse`, and a `RangeError` from a parser is a 500, not a 400. + * The walk itself must stay iterative for the same reason. + */ +function predicateTreeTooLarge(root: unknown): string | null { + const stack: Array<{ node: unknown; depth: number }> = [{ node: root, depth: 1 }] + let nodes = 0 + + while (stack.length > 0) { + const { node, depth } = stack.pop()! + if (++nodes > MAX_PREDICATE_NODES) { + return `Filter has too many conditions (max ${MAX_PREDICATE_NODES})` + } + if (depth > MAX_PREDICATE_DEPTH) { + return `Filter nesting is too deep (max ${MAX_PREDICATE_DEPTH} levels)` + } + if (typeof node !== 'object' || node === null) continue + const group = node as { all?: unknown; any?: unknown } + const members = Array.isArray(group.all) + ? group.all + : Array.isArray(group.any) + ? group.any + : null + if (!members) continue + for (const member of members) stack.push({ node: member, depth: depth + 1 }) + } + return null +} + +/** + * v2 filter wire format: the typed `{ all | any: [...] }` predicate tree (same + * shape the engine consumes). Structure is validated here; schema-awareness + * (unknown column, json-op rejection) is enforced server-side by + * `validatePredicate` once the table's columns are known. + */ +/** + * Both node shapes are `strictObject`, and that is load-bearing rather than + * fussiness. Zod strips unrecognized keys by default, so a hybrid node carrying + * BOTH a group key and a leaf's `field`/`op`/`value` parsed clean against the + * group branch with the leaf half silently deleted. On the bulk paths that turns + * "delete archived rows for tenant acme" into "delete every row for tenant acme". + * `validatePredicate`'s hybrid guard could never catch it — the keys were gone + * before it ran. Strict on BOTH branches is required: strict on the group alone + * would just fall through to the leaf branch, which is the more dangerous reading. + */ +// double-cast-allowed: `z.unknown()` keeps the runtime permissive (a leaf value +// is arbitrary JSON), but infers `unknown`, which is wider than +// `Predicate['value']`. The narrowing is type-level only — nothing is coerced. +const predicateLeafSchema = z.strictObject({ + field: z.string().min(1, 'field is required').max(128), + op: z.enum(FILTER_OPS), + value: z.unknown().optional(), +}) as unknown as z.ZodType + +const predicateNodeSchema: z.ZodType = z.lazy(() => + z.union([predicateGroupSchema, predicateLeafSchema]) +) + +const predicateTreeSchema: z.ZodType = z.lazy(() => + z.union([ + // `.min(1)`: an empty group compiles to no WHERE clause, which on the bulk + // delete/update paths reads as "match everything" rather than "match nothing". + z.strictObject({ + all: z + .array(predicateNodeSchema) + .min(1, 'A filter group must contain at least one condition') + .max(MAX_PREDICATE_GROUP_SIZE), + }), + z.strictObject({ + any: z + .array(predicateNodeSchema) + .min(1, 'A filter group must contain at least one condition') + .max(MAX_PREDICATE_GROUP_SIZE), + }), + ]) +) +const predicateGroupSchema = predicateTreeSchema + +/** + * The boundary predicate schema: depth/size guard first, then the recursive + * structural parse. The guard is only applied at the top level — every nested + * group is strictly shallower, so re-checking inside the recursion would be + * redundant work on the hot path. + */ +export const predicateSchema = z + .unknown() + .superRefine((value, ctx) => { + const problem = predicateTreeTooLarge(value) + if (problem) ctx.addIssue({ code: 'custom', message: problem }) + }) + // double-cast-allowed: the pipe's inferred input is `unknown`, and letting TS + // widen the recursive lazy union through it makes typecheck OOM + .pipe(predicateTreeSchema) as unknown as z.ZodType + +/** v2 sort wire format: an ordered list of `{ field, direction }`. */ +export const sortSpecSchema: z.ZodType = z + .array( + z.object({ + field: z.string().min(1, 'field is required').max(128), + direction: z.enum(SORT_DIRECTIONS), + }) + ) + .max(MAX_SORT_KEYS) + +/** + * Bulk update/delete filter accepts either the v2 predicate tree (v2 block / + * agent) or the legacy `$`-operator object (v1 callers / stored workflows). + */ +const bulkFilterSchema = z.union([predicateSchema, nonEmptyFilterSchema]) + const optionalPositiveLimit = (max: number, label: string) => z.preprocess( (value) => (value === null || value === undefined || value === '' ? undefined : Number(value)), @@ -388,7 +530,7 @@ export const deleteTableRowBodySchema = z.object({ export const deleteTableRowsBodySchema = z .object({ workspaceId: workspaceIdSchema, - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), limit: optionalPositiveLimit(TABLE_LIMITS.MAX_BULK_OPERATION_SIZE, 'Limit').optional(), rowIds: z .array(z.string().min(1)) @@ -403,17 +545,37 @@ export const deleteTableRowsBodySchema = z message: 'Provide either filter or rowIds, but not both', }) +/** + * Query-param transport for a structured value: `requestJson` serializes + * objects/arrays into a single JSON-string param, and this decodes it before + * the real schema runs. Non-JSON strings pass through untouched so the inner + * schema produces the real error; already-parsed values (POST bodies reusing a + * schema) are untouched too. + */ +const jsonQueryValue = (schema: S) => + z.preprocess((value) => { + if (typeof value !== 'string' || value === '') return value + try { + return JSON.parse(value) + } catch { + return value + } + }, schema) + /** Unrefined base so v1 contracts can `.extend()` — consumers use {@link tableRowsQuerySchema}. */ export const tableRowsQueryBaseSchema = z.object({ workspaceId: workspaceIdSchema, - filter: domainObjectSchema().optional(), - sort: domainObjectSchema().optional(), + // Dual-grammar during the v2 transition: the strict predicate tree wins the + // union; anything else falls through to the legacy `$`-object. Same for sort: + // an ordered spec array vs the legacy `{col: dir}` record. + filter: jsonQueryValue(z.union([predicateSchema, domainObjectSchema()])).optional(), + sort: jsonQueryValue(z.union([sortSpecSchema, domainObjectSchema()])).optional(), /** * Keyset cursor `(orderKey, id)` for the default row order — each page is an index seek * instead of OFFSET's scan-and-discard. Mutually exclusive with `sort` (cursors only make * sense on the default order); takes precedence over `offset`. */ - after: domainObjectSchema().optional(), + after: jsonQueryValue(domainObjectSchema()).optional(), limit: z .preprocess( (value) => @@ -453,7 +615,7 @@ export const tableRowsQuerySchema = tableRowsQueryBaseSchema.refine( export const updateRowsByFilterBodySchema = z.object({ workspaceId: workspaceIdSchema, - filter: nonEmptyFilterSchema, + filter: bulkFilterSchema, data: rowDataSchema, limit: optionalPositiveLimit(TABLE_LIMITS.MAX_BULK_OPERATION_SIZE, 'Limit').optional(), }) @@ -663,16 +825,65 @@ export const listTableRowsContract = defineRouteContract({ totalCount: z.number().nullable(), limit: z.number(), offset: z.number(), + /** Non-null when more rows exist — a page may be cut by the byte budget + * before reaching `limit`, so page fullness is not a termination signal. */ + nextCursor: z.string().nullable(), + }) + ), + }, +}) + +/** + * v2 query surface: typed `predicate`/`sort` objects + opaque cursor pagination. + * No `offset` (the cursor encodes paging state) and no after/sort refine (the + * cursor carries the sort context). Structure is validated here; column-level + * validation (unknown field, json-op) runs server-side via `validatePredicate`. + */ +export const rowQueryBodySchema = z.object({ + workspaceId: z.string().min(1, 'Workspace ID is required'), + predicate: predicateSchema.optional(), + sort: sortSpecSchema.optional(), + // Omitted limit returns the ENTIRE matching result, failing fast (400) when + // it exceeds the response byte budget. An explicit limit caps the page row + // count; the byte budget may still end a page early with nextCursor set. + limit: z.preprocess( + (value) => (value === null || value === undefined || value === '' ? undefined : Number(value)), + z + .number({ error: 'Limit must be a number' }) + .int('Limit must be an integer') + .min(1, 'Limit must be at least 1') + .optional() + ), + cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), +}) + +export type RowQueryBody = z.input + +export const rowQueryContract = defineRouteContract({ + method: 'POST', + path: '/api/table/[tableId]/query', + params: tableIdParamsSchema, + body: rowQueryBodySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + rows: z.array(tableRowSchema), + rowCount: z.number(), + totalCount: z.number().nullable(), + limit: z.number(), + nextCursor: z.string().nullable(), }) ), }, }) +export type RowQueryResponse = ContractJsonResponse export const findTableRowsQuerySchema = z.object({ workspaceId: workspaceIdSchema, q: requiredFieldSchema('Search query is required'), - filter: domainObjectSchema().optional(), - sort: domainObjectSchema().optional(), + filter: jsonQueryValue(z.union([predicateSchema, domainObjectSchema()])).optional(), + sort: jsonQueryValue(z.union([sortSpecSchema, domainObjectSchema()])).optional(), }) /** One matching cell: its 0-based ordinal in the filtered+sorted view, its row id, and the column name. */ @@ -1101,7 +1312,7 @@ export const deleteTableRowsContract = defineRouteContract({ */ export const deleteTableRowsAsyncBodySchema = z.object({ workspaceId: workspaceIdSchema, - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), excludeRowIds: z .array(z.string().min(1)) .max( @@ -1296,7 +1507,7 @@ export const cancelTableRunsBodySchema = z workspaceId: workspaceIdSchema, scope: z.enum(['all', 'row']), rowId: z.string().min(1).optional(), - filter: domainObjectSchema().optional(), + filter: z.union([predicateSchema, domainObjectSchema()]).optional(), /** Scope-`all` only: rows deselected from the selection — their cells keep running. */ excludeRowIds: z .array(z.string().min(1)) @@ -1400,7 +1611,7 @@ export const runColumnBodySchema = z rowIds: z.array(z.string().min(1)).min(1).optional(), /** "Select all under a filter" — run every row matching this filter instead of `rowIds`. The * dispatcher walks only matching rows (paginated), so no id list is materialized. */ - filter: nonEmptyFilterSchema.optional(), + filter: bulkFilterSchema.optional(), /** Select-all scope only: rows deselected from the selection — the dispatcher skips them. */ excludeRowIds: z .array(z.string().min(1)) @@ -1526,8 +1737,11 @@ export const tableEventStreamContract = defineRouteContract({ * never invalidates a view. */ export const tableViewConfigSchema = tableMetadataSchema.extend({ - filter: filterSchema.nullable().optional(), - sort: domainObjectSchema().nullable().optional(), + // The v2 predicate/sort grammar — same wire as the query routes, so a saved + // view gets the same strictness and depth bounds as a live filter, and its + // config can later feed the v2 surfaces without conversion. + filter: predicateSchema.nullable().optional(), + sort: sortSpecSchema.nullable().optional(), }) satisfies z.ZodType export const tableViewSchema = z.object({ diff --git a/apps/sim/lib/api/contracts/v2/tables/index.ts b/apps/sim/lib/api/contracts/v2/tables/index.ts new file mode 100644 index 00000000000..34e3902f01d --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/tables/index.ts @@ -0,0 +1,115 @@ +import { z } from 'zod' +import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { + predicateSchema, + sortSpecSchema, + tableColumnSchema, + tableIdParamsSchema, +} from '@/lib/api/contracts/tables' +import { defineRouteContract } from '@/lib/api/contracts/types' + +/** + * Public v2 tables API — typed predicate grammar, cursor paging. + * + * Filters are the `{ all | any: [...] }` predicate tree (same shape the engine + * consumes); no string querystring dialect. Response bodies are fully typed. Row + * data is name-keyed and carries no storage internals (`position`/`orderKey`/ + * `executions`) — the public wire is `{ id, data, createdAt, updatedAt }`. + */ + +/** Default page size when `limit` is omitted (bounded, unlike the internal surface). */ +export const V2_DEFAULT_ROW_LIMIT = 100 +/** Hard cap on an explicit page `limit`. Larger pulls use `limit=0` or the async export. */ +export const V2_MAX_ROW_LIMIT = 1000 + +const successResponseSchema = (dataSchema: T) => + z.object({ + success: z.literal(true), + data: dataSchema, + }) + +const v2TableSummarySchema = z.object({ + id: z.string(), + name: z.string(), + description: z.string().nullable(), + schema: z.object({ columns: z.array(tableColumnSchema) }), + rowCount: z.number(), + maxRows: z.number(), + createdAt: z.string(), + updatedAt: z.string(), +}) + +/** Public row shape: name-keyed data, no storage internals. */ +export const v2TableRowSchema = z.object({ + id: z.string(), + data: z.record(z.string(), z.unknown()), + createdAt: z.string(), + updatedAt: z.string(), +}) + +export const v2ListTablesQuerySchema = z.object({ + workspaceId: workspaceIdSchema, +}) + +/** + * Rows query body. `predicate`/`sort` are the typed predicate tree / sort spec. + * `limit`: omitted → {@link V2_DEFAULT_ROW_LIMIT}; `0` → unbounded (whole result + * or a 400 `TABLE_QUERY_RESULT_TOO_LARGE`); `1..{@link V2_MAX_ROW_LIMIT}` → page cap. + */ +export const v2QueryRowsBodySchema = z.object({ + workspaceId: workspaceIdSchema, + predicate: predicateSchema.optional(), + sort: sortSpecSchema.optional(), + limit: z + .number({ error: 'Limit must be a number' }) + .int('Limit must be an integer') + .min(0, 'Limit must be at least 0 (use 0 for an unbounded query)') + .max( + V2_MAX_ROW_LIMIT, + `Limit cannot exceed ${V2_MAX_ROW_LIMIT}; use limit=0 for a full result or the async export for large datasets` + ) + .optional(), + cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), +}) + +export type V2ListTablesQuery = z.output +export type V2QueryRowsBody = z.input +export type V2TableRow = z.output + +export const v2ListTablesContract = defineRouteContract({ + method: 'GET', + path: '/api/v2/tables', + query: v2ListTablesQuerySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + tables: z.array(v2TableSummarySchema), + totalCount: z.number(), + }) + ), + }, +}) + +export const v2QueryRowsContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/tables/[tableId]/query', + params: tableIdParamsSchema, + body: v2QueryRowsBodySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + rows: z.array(v2TableRowSchema), + rowCount: z.number(), + totalCount: z.number().nullable(), + limit: z.number(), + /** Opaque, short-lived token. Non-null when more rows remain; stop on null. */ + nextCursor: z.string().nullable(), + }) + ), + }, +}) + +export type V2ListTablesResponse = z.output<(typeof v2ListTablesContract)['response']['schema']> +export type V2QueryRowsResponse = z.output<(typeof v2QueryRowsContract)['response']['schema']> diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index bcdadb752fb..cda8d3097ea 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -4609,6 +4609,11 @@ export const UserTable: ToolCatalogEntry = { description: 'Array of column names to delete at once (for delete_column). Preferred over columnName when deleting multiple columns.', }, + cursor: { + type: 'string', + description: + 'Opaque pagination cursor for query_rows (optional). Omit for the first page; to fetch the next page, pass back the nextCursor from the previous result\'s "more available" message verbatim. Cannot be combined with a fresh order — the cursor already encodes the paging position.', + }, data: { type: 'object', description: 'Row data as key-value pairs (required for insert_row, update_row)', @@ -4646,7 +4651,7 @@ export const UserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'MongoDB-style filter for query_rows, update_rows_by_filter, delete_rows_by_filter', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -4681,7 +4686,7 @@ export const UserTable: ToolCatalogEntry = { limit: { type: 'number', description: - 'Maximum rows to return or affect (optional, default 100). Omit on update_rows_by_filter / delete_rows_by_filter to act on every match.', + 'Maximum rows per page for query_rows (optional). Omit to fetch the ENTIRE matching result in one response — the call fails if the result exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, a page may end early at the byte budget with more remaining; a non-null nextCursor in the result means more rows exist (continue with cursor). On update_rows_by_filter / delete_rows_by_filter, caps affected rows; omit to act on every match.', }, mapping: { type: 'object', @@ -4737,9 +4742,10 @@ export const UserTable: ToolCatalogEntry = { description: 'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.', }, - offset: { - type: 'number', - description: 'Number of rows to skip (optional for query_rows, default 0)', + order: { + type: 'array', + description: + 'Sort spec for query_rows (optional). Ordered list of {field, direction} where direction is asc or desc, e.g. [{"field":"wins","direction":"desc"},{"field":"name","direction":"asc"}].', }, options: { type: 'array', @@ -4831,11 +4837,6 @@ export const UserTable: ToolCatalogEntry = { "Cancellation scope for cancel_table_runs. 'all' cancels in-flight runs across the whole table; 'row' cancels only the row identified by rowId.", enum: ['all', 'row'], }, - sort: { - type: 'object', - description: - "Sort specification as { field: 'asc' | 'desc' } (optional for query_rows)", - }, tableId: { type: 'string', description: diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 6029f71c0fe..ecb3c630ff0 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -3,6 +3,7 @@ import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/ import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { getColumnId } from '@/lib/table/column-keys' +import { TABLE_LIMITS } from '@/lib/table/constants' import { formatCsvCell, neutralizeCsvFormula, toCsvRow } from '@/lib/table/export-format' import { queryRows } from '@/lib/table/rows/service' import { getTableById, listTables } from '@/lib/table/service' @@ -427,7 +428,13 @@ export async function resolveInputFiles( continue } - const rows = await queryRows(table, {}, 'copilot-fn-exec') + // Keep the prior bounded mount — draining the whole table here was backed + // out for OOM, so don't ride the new unbounded queryRows default. + const rows = await queryRows( + table, + { limit: TABLE_LIMITS.DEFAULT_QUERY_LIMIT }, + 'copilot-fn-exec' + ) const columns = table.schema.columns const csvLines = [toCsvRow(columns.map((column) => neutralizeCsvFormula(column.name)))] diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts index b0dd887cb49..adebd05bd7a 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.test.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.test.ts @@ -141,6 +141,7 @@ import { normalizeSelectOptionsInput, userTableServerTool, } from '@/lib/copilot/tools/server/table/user-table' +import { encodeCursor } from '@/lib/table/rows/cursor' function buildTable(overrides: Partial = {}): TableDefinition { return { @@ -783,7 +784,7 @@ describe('userTableServerTool.query_rows', () => { }) }) - it('clamps an over-large query limit to MAX_QUERY_LIMIT instead of rejecting', async () => { + it('passes an explicit limit through unchanged (no row cap)', async () => { const result = await userTableServerTool.execute( { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 100000 } }, { userId: 'user-1', workspaceId: 'workspace-1' } @@ -791,20 +792,75 @@ describe('userTableServerTool.query_rows', () => { expect(result.success).toBe(true) const options = mockQueryRows.mock.calls[0][1] as Record - expect(options.limit).toBe(1000) + expect(options.limit).toBe(100000) }) - it('queries without execution metadata and passes limit/offset through', async () => { + it('omits the limit so queryRows returns every matching row', async () => { const result = await userTableServerTool.execute( - { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2, offset: 10 } }, + { operation: 'query_rows', args: { tableId: 'tbl_1' } }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(true) + const options = mockQueryRows.mock.calls[0][1] as Record + expect(options.limit).toBeUndefined() + }) + + it('decodes an opaque cursor into after/offset and skips the count', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'row_9', orderKey: 'a9' }, + keysetValid: true, + nextOffset: 20, + }) + const result = await userTableServerTool.execute( + { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2, cursor } }, { userId: 'user-1', workspaceId: 'workspace-1' } ) expect(result.success).toBe(true) const options = mockQueryRows.mock.calls[0][1] as Record expect(options.withExecutions).toBe(false) - expect(options.offset).toBe(10) - expect(result.data?.nextCursor).toBeUndefined() + expect(options.after).toEqual({ orderKey: 'a9', id: 'row_9' }) + // A cursor page never re-counts. + expect(options.includeTotal).toBe(false) + }) + + it('surfaces the opaque nextCursor (not an offset) in the "more available" message', async () => { + mockQueryRows.mockResolvedValueOnce({ + rows: [queryRow(1), queryRow(2)], + rowCount: 2, + totalCount: 10, + limit: 2, + offset: 0, + nextCursor: 'CURSOR_TOKEN_ABC', + }) + const result = await userTableServerTool.execute( + { operation: 'query_rows', args: { tableId: 'tbl_1', limit: 2 } }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.message).toContain('more available') + expect(result.message).toContain('cursor=CURSOR_TOKEN_ABC') + expect(result.message).not.toContain('offset=') + }) + + it('rejects a keyset cursor combined with a custom sort', async () => { + const cursor = encodeCursor({ + lastRow: { id: 'row_9', orderKey: 'a9' }, + keysetValid: true, + nextOffset: 20, + }) + const result = await userTableServerTool.execute( + { + operation: 'query_rows', + args: { tableId: 'tbl_1', cursor, order: [{ field: 'name', direction: 'desc' }] }, + }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.success).toBe(false) + expect(result.message).toMatch(/not valid for a sorted query/i) + expect(mockQueryRows).not.toHaveBeenCalled() }) }) @@ -835,7 +891,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 5000 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 5000, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -856,7 +916,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { it('deletes inline when the unbounded match count is within the cap', async () => { const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -874,7 +937,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -894,7 +961,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { }) const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) await flushDetached() @@ -929,7 +999,10 @@ describe('userTableServerTool.delete_rows_by_filter', () => { mockMarkTableJobRunning.mockResolvedValueOnce(false) const result = await userTableServerTool.execute( - { operation: 'delete_rows_by_filter', args: { tableId: 'tbl_1', filter: { name: 'x' } } }, + { + operation: 'delete_rows_by_filter', + args: { tableId: 'tbl_1', filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] } }, + }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -943,7 +1016,11 @@ describe('userTableServerTool.delete_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'delete_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -974,7 +1051,12 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 }, limit: 5000 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + limit: 5000, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -994,7 +1076,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1015,7 +1101,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1051,7 +1141,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { email: 'x' }, data: { email: 'y' } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'email', op: 'eq', value: 'x' }] }, + data: { email: 'y' }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1073,7 +1167,11 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 } }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) @@ -1087,7 +1185,12 @@ describe('userTableServerTool.update_rows_by_filter', () => { const result = await userTableServerTool.execute( { operation: 'update_rows_by_filter', - args: { tableId: 'tbl_1', filter: { name: 'x' }, data: { age: 1 }, limit: 100 }, + args: { + tableId: 'tbl_1', + filter: { all: [{ field: 'name', op: 'eq', value: 'x' }] }, + data: { age: 1 }, + limit: 100, + }, }, { userId: 'user-1', workspaceId: 'workspace-1' } ) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 9d3f35438ff..9f66adcb9da 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -1,6 +1,6 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId, generateShortId } from '@sim/utils/id' import { UserTable } from '@/lib/copilot/generated/tool-catalog-v1' import { @@ -30,9 +30,8 @@ import { namedRowMapper } from '@/lib/table/cell-format' import { buildIdByName, columnMatchesRef, - filterNamesToIds, rowDataNameToId, - sortNamesToIds, + sortSpecNamesToIds, } from '@/lib/table/column-keys' import { columnTypeForLeaf, deriveOutputColumnName } from '@/lib/table/column-naming' import { @@ -48,6 +47,9 @@ import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' import { assertRowDelete, assertRowUpdate, patchColumnIds } from '@/lib/table/mutation-locks' +import { predicateToFilter } from '@/lib/table/query-builder/converters' +import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' +import { assertCursorSortBinding, decodeCursor } from '@/lib/table/rows/cursor' import { batchInsertRows, batchUpdateRows, @@ -61,15 +63,17 @@ import { updateRow, updateRowsByFilter, } from '@/lib/table/rows/service' -import { resolveFilterSelectValues } from '@/lib/table/select-values' +import { predicateToStorage } from '@/lib/table/select-values' import { createTable, deleteTable, getTableById, renameTable } from '@/lib/table/service' import type { ColumnDefinition, Filter, RowData, SelectOption, + SortSpec, TableDefinition, TableDeleteJobPayload, + TablePredicate, TableSchema, TableUpdateJobPayload, WorkflowGroup, @@ -675,33 +679,68 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) + // Typed predicate/sort objects, validated against the schema (column + // NAMES) then translated to storage ids. + let predicate: TablePredicate | undefined + if (args.filter) { + validatePredicate(args.filter, table.schema.columns) + predicate = predicateToStorage(args.filter, table.schema) + } + let orderSpec = args.order as SortSpec | undefined + if (orderSpec?.length) { + validateSortSpec(orderSpec, table.schema.columns) + orderSpec = sortSpecNamesToIds(orderSpec, idByName) + } + const sort = orderSpec?.length + ? Object.fromEntries(orderSpec.map((s) => [s.field, s.direction])) + : undefined + + // Opaque cursor pagination (keyset seek on the default order; the token + // hides an internal offset only for custom-sorted views, which a keyset + // physically can't page). A keyset cursor is bound to the default order, + // so it can't be combined with a fresh sort. + const cursor = args.cursor ? decodeCursor(args.cursor) : undefined + if (cursor) { + try { + // Keyset cursors bind to the default order; offset cursors to the + // exact sort they were minted under. + assertCursorSortBinding(cursor, sort) + } catch (bindError) { + return { success: false, message: getErrorMessage(bindError, 'Invalid cursor') } + } + } + + // No limit returns the ENTIRE matching result, failing fast once the + // 5MB byte budget is exceeded (caught below → structured tool error + // the model can react to by adding a filter or a limit). An explicit + // limit pages; byte-cut pages set nextCursor and the message says to + // continue with the opaque cursor. const toNamedRow = namedRowMapper(table.schema.columns) - // The model may request any number; we serve at most MAX_QUERY_LIMIT per page so a single - // tool result can't drain a whole table. `totalCount` in the response signals truncation, - // and the model pages with `offset`. const result = await queryRows( table, { - filter: args.filter - ? resolveFilterSelectValues( - filterNamesToIds(args.filter, idByName), - table.schema.columns - ) - : undefined, - sort: args.sort ? sortNamesToIds(args.sort, idByName) : undefined, - limit: - args.limit !== undefined - ? Math.min(args.limit, TABLE_LIMITS.MAX_QUERY_LIMIT) - : undefined, - offset: args.offset, + predicate, + sort, + limit: args.limit, + after: cursor?.after, + offset: cursor?.offset, + // Only the first page (no inbound cursor) pays for the COUNT(*). + includeTotal: !args.cursor, withExecutions: false, }, requestId ) + // nextCursor covers both cut kinds (explicit limit or the 5MB byte + // budget) — either way the truthful signal is "more rows exist". The + // token is opaque; the agent echoes it back as `cursor` to continue. + const countSuffix = result.totalCount != null ? ` of ${result.totalCount}` : '' + const message = result.nextCursor + ? `Returned ${result.rows.length}${countSuffix} rows (more available — pass cursor=${result.nextCursor} to continue)` + : `Returned ${result.rows.length}${countSuffix} rows` return { success: true, - message: `Returned ${result.rows.length} of ${result.totalCount} rows`, + message, data: { ...result, rows: result.rows.map((r) => ({ @@ -822,10 +861,11 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - const idFilter = resolveFilterSelectValues( - filterNamesToIds(args.filter, idByName), - table.schema.columns - ) + // Agent authors a predicate object; validate → translate → Filter for + // the bulk engine (same fieldPredicate leaf → identical SQL). Select + // operands arrive as option NAMES and must resolve to stored ids. + validatePredicate(args.filter, table.schema.columns) + const idFilter = predicateToFilter(predicateToStorage(args.filter, table.schema)) const idData = rowDataNameToId(args.data, idByName) // Inline handles up to MAX_BULK_OPERATION_SIZE rows in one request; a larger operation @@ -923,10 +963,11 @@ export const userTableServerTool: BaseServerTool const requestId = generateId().slice(0, 8) const idByName = buildIdByName(table.schema) - const idFilter = resolveFilterSelectValues( - filterNamesToIds(args.filter, idByName), - table.schema.columns - ) + // Agent authors a predicate object; validate → translate → Filter for + // the bulk engine (same fieldPredicate leaf → identical SQL). Select + // operands arrive as option NAMES and must resolve to stored ids. + validatePredicate(args.filter, table.schema.columns) + const idFilter = predicateToFilter(predicateToStorage(args.filter, table.schema)) // Inline handles up to MAX_BULK_OPERATION_SIZE rows; a larger delete (an explicit limit // above the cap, or unbounded "delete everything matching") hands off to the background diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index ab6a5de43f1..8c28ba643f8 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -106,7 +106,7 @@ export const env = createEnv({ ENTERPRISE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on enterprise tier (default: 10000) ENTERPRISE_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on enterprise tier (default: 1000000) TABLE_MAX_ROW_SIZE_BYTES: z.number().optional(), // Max serialized size in bytes of a single user-table row (default: 409600) - TABLE_MAX_PAGE_BYTES: z.number().optional(), // Dev-preview: byte budget per row-page read; pages cut early past it (unset = disabled) + TABLE_MAX_PAGE_BYTES: z.number().optional(), // Byte budget per row-page read; pages cut early past it (unset = disabled) TABLE_DISPATCH_CONCURRENCY_FREE: z.number().optional(), // Rows one table run executes in parallel on free tier (default: 20) TABLE_DISPATCH_CONCURRENCY_PAID: z.number().optional(), // Rows one table run executes in parallel on paid tiers (default: 50) @@ -481,6 +481,7 @@ export const env = createEnv({ SESSION_POLICIES_ENABLED: z.boolean().optional(), // Enable org session policies on self-hosted (bypasses hosted requirements) FORKING_ENABLED: z.boolean().optional(), // Enable workspace forking on self-hosted (bypasses hosted requirements) DEPLOY_AS_BLOCK: z.boolean().optional(), // Enable deploy-as-block (publish a workflow as a reusable org-wide custom block) + TABLES_V2_API: z.boolean().optional(), // Enable the v2 tables HTTP API (public /api/v2/tables + internal /api/table/[tableId]/query predicate-grammar route) TABLE_LOCKS: z.boolean().optional(), // Enable per-table mutation locks (schema/insert/update/delete toggles) TABLE_VIEWS: z.boolean().optional(), // Enable saved table views (named filter/sort/column-visibility presets) and the column show/hide menu diff --git a/apps/sim/lib/core/config/feature-flags.test.ts b/apps/sim/lib/core/config/feature-flags.test.ts index ebf9e70099b..7f0d0dbea50 100644 --- a/apps/sim/lib/core/config/feature-flags.test.ts +++ b/apps/sim/lib/core/config/feature-flags.test.ts @@ -13,6 +13,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({ APPCONFIG_ENVIRONMENT: 'staging' as string | undefined, FORKING_ENABLED: undefined as boolean | undefined, DEPLOY_AS_BLOCK: undefined as boolean | undefined, + TABLES_V2_API: undefined as boolean | undefined, }, })) @@ -245,3 +246,29 @@ describe('isFeatureEnabled', () => { }) }) }) + +describe('tables-v2-api flag', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isAppConfigEnabled: false }) + envRef.TABLES_V2_API = undefined + }) + + it('is off by default off-AppConfig, on when the fallback secret is set', async () => { + expect(await isFeatureEnabled('tables-v2-api')).toBe(false) + envRef.TABLES_V2_API = true + expect(await isFeatureEnabled('tables-v2-api')).toBe(true) + }) + + it('gates by org cohort via AppConfig', async () => { + withAppConfig({ 'tables-v2-api': { orgIds: ['org-1'] } }) + expect(await isFeatureEnabled('tables-v2-api', { orgId: 'org-1' })).toBe(true) + expect(await isFeatureEnabled('tables-v2-api', { orgId: 'org-2' })).toBe(false) + expect(await isFeatureEnabled('tables-v2-api', { userId: 'u1' })).toBe(false) + }) + + it('global enabled turns it on for everyone', async () => { + withAppConfig({ 'tables-v2-api': { enabled: true } }) + expect(await isFeatureEnabled('tables-v2-api')).toBe(true) + }) +}) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 167b1e50b6e..2989f04bcf3 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -104,6 +104,14 @@ const FEATURE_FLAGS = { 'custom-block publish/list routes. Off-AppConfig falls back to DEPLOY_AS_BLOCK.', fallback: 'DEPLOY_AS_BLOCK', }, + 'tables-v2-api': { + description: + 'Gate the v2 tables HTTP API — the public read API (GET /api/v2/tables, POST ' + + '/api/v2/tables/[tableId]/query) and the internal predicate-grammar query route (POST ' + + '/api/table/[tableId]/query). When off, those routes return 404 as if the surface does not ' + + 'exist. Gated by userId/orgId/admins via AppConfig; off-AppConfig falls back to TABLES_V2_API.', + fallback: 'TABLES_V2_API', + }, 'table-locks': { description: 'Per-table mutation locks (schema/insert/update/delete) an admin toggles to make a table ' + diff --git a/apps/sim/lib/table/__tests__/paging.test.ts b/apps/sim/lib/table/__tests__/paging.test.ts deleted file mode 100644 index 22a4a55bcf5..00000000000 --- a/apps/sim/lib/table/__tests__/paging.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { trimRowsToByteBudget } from '@/lib/table/rows/paging' - -function row(id: string, bytes: number) { - // {"b":"aaa…"} serializes to bytes + 8 chars of envelope. - return { id, data: { b: 'a'.repeat(Math.max(0, bytes - 8)) } } -} - -describe('trimRowsToByteBudget', () => { - it('returns all rows when they fit the budget', () => { - const rows = [row('r1', 100), row('r2', 100)] - expect(trimRowsToByteBudget(rows, 1000)).toBe(rows) - }) - - it('keeps the longest prefix within the budget', () => { - const rows = [row('r1', 400), row('r2', 400), row('r3', 400)] - const kept = trimRowsToByteBudget(rows, 900) - expect(kept.map((r) => r.id)).toEqual(['r1', 'r2']) - }) - - it('always keeps the first row even when it alone exceeds the budget', () => { - const rows = [row('r1', 5000), row('r2', 100)] - const kept = trimRowsToByteBudget(rows, 1000) - expect(kept.map((r) => r.id)).toEqual(['r1']) - }) - - it('returns empty input unchanged', () => { - expect(trimRowsToByteBudget([], 1000)).toEqual([]) - }) -}) diff --git a/apps/sim/lib/table/__tests__/select-values.test.ts b/apps/sim/lib/table/__tests__/select-values.test.ts new file mode 100644 index 00000000000..32c0891e30c --- /dev/null +++ b/apps/sim/lib/table/__tests__/select-values.test.ts @@ -0,0 +1,128 @@ +/** + * @vitest-environment node + * + * Select-column operand resolution. A select cell stores option IDs, so a filter + * written with the option NAME must be rewritten before it reaches SQL — + * otherwise it compares a name against an id and matches nothing while reporting + * success. Both wire grammars have to do this identically. + */ +import { describe, expect, it } from 'vitest' +import { resolveFilterSelectValues, resolvePredicateSelectValues } from '@/lib/table/select-values' +import type { ColumnDefinition } from '@/lib/table/types' + +const MULTI: ColumnDefinition = { + id: 'col_color', + name: 'Color', + type: 'select', + multiple: true, + options: [ + { id: 'opt_teal', name: 'Teal' }, + { id: 'opt_green', name: 'Green' }, + ], +} +const SINGLE: ColumnDefinition = { + id: 'col_status', + name: 'Status', + type: 'select', + options: [{ id: 'opt_open', name: 'Open' }], +} +const PLAIN: ColumnDefinition = { id: 'col_name', name: 'name', type: 'string' } +const COLS = [MULTI, SINGLE, PLAIN] + +const leaf = (p: unknown) => (p as { all: Array<{ value: unknown }> }).all[0] + +describe('resolvePredicateSelectValues', () => { + /** + * Regression: `contains`/`ncontains` were excluded as "pattern ops". On a + * multi-select they are not pattern ops — the cell is an array of ids and they + * express membership. Mothership sent exactly this and silently got zero rows. + */ + it('resolves contains / ncontains on a MULTI-select (membership, not pattern)', () => { + for (const op of ['contains', 'ncontains'] as const) { + const out = resolvePredicateSelectValues( + { all: [{ field: 'col_color', op, value: 'Teal' }] }, + COLS + ) + expect(leaf(out).value).toBe('opt_teal') + } + }) + + it('resolves eq / ne / in / nin on a single select', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_status', op: 'eq', value: 'Open' }] }, + COLS + ) + ).value + ).toBe('opt_open') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_status', op: 'in', value: ['Open'] }] }, + COLS + ) + ).value + ).toEqual(['opt_open']) + }) + + it('matches option names case-insensitively and passes ids through', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'opt_teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + }) + + it('leaves non-select columns and unknown option names alone', () => { + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_name', op: 'contains', value: 'Teal' }] }, + COLS + ) + ).value + ).toBe('Teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'Nope' }] }, + COLS + ) + ).value + ).toBe('Nope') + }) + + it('recurses through nested groups', () => { + const out = resolvePredicateSelectValues( + { any: [{ all: [{ field: 'col_color', op: 'contains', value: 'Green' }] }] }, + COLS + ) as { any: Array<{ all: Array<{ value: unknown }> }> } + expect(out.any[0].all[0].value).toBe('opt_green') + }) + + /** The two grammars must resolve the same operand set, or they drift. */ + it('agrees with the $-grammar sibling on the same filter', () => { + const legacy = resolveFilterSelectValues({ col_color: { $contains: 'Teal' } }, COLS) + expect((legacy.col_color as { $contains: unknown }).$contains).toBe('opt_teal') + expect( + leaf( + resolvePredicateSelectValues( + { all: [{ field: 'col_color', op: 'contains', value: 'Teal' }] }, + COLS + ) + ).value + ).toBe('opt_teal') + }) +}) diff --git a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts index 582df6cd2c3..889e75de2d8 100644 --- a/apps/sim/lib/table/__tests__/service-filter-threading.test.ts +++ b/apps/sim/lib/table/__tests__/service-filter-threading.test.ts @@ -7,15 +7,19 @@ * timestamp for dates) are always available at the SQL builder layer — the * latent bug that PR #4657 was originally fixing. */ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, resetDbChainMock, setEnv } from '@sim/testing' import { sql } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { decodeCursor } from '@/lib/table/rows/cursor' import { buildFilterClause, buildSortClause } from '@/lib/table/sql' import type { ColumnDefinition, TableDefinition } from '@/lib/table/types' vi.mock('@/lib/table/sql', () => ({ buildFilterClause: vi.fn(() => sql`true`), buildSortClause: vi.fn(() => sql`true`), + buildPredicateClause: vi.fn(() => sql`true`), + TableQueryValidationError: class TableQueryValidationError extends Error {}, })) vi.mock('@/lib/table/trigger', () => ({ @@ -122,3 +126,316 @@ describe('service filter threading', () => { ) }) }) + +describe('bulk update/delete limited-subset ordering', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('orders the match query when updateRowsByFilter has a limit', async () => { + await updateRowsByFilter( + TABLE, + { filter: { score: { $gt: 0 } }, data: { name: 'x' }, limit: 5 }, + 'req-1' + ) + expect(dbChainMockFns.orderBy).toHaveBeenCalled() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(5) + }) + + it('does not order an unbounded updateRowsByFilter', async () => { + dbChainMockFns.where.mockResolvedValueOnce([]) + await updateRowsByFilter(TABLE, { filter: { score: { $gt: 0 } }, data: { name: 'x' } }, 'req-1') + expect(dbChainMockFns.orderBy).not.toHaveBeenCalled() + }) + + it('orders the match query when deleteRowsByFilter has a limit', async () => { + await deleteRowsByFilter(TABLE, { filter: { score: { $gt: 0 } }, limit: 3 }, 'req-1') + expect(dbChainMockFns.orderBy).toHaveBeenCalled() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(3) + }) +}) + +describe('queryRows byte budget', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + // The bounded-page byte cut is opt-in; pin it on rather than inheriting + // whatever the developer's local `.env` happens to set. + setEnv({ TABLE_MAX_PAGE_BYTES: TABLE_LIMITS.MAX_QUERY_RESULT_BYTES }) + }) + + const row = (i: number, blobBytes: number) => ({ + id: `row_${i}`, + data: { blob: 'x'.repeat(blobBytes) }, + position: i, + orderKey: `a${i}`, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + }) + + it('returns an empty page with a null cursor', async () => { + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + expect(result.rows).toEqual([]) + expect(result.nextCursor).toBeNull() + }) + + it('fails fast when an UNBOUNDED query exceeds the byte budget (no partial page)', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + // First .limit() call is pendingDeleteMask's job probe; the second is the drain batch. + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + await expect( + queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + ).rejects.toThrow(/exceeds the 5MB limit/) + }) + + it('returns the entire result when an unbounded query fits the budget', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8)]) + + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + + expect(result.rows).toHaveLength(2) + expect(result.nextCursor).toBeNull() + }) + + it('byte-cuts a BOUNDED page and returns a resume cursor instead of throwing', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0].id).toBe('row_1') + // Fractional ordering is on (global flag mock) → keyset cursor resuming + // after the last included row. + expect(decodeCursor(result.nextCursor as string)).toEqual({ + after: { orderKey: 'a1', id: 'row_1' }, + }) + }) + + it('does NOT byte-cut a bounded page when TABLE_MAX_PAGE_BYTES is unset', async () => { + // Default-off: a short page is only safe for a client that terminates on + // `nextCursor === null`. A pre-existing v1 pager stopping at + // `rows.length < limit` would read the cut as end-of-data and truncate. + setEnv({ TABLE_MAX_PAGE_BYTES: undefined }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(2) + expect(result.nextCursor).toBeNull() + }) + + it('still fails fast on an UNBOUNDED query with TABLE_MAX_PAGE_BYTES unset', async () => { + setEnv({ TABLE_MAX_PAGE_BYTES: undefined }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + await expect( + queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + ).rejects.toThrow(/exceeds the 5MB limit/) + }) + + it('returns a single over-budget row alone on a bounded page', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([ + row(1, TABLE_LIMITS.MAX_QUERY_RESULT_BYTES + 1024), + row(2, 8), + ]) + + const result = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0].id).toBe('row_1') + expect(result.nextCursor).not.toBeNull() + }) + + // First-batch worst-case bytes stay within ~4× the budget at the max row size. + const FIRST_BATCH_CAP = Math.max( + 1, + Math.floor((4 * TABLE_LIMITS.MAX_QUERY_RESULT_BYTES) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES) + ) + + /** + * The seek predicate that pages the drain. `order_key` is nullable (rows + * predating the backfill, forked rows), and a bare + * `(order_key, id) > (:k, :i)` evaluates to NULL for those rows, so WHERE drops + * them. Because NULLs sort last, the whole unkeyed tail then becomes + * unreachable and the drain reports `hasMore: false` — 52 of 121 rows returned + * with no error, reproduced against a real table. + */ + it('seeks with a NULL-admitting comparison so unkeyed rows stay reachable', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([]) + + await queryRows( + TABLE, + { + after: { orderKey: 'a5', id: 'row_5' }, + limit: 10, + includeTotal: false, + withExecutions: false, + }, + 'req-1' + ) + + const seekWhere = JSON.stringify(dbChainMockFns.where.mock.calls.at(-1)) + expect(seekWhere).toMatch(/is null/i) + expect(seekWhere).toContain('a5') + expect(seekWhere).toContain('row_5') + }) + + it('resumes exactly at the last returned row when the cursor is fed back', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + // limit 2 + witness row_3 → page ends at row_2 with a resume cursor. + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8), row(3, 8)]) + + const page1 = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(page1.rows.map((r) => r.id)).toEqual(['row_1', 'row_2']) + + const cursor = decodeCursor(page1.nextCursor as string) + expect(cursor).toEqual({ after: { orderKey: 'a2', id: 'row_2' } }) + + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(3, 8), row(4, 8)]) + + const page2 = await queryRows( + TABLE, + { ...cursor, limit: 2, includeTotal: false, withExecutions: false }, + 'req-2' + ) + + // No gap (row_3 is the first row past the anchor) and no duplicate of row_2. + expect(page2.rows.map((r) => r.id)).toEqual(['row_3', 'row_4']) + const seekWhere = JSON.stringify(dbChainMockFns.where.mock.calls.at(-1)) + expect(seekWhere).toContain('a2') + expect(seekWhere).toContain('row_2') + }) + + it('caps the first unbounded batch and asks limit+1 when a limit is set', async () => { + await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + // Call 1 = pendingDeleteMask probe (limit 1); call 2 = first drain batch. + expect(dbChainMockFns.limit).toHaveBeenNthCalledWith(2, FIRST_BATCH_CAP + 1) + + vi.clearAllMocks() + resetDbChainMock() + await queryRows(TABLE, { limit: 5, includeTotal: false, withExecutions: false }, 'req-1') + expect(dbChainMockFns.limit).toHaveBeenNthCalledWith(2, 6) + }) + + it('limit-cut with a witness row emits a cursor; exact-boundary page does not', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8), row(3, 8)]) + const cut = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(cut.rows.map((r) => r.id)).toEqual(['row_1', 'row_2']) + expect(cut.nextCursor).not.toBeNull() + + vi.clearAllMocks() + resetDbChainMock() + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([row(1, 8), row(2, 8)]) + const exact = await queryRows( + TABLE, + { limit: 2, includeTotal: false, withExecutions: false }, + 'req-1' + ) + expect(exact.rows).toHaveLength(2) + // The +1 peek proved the data ended exactly at the limit — no dead cursor. + expect(exact.nextCursor).toBeNull() + }) + + it('drains across multiple batches and grows the batch size adaptively', async () => { + const firstAsk = FIRST_BATCH_CAP + 1 + const batch1 = Array.from({ length: firstAsk }, (_, i) => row(i, 8)) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce(batch1) + dbChainMockFns.limit.mockResolvedValueOnce([row(firstAsk, 8)]) + + const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1') + + expect(result.rows).toHaveLength(firstAsk + 1) + expect(result.nextCursor).toBeNull() + const firstBatchAsk = (dbChainMockFns.limit.mock.calls[1] ?? [])[0] as number + const secondBatchAsk = (dbChainMockFns.limit.mock.calls[2] ?? [])[0] as number + expect(secondBatchAsk).toBeGreaterThan(firstBatchAsk) + }) + + it('sort path advances by offset and emits an offset cursor with inbound offset applied', async () => { + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6) + dbChainMockFns.limit.mockResolvedValueOnce([]) + // Sorted batch with inbound offset 40 terminates at .offset(40). + dbChainMockFns.offset.mockResolvedValueOnce([row(1, perRow), row(2, perRow)]) + + const result = await queryRows( + TABLE, + { sort: { name: 'asc' }, offset: 40, limit: 5, includeTotal: false, withExecutions: false }, + 'req-1' + ) + + expect(dbChainMockFns.offset).toHaveBeenCalledWith(40) + expect(result.rows).toHaveLength(1) + // The offset cursor is stamped with the sort it was minted under, so a + // replay against a different ordering is rejected instead of paging wrong. + expect(decodeCursor(result.nextCursor as string)).toEqual({ + offset: 41, + sortKey: JSON.stringify([['name', 'asc']]), + }) + }) + + it('emits a compound cursor when rows past the inbound anchor are unkeyed', async () => { + const unkeyedRow = (i: number, blobBytes: number) => ({ ...row(i, blobBytes), orderKey: null }) + const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.4) + dbChainMockFns.limit.mockResolvedValueOnce([]) + dbChainMockFns.limit.mockResolvedValueOnce([ + unkeyedRow(6, perRow), + unkeyedRow(7, perRow), + unkeyedRow(8, perRow), + ]) + + const result = await queryRows( + TABLE, + { + after: { orderKey: 'a5', id: 'row_5' }, + limit: 10, + includeTotal: false, + withExecutions: false, + }, + 'req-1' + ) + + expect(result.rows.map((r) => r.id)).toEqual(['row_6', 'row_7']) + expect(decodeCursor(result.nextCursor as string)).toEqual({ + after: { orderKey: 'a5', id: 'row_5' }, + offset: 2, + }) + }) +}) diff --git a/apps/sim/lib/table/__tests__/sql.test.ts b/apps/sim/lib/table/__tests__/sql.test.ts index 367a9eb1704..dcca281da94 100644 --- a/apps/sim/lib/table/__tests__/sql.test.ts +++ b/apps/sim/lib/table/__tests__/sql.test.ts @@ -20,11 +20,13 @@ import { } from '@/lib/table/query-builder/constants' import { buildFilterClause, + buildPredicateClause, buildSortClause, + fieldPredicate, MULTI_SELECT_OPERATORS, SINGLE_SELECT_OPERATORS, } from '@/lib/table/sql' -import type { ColumnDefinition, Filter, Sort } from '@/lib/table/types' +import type { ColumnDefinition, Filter, Sort, TablePredicate } from '@/lib/table/types' type SqlNode = | { strings: ArrayLike; values: unknown[] } @@ -85,6 +87,11 @@ function render(node: unknown): string { return renderSql(node) } +/** fieldPredicate takes a ColumnDefinition; these tests only care about its type. */ +function col(type: ColumnDefinition['type'], name = 'c'): ColumnDefinition { + return { name, type } +} + const TABLE = 'user_table_rows' const NO_COLUMNS: ColumnDefinition[] = [] @@ -400,12 +407,12 @@ describe('SQL Builder', () => { expect(out).toBe(`(${TABLE}.data->>'birthDate')::timestamptz ASC NULLS LAST`) }) - it('sorts createdAt / updatedAt as direct column refs', () => { + it('sorts createdAt / updatedAt as direct snake_case column refs', () => { expect(render(buildSortClause({ createdAt: 'desc' }, TABLE, NO_COLUMNS))).toBe( - `${TABLE}.createdAt DESC` + `${TABLE}.created_at DESC` ) expect(render(buildSortClause({ updatedAt: 'asc' }, TABLE, NO_COLUMNS))).toBe( - `${TABLE}.updatedAt ASC` + `${TABLE}.updated_at ASC` ) }) @@ -592,3 +599,363 @@ describe('SQL Builder', () => { }) }) }) + +describe('fieldPredicate (shared leaf)', () => { + const r = ( + op: Parameters[2], + value: unknown, + colType?: ColumnDefinition['type'] + ) => render(fieldPredicate(TABLE, 'wins', op, value as never, colType && col(colType, 'wins'))) + + it('eq emits case-sensitive JSONB containment (no lower())', () => { + const out = render(fieldPredicate(TABLE, 'slack_user_id', 'eq', 'U333', undefined)) + expect(out).toContain('user_table_rows.data @>') + expect(out).toContain('"slack_user_id":"U333"') + expect(out).not.toContain('lower(') + // Case is preserved verbatim — U333 and u333 are distinct values. + expect(out).not.toContain('u333') + }) + + it('ne negates the containment clause', () => { + expect(r('ne', 'x')).toContain('NOT (') + expect(r('ne', 'x')).toContain('data @>') + }) + + it('in with one value is a single containment; many values OR together', () => { + expect(render(fieldPredicate(TABLE, 'slack_user_id', 'in', ['U1'], undefined))).toContain( + '"slack_user_id":"U1"' + ) + const many = render(fieldPredicate(TABLE, 'slack_user_id', 'in', ['U1', 'U2'], undefined)) + expect(many).toContain('"slack_user_id":"U1"') + expect(many).toContain('"slack_user_id":"U2"') + expect(many).toContain(' OR ') + }) + + it('nin ANDs negated containments', () => { + const out = render(fieldPredicate(TABLE, 'slack_user_id', 'nin', ['U1', 'U2'], undefined)) + expect(out).toContain('NOT (') + expect(out).toContain(' AND ') + }) + + it('empty in/nin arrays are a no-op (undefined)', () => { + expect(fieldPredicate(TABLE, 'wins', 'in', [], undefined)).toBeUndefined() + expect(fieldPredicate(TABLE, 'wins', 'nin', [], undefined)).toBeUndefined() + }) + + it('range ops cast by column type', () => { + expect(r('gte', 10, 'number')).toContain('::numeric') + expect(r('gt', '2024-01-01', 'date')).toContain('::timestamptz') + }) + + it('text ops use case-insensitive ILIKE', () => { + expect(render(fieldPredicate(TABLE, 'name', 'contains', 'jo', undefined))).toContain('ILIKE') + expect(render(fieldPredicate(TABLE, 'name', 'startsWith', 'jo', undefined))).toContain('ILIKE') + }) + + it('isEmpty / isNotEmpty emit emptiness checks (null OR empty string)', () => { + const empty = render(fieldPredicate(TABLE, 'name', 'isEmpty', undefined, undefined)) + expect(empty).toContain('IS NULL') + expect(empty).toContain("= ''") + const notEmpty = render(fieldPredicate(TABLE, 'name', 'isNotEmpty', undefined, undefined)) + expect(notEmpty).toContain('IS NOT NULL') + }) + + it('isNull / isNotNull are strict null checks (no empty-string clause)', () => { + const isNull = render(fieldPredicate(TABLE, 'name', 'isNull', undefined, undefined)) + expect(isNull).toContain('IS NULL') + expect(isNull).not.toContain("= ''") + expect(render(fieldPredicate(TABLE, 'name', 'isNotNull', undefined, undefined))).toContain( + 'IS NOT NULL' + ) + }) + + it('like / ilike map * to % and escape literal % / _', () => { + const like = render(fieldPredicate(TABLE, 'name', 'like', 'jo*n', undefined)) + expect(like).toContain("data->>'name'") + expect(like).toContain('LIKE') + expect(like).not.toContain('ILIKE') + expect(like).toContain('jo%n') + expect(render(fieldPredicate(TABLE, 'name', 'ilike', '*foo*', undefined))).toContain('ILIKE') + // literal % is escaped to match itself, not act as a wildcard + expect(render(fieldPredicate(TABLE, 'name', 'like', '50%*', undefined))).toContain('50\\%%') + }) + + it('nlike / nilike negate the match and keep null cells', () => { + // "does not match X" must retain rows where the cell is absent — otherwise a + // negated filter silently drops every row with an empty value for that column. + const nlike = render(fieldPredicate(TABLE, 'name', 'nlike', 'jo*n', undefined)) + expect(nlike).toContain('NOT LIKE') + expect(nlike).not.toContain('NOT ILIKE') + expect(nlike).toContain('jo%n') + expect(nlike).toContain('IS NULL') + + const nilike = render(fieldPredicate(TABLE, 'name', 'nilike', '*foo*', undefined)) + expect(nilike).toContain('NOT ILIKE') + expect(nilike).toContain('IS NULL') + }) + + it('reaches nlike / nilike through the legacy $-grammar too', () => { + expect(render(buildFilterClause({ name: { $nlike: 'jo*' } }, TABLE, NO_COLUMNS))).toContain( + 'NOT LIKE' + ) + expect(render(buildFilterClause({ name: { $nilike: 'jo*' } }, TABLE, NO_COLUMNS))).toContain( + 'NOT ILIKE' + ) + }) + + it('no longer accepts the regex ops (removed from FILTER_OPS)', () => { + for (const op of ['match', 'imatch'] as const) { + expect(() => fieldPredicate(TABLE, 'name', op as never, '^jo', undefined)).toThrow( + 'Invalid operator' + ) + } + expect(() => buildFilterClause({ name: { $match: '^jo' } }, TABLE, NO_COLUMNS)).toThrow( + 'Invalid operator' + ) + }) + + it('validates the field name', () => { + expect(() => fieldPredicate(TABLE, "x'; DROP", 'eq', 1, undefined)).toThrow( + 'Invalid field name' + ) + }) + + it('rejects an unknown operator', () => { + expect(() => fieldPredicate(TABLE, 'wins', 'bogus' as never, 1, undefined)).toThrow( + 'Invalid operator' + ) + }) + + it('filters createdAt / updatedAt as real timestamptz columns, not JSONB keys', () => { + const gte = render(fieldPredicate(TABLE, 'createdAt', 'gte', '2026-01-01', col('date'))) + expect(gte).toContain(`${TABLE}.created_at`) + expect(gte).toContain('::timestamptz') + expect(gte).not.toContain("data->>'createdAt'") + + const lt = render(fieldPredicate(TABLE, 'updatedAt', 'lt', '2026-06-01', col('date'))) + expect(lt).toContain(`${TABLE}.updated_at`) + expect(lt).toContain('::timestamptz') + }) + + it('supports in / isNull on system columns', () => { + expect(render(fieldPredicate(TABLE, 'createdAt', 'isNull', undefined, col('date')))).toContain( + `${TABLE}.created_at IS NULL` + ) + const inClause = render( + fieldPredicate(TABLE, 'createdAt', 'in', ['2026-01-01', '2026-02-01'], col('date')) + ) + expect(inClause).toContain(`${TABLE}.created_at`) + expect(inClause).toContain('::timestamptz') + }) +}) + +/** + * Regression coverage for #5920 — "filtering on built-in columns silently + * returns zero rows". Three distinct defects fed that report: the missing + * system-column dispatch (fixed above), `id` never being a system column at + * all, and the session-dependent `timestamptz` promotion that shifts every + * bound by the server's `TimeZone` GUC. + */ +describe('system columns (#5920)', () => { + it('normalizes timestamp bounds to UTC wall clock, not the session TimeZone', () => { + // `created_at`/`updated_at` are `timestamp WITHOUT time zone` holding UTC. + // Without `AT TIME ZONE 'UTC'` the comparison result depends on the server's + // TimeZone setting, so a UTC-3 day range (the reporter's exact query) lands + // off by the offset at both boundaries. + for (const field of ['createdAt', 'updatedAt'] as const) { + const out = render( + fieldPredicate(TABLE, field, 'gte', '2026-07-24T03:00:00.000Z', col('date')) + ) + expect(out).toContain("::timestamptz AT TIME ZONE 'UTC'") + } + }) + + it('filters id as the real text column, with no timestamptz cast', () => { + const out = render(fieldPredicate(TABLE, 'id', 'eq', 'row-123', col('string'))) + expect(out).toContain(`${TABLE}.id =`) + expect(out).not.toContain("data->>'id'") + expect(out).not.toContain('timestamptz') + }) + + it('supports the pattern ops on id (text), which are meaningless on timestamps', () => { + expect(render(fieldPredicate(TABLE, 'id', 'contains', 'abc', col('string')))).toContain( + `${TABLE}.id ILIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'startsWith', 'abc', col('string')))).toContain( + `${TABLE}.id ILIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'like', 'ab*', col('string')))).toContain( + `${TABLE}.id LIKE` + ) + expect(render(fieldPredicate(TABLE, 'id', 'ncontains', 'abc', col('string')))).toContain( + 'NOT (' + ) + + expect(() => fieldPredicate(TABLE, 'createdAt', 'contains', 'abc', col('date'))).toThrow( + /not supported on the built-in column "createdAt"/ + ) + }) + + it('rejects an empty pattern on id rather than matching every row', () => { + expect(() => fieldPredicate(TABLE, 'id', 'contains', '', col('string'))).toThrow( + /requires a non-empty value/ + ) + }) + + it('supports id in ranges and membership', () => { + expect(render(fieldPredicate(TABLE, 'id', 'gt', 'row-100', col('string')))).toContain( + `${TABLE}.id >` + ) + const inClause = render(fieldPredicate(TABLE, 'id', 'in', ['a', 'b'], col('string'))) + expect(inClause).toContain(`${TABLE}.id IN (`) + }) + + it('sorts id as a direct column ref', () => { + expect(render(buildSortClause({ id: 'asc' }, TABLE, NO_COLUMNS))).toBe(`${TABLE}.id ASC`) + }) + + it('maps isEmpty/isNotEmpty to IS NULL / IS NOT NULL (not inverted)', () => { + expect(render(fieldPredicate(TABLE, 'createdAt', 'isEmpty', undefined, col('date')))).toContain( + 'IS NULL' + ) + expect( + render(fieldPredicate(TABLE, 'createdAt', 'isNotEmpty', undefined, col('date'))) + ).toContain('IS NOT NULL') + }) + + it('reaches the same clause through the legacy $-grammar', () => { + // The v1 API path in the bug report goes through buildFilterClause, so the + // shared `fieldPredicate` leaf must cover it identically. + const out = render( + buildFilterClause( + { createdAt: { $gte: '2026-07-24T03:00:00.000Z', $lte: '2026-07-25T02:59:59.999Z' } }, + TABLE, + NO_COLUMNS + ) + ) + expect(out).toContain(`${TABLE}.created_at >=`) + expect(out).toContain(`${TABLE}.created_at <=`) + expect(out).toContain("AT TIME ZONE 'UTC'") + expect(out).not.toContain('requires a number') + }) +}) + +describe('range operators on non-numeric column types', () => { + it('compares string columns lexicographically as text (no numeric cast)', () => { + const cols: ColumnDefinition[] = [{ name: 'name', type: 'string' }] + const out = render(buildFilterClause({ name: { $gte: 'M' } }, TABLE, cols)) + expect(out).toContain(`${TABLE}.data->>'name' >=`) + expect(out).not.toContain('::numeric') + }) + + it('rejects ranges on boolean / json columns with a type-naming message', () => { + for (const type of ['boolean', 'json'] as const) { + const cols: ColumnDefinition[] = [{ name: 'flag', type }] + expect(() => buildFilterClause({ flag: { $gt: 1 } }, TABLE, cols)).toThrow( + new RegExp(`\\(${type}\\) is not supported`) + ) + } + }) +}) + +describe('buildPredicateClause (v2 grammar)', () => { + it('all joins members with AND', () => { + const p: TablePredicate = { + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [{ name: 'wins', type: 'number' }])) + expect(out).toContain(' AND ') + expect(out).toContain('"slack_user_id":"U1"') + expect(out).toContain('::numeric') + }) + + it('any joins members with OR', () => { + const p: TablePredicate = { + any: [ + { field: 'status', op: 'eq', value: 'active' }, + { field: 'status', op: 'eq', value: 'pending' }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [])) + expect(out).toContain(' OR ') + expect(out).toContain('"status":"active"') + expect(out).toContain('"status":"pending"') + }) + + it('nests groups', () => { + const p: TablePredicate = { + all: [ + { field: 'wins', op: 'gte', value: 1 }, + { + any: [ + { field: 's', op: 'eq', value: 'a' }, + { field: 's', op: 'eq', value: 'b' }, + ], + }, + ], + } + const out = render(buildPredicateClause(p, TABLE, [])) + expect(out).toContain(' AND ') + expect(out).toContain(' OR ') + }) + + it('an empty group is a no-op (undefined)', () => { + expect(buildPredicateClause({ all: [] }, TABLE, [])).toBeUndefined() + expect(buildPredicateClause({ any: [] }, TABLE, [])).toBeUndefined() + }) + + it('validates leaf field names', () => { + const p: TablePredicate = { all: [{ field: 'bad name', op: 'eq', value: 1 }] } + expect(() => buildPredicateClause(p, TABLE, [])).toThrow('Invalid field name') + }) +}) + +/** + * Cross-version safety. If a client speaking the v2 predicate grammar reaches a + * server that predates it, the predicate arrives at the LEGACY `$`-compiler as + * `{ all: [...] }`. That used to be skipped as "an array on a regular field", + * compiling to no WHERE clause — which on a bulk delete means every row rather + * than none. The guard turns that into a loud, self-describing failure, and it + * sits at the one choke point every filter path shares (`queryRows`, + * `update-runner`, `delete-runner`, inline and background). + */ +describe('legacy compiler rejects a v2 predicate (version-mismatch fail-fast)', () => { + it('throws on a top-level all/any group instead of emitting no clause', () => { + for (const group of ['all', 'any'] as const) { + expect(() => + buildFilterClause( + { [group]: [{ field: 'tenant_id', op: 'eq', value: 'acme' }] } as unknown as Filter, + TABLE, + NO_COLUMNS + ) + ).toThrow(/v2 predicate tree/) + } + }) + + it('catches one nested inside a legacy $or', () => { + expect(() => + buildFilterClause( + { + $or: [{ status: 'a' }, { all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }] }], + } as unknown as Filter, + TABLE, + NO_COLUMNS + ) + ).toThrow(/v2 predicate tree/) + }) + + it('leaves legitimate legacy filters alone', () => { + expect(buildFilterClause({ status: 'archived' }, TABLE, NO_COLUMNS)).toBeDefined() + expect( + buildFilterClause({ $or: [{ status: 'a' }, { status: 'b' }] }, TABLE, NO_COLUMNS) + ).toBeDefined() + // An ordinary column holding an array stays a silent skip — only the + // predicate discriminators `all`/`any` are treated as a version mismatch. + expect(() => + buildFilterClause({ status: ['a', 'b'] } as unknown as Filter, TABLE, NO_COLUMNS) + ).not.toThrow() + }) +}) diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index bcb9a224ab8..f6b8be7dc36 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -527,10 +527,12 @@ describe('Validation', () => { expect(result.errors[0]).toContain('abc123') }) - it('should be case-insensitive for string comparisons', () => { + it('should be case-sensitive for string comparisons', () => { + // U333 vs u333: differing case is a DISTINCT value (matches the DB + // containment leaf). This is the v2 contract that fixes the upsert wedge. const data = { id: 'ABC123', email: 'new@example.com', name: 'New User' } const result = validateUniqueConstraints(data, schema, existingRows) - expect(result.valid).toBe(false) + expect(result.valid).toBe(true) }) it('should exclude specified row from checks (for updates)', () => { diff --git a/apps/sim/lib/table/column-keys.ts b/apps/sim/lib/table/column-keys.ts index eb34f094861..71565101f4d 100644 --- a/apps/sim/lib/table/column-keys.ts +++ b/apps/sim/lib/table/column-keys.ts @@ -12,8 +12,11 @@ import { generateId } from '@sim/utils/id' import type { ColumnDefinition, Filter, + PredicateNode, RowData, Sort, + SortSpec, + TablePredicate, TableSchema, WorkflowGroup, } from '@/lib/table/types' @@ -171,6 +174,32 @@ export function sortNamesToIds(sort: Sort, idByName: ReadonlyMap return out } +/** + * Translates a v2 predicate's leaf `field` names → column ids (recursing through + * `all`/`any` groups). Fields with no matching column pass through unchanged. + * The v2 analogue of {@link filterNamesToIds}. + */ +export function predicateNamesToIds( + predicate: TablePredicate, + idByName: ReadonlyMap +): TablePredicate { + const remap = (node: PredicateNode): PredicateNode => { + // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. + if ('all' in node) return { all: node.all.map(remap) } + if ('any' in node) return { any: node.any.map(remap) } + return { ...node, field: idByName.get(node.field) ?? node.field } + } + return remap(predicate) as TablePredicate +} + +/** Translates a v2 sort spec's field names → column ids. Unknown fields pass through. */ +export function sortSpecNamesToIds( + sort: SortSpec, + idByName: ReadonlyMap +): SortSpec { + return sort.map((s) => ({ field: idByName.get(s.field) ?? s.field, direction: s.direction })) +} + // The outbound direction (stored id-keyed row → name-keyed) deliberately does // NOT live here: a `select` cell's value also needs translating, and keeping the // key half separately callable is what let boundaries translate the keys and diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 5201815e8f3..1384d3dc181 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -15,6 +15,17 @@ export const TABLE_LIMITS = { MAX_DESCRIPTION_LENGTH: 500, DEFAULT_QUERY_LIMIT: 100, MAX_QUERY_LIMIT: 1000, + /** + * Byte budget for one query response. `queryRows` drains rows in bounded + * batches; an UNBOUNDED query (no limit) that outgrows the budget fails fast + * (the whole result was promised — a partial page would be silent truncation), + * while a BOUNDED page cuts early and returns the partial list with + * `nextCursor` set. At least one row is always returned on a bounded page. + * Measured against serialized row `data` only, not the full HTTP envelope. + */ + MAX_QUERY_RESULT_BYTES: 5 * 1024 * 1024, // 5MB + /** Hard row cap per internal fetch batch inside queryRows' bounded drain loop. */ + QUERY_BATCH_MAX_ROWS: 10000, /** Batch size for bulk update operations */ UPDATE_BATCH_SIZE: 100, /** Batch size for bulk delete operations */ @@ -62,10 +73,14 @@ export const DEFAULT_TABLE_PLAN_LIMITS = { } as const /** - * Byte budget for one page of row reads, or null when disabled (the default). - * Dev-preview of the byte-bounded pagination follow-up: set `TABLE_MAX_PAGE_BYTES` - * to cut pages early once their serialized row data exceeds the budget. The - * production version moves the cut into SQL — see the pagination-hardening plan. + * Byte budget at which a **bounded** page (one with an explicit `limit`) is cut + * short, or `null` when disabled — the default. Opt in with `TABLE_MAX_PAGE_BYTES`. + * + * Off by default because a short page is only safe for a client that terminates + * on `nextCursor === null`; a pre-existing v1 pager terminating on + * `rows.length < limit` would read the cut as end-of-data and silently truncate. + * Unbounded queries (no `limit`) are unaffected — they always fail fast at + * `TABLE_LIMITS.MAX_QUERY_RESULT_BYTES` rather than return a partial result. */ export function getMaxPageBytes(): number | null { const value = envNumber(env.TABLE_MAX_PAGE_BYTES, 0, { min: 0, integer: true }) @@ -153,6 +168,35 @@ export const COLUMN_TYPES = ['string', 'number', 'boolean', 'date', 'json', 'sel /** Maximum number of options a `select`/`multiselect` column may declare. */ export const MAX_SELECT_OPTIONS = 100 +/** + * The v2 filter operators, as a runtime tuple so both the `FilterOp` type and + * the boundary Zod enum derive from one source. Order is not significant. + */ +export const FILTER_OPS = [ + 'eq', + 'ne', + 'gt', + 'gte', + 'lt', + 'lte', + 'in', + 'nin', + 'contains', + 'ncontains', + 'startsWith', + 'endsWith', + 'like', + 'ilike', + 'nlike', + 'nilike', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +] as const + +export const SORT_DIRECTIONS = ['asc', 'desc'] as const + export const NAME_PATTERN = /^[a-z_][a-z0-9_]*$/i export const USER_TABLE_ROWS_SQL_NAME = 'user_table_rows' diff --git a/apps/sim/lib/table/delete-runner.ts b/apps/sim/lib/table/delete-runner.ts index a10a9b309fb..e2ff5e6773e 100644 --- a/apps/sim/lib/table/delete-runner.ts +++ b/apps/sim/lib/table/delete-runner.ts @@ -110,6 +110,11 @@ export async function runTableDelete(payload: TableDeletePayload): Promise const filterClause = filter ? buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, table.schema.columns) : undefined + // A filter that was SUPPLIED but compiles to no clause must never widen into + // "delete every row" — `and()` silently drops an undefined clause downstream. + // Mirrors the guard in update-runner and the inline deleteRowsByFilter path; + // an absent filter is still legitimate (delete-all is an explicit caller mode). + if (filter && !filterClause) throw new Error('Filter is required for bulk delete') const excluded = new Set(excludeRowIds ?? []) // Resume the persisted count: a retried attempt's earlier batches are already committed, diff --git a/apps/sim/lib/table/errors.ts b/apps/sim/lib/table/errors.ts new file mode 100644 index 00000000000..6b80b6a7260 --- /dev/null +++ b/apps/sim/lib/table/errors.ts @@ -0,0 +1,28 @@ +/** + * Stable, machine-readable codes for table query failures. SDKs and clients + * branch on these instead of string-matching human-facing messages. + */ +export type TableQueryErrorCode = + | 'TABLE_QUERY_RESULT_TOO_LARGE' + | 'INVALID_CURSOR' + | 'CURSOR_SORT_CONFLICT' + | 'INVALID_FILTER' + | 'INVALID_ORDER' + +/** + * Error thrown when caller-supplied filter or sort input is malformed. + * Routes should map this to HTTP 400 with the message preserved. + * + * Lives outside `sql.ts` so client-bundled modules (the block definitions pull + * in the query-builder converters) can reference it without dragging drizzle-orm + * into the browser chunk. + */ +export class TableQueryValidationError extends Error { + readonly code?: TableQueryErrorCode + + constructor(message: string, code?: TableQueryErrorCode) { + super(message) + this.name = 'TableQueryValidationError' + this.code = code + } +} diff --git a/apps/sim/lib/table/export-runner.ts b/apps/sim/lib/table/export-runner.ts index 8af3628194c..e8a9cf5fb54 100644 --- a/apps/sim/lib/table/export-runner.ts +++ b/apps/sim/lib/table/export-runner.ts @@ -80,7 +80,9 @@ export async function runTableExport(payload: TableExportPayload): Promise let exported = 0 let firstJsonRow = true - let after: { orderKey: string; id: string } | null = null + // `order_key` is nullable (rows predating the backfill), and the page query + // seeks NULLs explicitly — so the cursor has to carry a null too. + let after: { orderKey: string | null; id: string } | null = null while (true) { // Ownership gate before every page: a canceled job stops within one batch. const owns = await updateJobProgress(tableId, exported, jobId) diff --git a/apps/sim/lib/table/index.ts b/apps/sim/lib/table/index.ts index b4352ff2b21..fad60f65370 100644 --- a/apps/sim/lib/table/index.ts +++ b/apps/sim/lib/table/index.ts @@ -10,6 +10,7 @@ export * from '@/lib/table/column-keys' export * from '@/lib/table/columns/service' export * from '@/lib/table/constants' export * from '@/lib/table/dates' +export * from '@/lib/table/errors' export * from '@/lib/table/import' export * from '@/lib/table/import-data' export * from '@/lib/table/jobs/service' diff --git a/apps/sim/lib/table/jobs/service.ts b/apps/sim/lib/table/jobs/service.ts index 7b1706e32da..ca916e689b8 100644 --- a/apps/sim/lib/table/jobs/service.ts +++ b/apps/sim/lib/table/jobs/service.ts @@ -209,16 +209,17 @@ export async function getJobProgress(tableId: string, jobId: string): Promise> { +): Promise> { const deleteMask = await pendingDeleteMask(table) const rows = await db .select({ id: userTableRows.id, data: userTableRows.data, orderKey: userTableRows.orderKey }) @@ -228,14 +229,22 @@ export async function selectExportRowPage( eq(userTableRows.tableId, table.id), eq(userTableRows.workspaceId, table.workspaceId), deleteMask, + // `order_key` is nullable and sorts LAST, so the page order is "keyed rows by + // key, then the unkeyed tail by id". A bare row-constructor comparison is NULL + // for unkeyed rows (dropping them) and NULL for an unkeyed anchor (dropping + // everything), which silently truncates exports. Seek per anchor kind instead. after - ? sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})` + ? after.orderKey === null + ? sql`${userTableRows.orderKey} IS NULL AND ${userTableRows.id} > ${after.id}` + : sql`(${userTableRows.orderKey} IS NULL OR (${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id}))` : undefined ) ) .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id)) .limit(limit) - return rows as Array<{ id: string; data: RowData; orderKey: string }> + // drizzle types a jsonb column as `unknown`; every writer goes through the + // row-data validators, so narrowing here is a projection, not an assumption. + return rows.map((r) => ({ ...r, data: r.data as RowData })) } /** How long a terminal export stays listable (and re-downloadable from the tray). */ diff --git a/apps/sim/lib/table/planner.ts b/apps/sim/lib/table/planner.ts index 848e2c2d6de..85f4969b6f6 100644 --- a/apps/sim/lib/table/planner.ts +++ b/apps/sim/lib/table/planner.ts @@ -5,19 +5,52 @@ export type DbExecutor = typeof db | DbTransaction export type DbTransaction = Parameters[0]>[0] /** - * Runs `fn` with seq scans penalized (`SET LOCAL`, so the flag dies with the - * transaction). JSONB predicates and sort keys (`->>` extraction, `@>` - * containment, lateral `jsonb_each_text`) are opaque to the planner — it - * estimates a handful of matching rows and picks a parallel seq scan over the - * entire shared `user_table_rows` relation (every tenant's rows) instead of the - * tenant's own index. Measured on a 1M-row table inside a 12M-row relation: - * filtered count 12.7s → 1.0s, sorted page 9.7s → 0.76s, filtered bulk select - * 14.4s → tenant-bounded. The flag only penalizes the plan shape: if no index - * plan exists, the seq scan still runs. + * Statement/lock timeout for user-table READ queries. Reads are bounded tighter + * than the write path because filter shape is caller-controlled: a public API + * key can drive an arbitrary predicate (a catastrophic-backtracking `match` + * regex, a wide unindexed scan), and an untimed read pins a connection from the + * small shared `web` pool — one abusive key can starve every tenant. `SET LOCAL` + * scopes both to the surrounding transaction, so they die when it commits. */ -export async function withSeqscanOff(fn: (trx: DbTransaction) => Promise): Promise { +const READ_STATEMENT_TIMEOUT_MS = 15_000 +const READ_LOCK_TIMEOUT_MS = 3_000 + +async function setReadTimeouts(trx: DbTransaction): Promise { + await trx.execute(sql.raw(`SET LOCAL statement_timeout = '${READ_STATEMENT_TIMEOUT_MS}ms'`)) + await trx.execute(sql.raw(`SET LOCAL lock_timeout = '${READ_LOCK_TIMEOUT_MS}ms'`)) +} + +/** + * Runs a user-table read inside a transaction that always caps `statement_timeout` + * / `lock_timeout` (see {@link setReadTimeouts}). Pass `seqscanOff` for queries + * with no tenant-bounded index plan — custom column sorts and filtered counts — + * where the planner otherwise seq-scans the whole shared `user_table_rows` + * relation (every tenant's rows); see {@link withSeqscanOff} for the measured + * deltas. Default-order keyset/offset pages already stream the + * `(table_id, order_key, id)` index, so they take the timeout without the flag. + */ +export async function withReadGuards( + fn: (trx: DbTransaction) => Promise, + opts?: { seqscanOff?: boolean } +): Promise { return db.transaction(async (trx) => { - await trx.execute(sql`SET LOCAL enable_seqscan = off`) + await setReadTimeouts(trx) + if (opts?.seqscanOff) await trx.execute(sql`SET LOCAL enable_seqscan = off`) return fn(trx) }) } + +/** + * Runs `fn` with seq scans penalized (`SET LOCAL`, so the flag dies with the + * transaction) AND the read timeouts above. JSONB predicates and sort keys + * (`->>` extraction, `@>` containment, lateral `jsonb_each_text`) are opaque to + * the planner — it estimates a handful of matching rows and picks a parallel + * seq scan over the entire shared `user_table_rows` relation (every tenant's + * rows) instead of the tenant's own index. Measured on a 1M-row table inside a + * 12M-row relation: filtered count 12.7s → 1.0s, sorted page 9.7s → 0.76s, + * filtered bulk select 14.4s → tenant-bounded. The flag only penalizes the plan + * shape: if no index plan exists, the seq scan still runs (and the timeout caps it). + */ +export async function withSeqscanOff(fn: (trx: DbTransaction) => Promise): Promise { + return withReadGuards(fn, { seqscanOff: true }) +} diff --git a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts index 131fdf7a74a..fa9dd5fb93e 100644 --- a/apps/sim/lib/table/query-builder/__tests__/converters.test.ts +++ b/apps/sim/lib/table/query-builder/__tests__/converters.test.ts @@ -8,10 +8,17 @@ import { describe, expect, it } from 'vitest' import { filterRulesToFilter, + filterRulesToPredicate, filterToRules, + isTablePredicate, + predicateToFilter, + predicateToFilterRules, pruneFilterForColumns, + prunePredicateForColumns, + sortRulesToSortSpec, + toLegacyFilter, } from '@/lib/table/query-builder/converters' -import type { ColumnDefinition, FilterRule } from '@/lib/table/types' +import type { ColumnDefinition, FilterRule, SortRule, TablePredicate } from '@/lib/table/types' function rule(overrides: Partial): FilterRule { return { @@ -122,6 +129,158 @@ describe('filterToRules', () => { }) }) +describe('filterRulesToPredicate (v2)', () => { + it('an AND group becomes a single all-group', () => { + const p = filterRulesToPredicate([ + rule({ column: 'slack_user_id', operator: 'in', value: 'U1, U2' }), + rule({ column: 'wins', operator: 'gte', value: '10' }), + ]) + expect(p).toEqual({ + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + ], + }) + }) + + it('an or boundary splits into an any-of-all groups', () => { + const p = filterRulesToPredicate([ + rule({ column: 'status', operator: 'eq', value: 'active' }), + rule({ column: 'status', operator: 'eq', value: 'pending', logicalOperator: 'or' }), + ]) + expect(p).toEqual({ + any: [ + { all: [{ field: 'status', op: 'eq', value: 'active' }] }, + { all: [{ field: 'status', op: 'eq', value: 'pending' }] }, + ], + }) + }) + + it('valueless ops omit the value', () => { + const p = filterRulesToPredicate([rule({ column: 'note', operator: 'isEmpty' })]) + expect(p).toEqual({ all: [{ field: 'note', op: 'isEmpty' }] }) + }) + + it('returns null for no rules', () => { + expect(filterRulesToPredicate([])).toBeNull() + }) + + it('round-trips rules → predicate → rules (operator + value preserved)', () => { + const rules: FilterRule[] = [ + rule({ column: 'wins', operator: 'gte', value: '10' }), + rule({ column: 'name', operator: 'contains', value: 'jo', logicalOperator: 'or' }), + ] + const back = predicateToFilterRules(filterRulesToPredicate(rules)) + expect(back.map((r) => [r.column, r.operator, r.value, r.logicalOperator])).toEqual([ + ['wins', 'gte', '10', 'and'], + ['name', 'contains', 'jo', 'or'], + ]) + }) +}) + +describe('predicateToFilterRules (v2)', () => { + it('flattens an all-group to and-joined rules', () => { + const p: TablePredicate = { + all: [ + { field: 'a', op: 'eq', value: 1 }, + { field: 'b', op: 'isNotEmpty' }, + ], + } + const rules = predicateToFilterRules(p) + expect(rules.map((r) => [r.column, r.operator, r.value])).toEqual([ + ['a', 'eq', '1'], + ['b', 'isNotEmpty', ''], + ]) + }) + + it('returns [] for null', () => { + expect(predicateToFilterRules(null)).toEqual([]) + }) +}) + +describe('predicateToFilter (v2 → legacy)', () => { + it('maps an all-group to $and with bare-op leaves', () => { + const p: TablePredicate = { + all: [ + { field: 'slack_user_id', op: 'in', value: ['U1', 'U2'] }, + { field: 'wins', op: 'gte', value: 10 }, + { field: 'name', op: 'eq', value: 'x' }, + ], + } + expect(predicateToFilter(p)).toEqual({ + $and: [{ slack_user_id: { $in: ['U1', 'U2'] } }, { wins: { $gte: 10 } }, { name: 'x' }], + }) + }) + + it('maps an any-group to $or and valueless ops to $empty', () => { + const p: TablePredicate = { + any: [ + { field: 'a', op: 'isEmpty' }, + { field: 'b', op: 'isNotEmpty' }, + ], + } + expect(predicateToFilter(p)).toEqual({ + $or: [{ a: { $empty: true } }, { b: { $empty: false } }], + }) + }) + + /** + * The conversion is only safe if it is lossless: `buildFilterClause` skips an + * `undefined` condition and any array-valued one, so a leaf that converts to + * either DISAPPEARS from the WHERE — and a predicate whose only leaf disappears + * compiles to no WHERE at all, turning a bulk delete into a table wipe. + */ + it('throws rather than emit a leaf the legacy compiler would discard', () => { + expect(() => + predicateToFilter({ all: [{ field: 'status', op: 'eq', value: ['a', 'b'] }] }) + ).toThrow(/does not accept an array/) + expect(() => predicateToFilter({ all: [{ field: 'status', op: 'eq' }] })).toThrow( + /requires a value/ + ) + }) +}) + +describe('sortRulesToSortSpec (v2)', () => { + it('maps rules to an ordered field/direction list', () => { + const rules: SortRule[] = [ + { id: '1', column: 'wins', direction: 'desc' }, + { id: '2', column: 'name', direction: 'asc' }, + ] + expect(sortRulesToSortSpec(rules)).toEqual([ + { field: 'wins', direction: 'desc' }, + { field: 'name', direction: 'asc' }, + ]) + }) + + it('skips column-less rules and returns null when empty', () => { + expect(sortRulesToSortSpec([{ id: '1', column: '', direction: 'asc' }])).toBeNull() + }) +}) + +it('does not throw "rules is not iterable" for a non-array builder value', () => { + expect(filterRulesToPredicate({} as unknown as FilterRule[])).toBeNull() + expect(filterRulesToPredicate(undefined as unknown as FilterRule[])).toBeNull() +}) + +it('rejects predicate-shaped members rather than dropping them', () => { + // Mixing grammars used to silently discard the predicate-shaped leaf, widening + // the filter — "delete archived rows for tenant acme" became "…for every tenant". + expect(() => + filterRulesToPredicate([ + { field: 'tenant_id', op: 'eq', value: 'acme' }, + rule({ column: 'status', operator: 'eq', value: 'archived' }), + ] as unknown as FilterRule[]) + ).toThrow(/predicate condition/) +}) + +it('still ignores a genuinely blank builder row', () => { + expect( + filterRulesToPredicate([ + rule({ column: '', operator: 'eq', value: '' }), + rule({ column: 'status', operator: 'eq', value: 'archived' }), + ]) + ).toEqual({ all: [{ field: 'status', op: 'eq', value: 'archived' }] }) +}) describe('select option ids survive as text', () => { const numericIdColumn: ColumnDefinition = { id: 'status', @@ -210,3 +369,144 @@ describe('pruneFilterForColumns', () => { expect(pruneFilterForColumns(null, [single])).toBeNull() }) }) + +describe('filterRulesToPredicate select-awareness (grid parity)', () => { + const SELECT_COLS: ColumnDefinition[] = [ + { + id: 'col_status', + name: 'col_status', + type: 'select', + options: [{ id: '123', name: 'One Two Three' }], + }, + ] + + it('never scalar-coerces a select operand (a numeric-looking option id stays a string)', () => { + const p = filterRulesToPredicate( + [rule({ column: 'col_status', operator: 'eq', value: '123' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'col_status', op: 'eq', value: '123' }] }) + }) + + it('keeps select in-list members as strings', () => { + const p = filterRulesToPredicate( + [rule({ column: 'col_status', operator: 'in', value: '123, 456' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'col_status', op: 'in', value: ['123', '456'] }] }) + }) + + it('still coerces non-select columns', () => { + const p = filterRulesToPredicate( + [rule({ column: 'wins', operator: 'eq', value: '123' })], + SELECT_COLS + ) + expect(p).toEqual({ all: [{ field: 'wins', op: 'eq', value: 123 }] }) + }) +}) + +describe('prunePredicateForColumns', () => { + const COLS: ColumnDefinition[] = [ + { id: 'col_s', name: 'col_s', type: 'select', options: [{ id: 'o1', name: 'One' }] }, + { id: 'col_n', name: 'col_n', type: 'number' }, + ] + + it('drops an operator a select column no longer accepts, keeps the rest', () => { + const p = prunePredicateForColumns( + { + all: [ + { field: 'col_s', op: 'contains', value: 'o1' }, // single-select: contains not allowed + { field: 'col_n', op: 'gte', value: 5 }, + ], + }, + COLS + ) + expect(p).toEqual({ all: [{ field: 'col_n', op: 'gte', value: 5 }] }) + }) + + it('returns the same reference when nothing is dropped', () => { + const p = { all: [{ field: 'col_n', op: 'gte' as const, value: 5 }] } + expect(prunePredicateForColumns(p, COLS)).toBe(p) + }) + + it('collapses to null when every condition is dropped', () => { + expect( + prunePredicateForColumns({ all: [{ field: 'col_s', op: 'contains', value: 'o1' }] }, COLS) + ).toBeNull() + }) +}) + +describe('isTablePredicate', () => { + it('discriminates the two grammars group-first', () => { + expect(isTablePredicate({ all: [] })).toBe(true) + expect(isTablePredicate({ any: [] })).toBe(true) + expect(isTablePredicate({ status: 'active' })).toBe(false) + expect(isTablePredicate({ $or: [{ a: 1 }] } as never)).toBe(false) + }) +}) + +it('prunePredicateForColumns fails closed on a malformed value instead of throwing', () => { + expect(prunePredicateForColumns({ column: 'name', operator: 'eq' } as never, [])).toBeNull() +}) + +/** + * Review findings from PR #6067 (greptile P1 ×2, bugbot High). The dual-grammar + * union's legacy branch accepts any non-empty object WITHOUT stripping, so a + * hybrid node reaches the downgrade Zod-approved — and predicateToFilter used + * to convert it group-first, silently DROPPING the leaf half and widening the + * filter on the async destructive routes (delete-async, cancel-runs, columns/run). + */ +describe('toLegacyFilter / predicateToFilter hybrid safety', () => { + const hybrid = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + field: 'status', + op: 'eq', + value: 'archived', + } as never + + it('predicateToFilter throws on a hybrid node instead of dropping the leaf', () => { + expect(() => predicateToFilter(hybrid)).toThrow(/not both/) + }) + + it('throws on a node with both group keys instead of dropping the "any" half', () => { + const dual = { + all: [{ field: 'tenant_id', op: 'eq', value: 'acme' }], + any: [{ field: 'status', op: 'eq', value: 'archived' }], + } as never + expect(() => predicateToFilter(dual)).toThrow(/either "all" or "any"/) + expect(() => toLegacyFilter(dual)).toThrow(/either "all" or "any"/) + }) + + it('toLegacyFilter shape-validates before converting', () => { + expect(() => toLegacyFilter(hybrid)).toThrow(/not both/) + // malformed group value is also caught, not crashed on + expect(() => toLegacyFilter({ all: [{ all: 'x' }] } as never)).toThrow() + }) + + it('toLegacyFilter passes a well-formed predicate and a legacy filter through', () => { + expect(toLegacyFilter({ all: [{ field: 'a', op: 'eq', value: 1 }] })).toEqual({ + $and: [{ a: 1 }], + }) + expect(toLegacyFilter({ status: 'archived' })).toEqual({ status: 'archived' }) + }) +}) + +describe('isTablePredicate vs columns literally named all/any', () => { + it('routes a non-array all/any value to the legacy compiler (a real column)', () => { + // NAME_PATTERN allows a column named "all". Equality shorthand and operator + // objects on it are legacy filters, not predicate groups. + expect(isTablePredicate({ all: 'x' } as never)).toBe(false) + expect(isTablePredicate({ any: { $eq: 'x' } } as never)).toBe(false) + }) + + it('array-valued all/any is a predicate group (documented precedence)', () => { + // A legacy filter with an ARRAY on a regular field was always a dropped + // no-op condition, so predicate precedence here regresses nothing. + expect(isTablePredicate({ all: [] })).toBe(true) + expect(isTablePredicate({ any: [{ field: 'a', op: 'eq', value: 1 }] })).toBe(true) + }) +}) + +it('toLegacyFilter rejects an empty group instead of downgrading it to no WHERE', () => { + expect(() => toLegacyFilter({ all: [] })).toThrow(/at least one condition/) +}) diff --git a/apps/sim/lib/table/query-builder/__tests__/validate.test.ts b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts new file mode 100644 index 00000000000..12eea2427a0 --- /dev/null +++ b/apps/sim/lib/table/query-builder/__tests__/validate.test.ts @@ -0,0 +1,215 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { TableQueryValidationError } from '@/lib/table/errors' +import { + validatePredicate, + validatePredicateShape, + validateSortSpec, +} from '@/lib/table/query-builder/validate' +import type { ColumnDefinition } from '@/lib/table/types' + +const COLS: ColumnDefinition[] = [ + { name: 'wins', type: 'number' }, + { name: 'status', type: 'string' }, + { name: 'metadata', type: 'json' }, +] + +describe('validatePredicate', () => { + it('accepts a valid predicate over real + system columns', () => { + expect(() => + validatePredicate( + { + all: [ + { field: 'wins', op: 'gte', value: 10 }, + { field: 'createdAt', op: 'lt', value: '2026-01-01' }, + { any: [{ field: 'status', op: 'eq', value: 'active' }] }, + ], + }, + COLS + ) + ).not.toThrow() + }) + + it('rejects an unknown column', () => { + expect(() => validatePredicate({ all: [{ field: 'nope', op: 'eq', value: 1 }] }, COLS)).toThrow( + /Unknown filter column/ + ) + }) + + it('rejects equality/containment ops on a json column', () => { + for (const op of ['eq', 'ne', 'in', 'nin'] as const) { + const value = op === 'in' || op === 'nin' ? ['x'] : 'x' + expect(() => validatePredicate({ all: [{ field: 'metadata', op, value }] }, COLS)).toThrow( + /json column/ + ) + } + }) + + it('allows text-match / null ops on a json column', () => { + expect(() => + validatePredicate({ all: [{ field: 'metadata', op: 'ilike', value: '*x*' }] }, COLS) + ).not.toThrow() + expect(() => + validatePredicate({ all: [{ field: 'metadata', op: 'isNull' }] }, COLS) + ).not.toThrow() + }) + + it('rejects an empty in/nin array', () => { + expect(() => + validatePredicate({ all: [{ field: 'wins', op: 'in', value: [] }] }, COLS) + ).toThrow(/non-empty array/) + }) + + it('rejects an invalid field name', () => { + expect(() => + validatePredicate({ all: [{ field: "x'; DROP", op: 'eq', value: 1 }] }, COLS) + ).toThrow(/Invalid filter column/) + }) + + it('carries the INVALID_FILTER code', () => { + try { + validatePredicate({ all: [{ field: 'nope', op: 'eq', value: 1 }] }, COLS) + expect.unreachable() + } catch (e) { + expect(e).toBeInstanceOf(TableQueryValidationError) + expect((e as TableQueryValidationError).code).toBe('INVALID_FILTER') + } + }) +}) + +describe('validateSortSpec', () => { + it('accepts real and system columns', () => { + expect(() => + validateSortSpec( + [ + { field: 'wins', direction: 'desc' }, + { field: 'updatedAt', direction: 'asc' }, + ], + COLS + ) + ).not.toThrow() + }) + + it('rejects an unknown sort column with INVALID_ORDER', () => { + try { + validateSortSpec([{ field: 'nope', direction: 'asc' }], COLS) + expect.unreachable() + } catch (e) { + expect((e as TableQueryValidationError).code).toBe('INVALID_ORDER') + } + }) +}) + +/** + * These shapes pass structural validation but compile to a clause the legacy + * `$`-grammar discards — which turns a bulk delete/update into "match everything". + */ +describe('validatePredicate — leaves that would silently widen a bulk write', () => { + it('rejects a scalar op given an array (the eq-instead-of-in mistake)', () => { + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'eq', value: ['a', 'b'] }] }, COLS) + ).toThrow(/does not accept an array — use "in"/) + }) + + it('rejects a value-taking op with no value', () => { + expect(() => validatePredicate({ all: [{ field: 'status', op: 'eq' }] }, COLS)).toThrow( + /requires a value/ + ) + expect(() => validatePredicate({ all: [{ field: 'wins', op: 'gte' }] }, COLS)).toThrow( + /requires a value/ + ) + }) + + it('still allows the valueless ops', () => { + for (const op of ['isNull', 'isNotNull', 'isEmpty', 'isNotEmpty'] as const) { + expect(() => validatePredicate({ all: [{ field: 'status', op }] }, COLS)).not.toThrow() + } + }) + + it('rejects a hybrid group+leaf node instead of silently picking one', () => { + // Read group-first by the engine, leaf-first by predicateToFilter — so the gate + // would validate one predicate and the bulk-write path execute another. + expect(() => + validatePredicate( + { + all: [{ field: 'status', op: 'eq', value: 'benign' }], + field: 'nope', + op: 'isNotNull', + } as never, + COLS + ) + ).toThrow(/not both/) + }) + + it('rejects a node carrying both group keys instead of dropping the "any" half', () => { + // Every group-first traversal reads `all` and drops `any` — half the + // caller's conditions would vanish, widening a bulk delete/update. + expect(() => + validatePredicate( + { + all: [{ field: 'status', op: 'eq', value: 'archived' }], + any: [{ field: 'status', op: 'eq', value: 'stale' }], + } as never, + COLS + ) + ).toThrow(/either "all" or "any"/) + }) + + it('caps in/nin list length', () => { + const huge = Array.from({ length: 1001 }, (_, i) => `v${i}`) + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'in', value: huge }] }, COLS) + ).toThrow(/at most 1000 values/) + expect(() => + validatePredicate({ all: [{ field: 'status', op: 'in', value: huge.slice(0, 1000) }] }, COLS) + ).not.toThrow() + }) +}) + +/** + * The copilot's `query_user_table` catalog entry still advertises the legacy + * `$`-grammar, so the agent sends `{ status: { $eq: 'x' } }`. That shape hit + * `validateLeaf` with `field: undefined` and came back as + * `Unknown filter column "undefined"` — a message that sends an LLM retrying + * column names instead of switching grammars. + */ +describe('validatePredicate — legacy $-grammar diagnostics', () => { + it('names the grammar mistake instead of blaming a phantom column', () => { + for (const legacy of [ + { status: { $eq: 'active' } }, + { $or: [{ status: 'a' }, { status: 'b' }] }, + { wins: { $gte: 10 } }, + ]) { + expect(() => validatePredicate(legacy as never, COLS)).toThrow(/legacy operator-object/) + expect(() => validatePredicate(legacy as never, COLS)).not.toThrow(/undefined/) + } + }) + + it('still rejects a plain non-predicate object clearly', () => { + expect(() => validatePredicate({ nonsense: 'x' } as never, COLS)).toThrow( + /must be a group .* or a condition/ + ) + }) +}) + +/** + * Bugbot round 2: `{all: []}` passes the dual union via the non-empty-OBJECT + * legacy branch, then compiled to no WHERE clause — silently widening a run / + * cancel / delete scope to every row. The shape validator now mirrors the Zod + * contract's .min(1). + */ +describe('empty groups are rejected at every layer', () => { + it('validatePredicateShape rejects an empty group', () => { + expect(() => validatePredicateShape({ all: [] })).toThrow(/at least one condition/) + expect(() => validatePredicateShape({ any: [] })).toThrow(/at least one condition/) + expect(() => validatePredicateShape({ all: [{ any: [] }] } as never)).toThrow( + /at least one condition/ + ) + }) + + it('validatePredicate rejects it too', () => { + expect(() => validatePredicate({ all: [] }, COLS)).toThrow(/at least one condition/) + }) +}) diff --git a/apps/sim/lib/table/query-builder/constants.ts b/apps/sim/lib/table/query-builder/constants.ts index a881fd0aba4..8a5b9af4117 100644 --- a/apps/sim/lib/table/query-builder/constants.ts +++ b/apps/sim/lib/table/query-builder/constants.ts @@ -34,7 +34,8 @@ export const VALUELESS_OPERATORS = new Set(['isEmpty', 'isNotEmpty']) * against the whole array can never be true. * * These must stay in step with the server whitelist in `lib/table/sql.ts`: the - * filter picker offers exactly these, and `pruneFilterForColumns` DROPS + * filter picker offers exactly these, and `pruneFilterForColumns` / + * `prunePredicateForColumns` DROP * anything outside them, so an operator missing here is silently discarded from * a filter the server would have accepted. `sql.test.ts` asserts the two sets * agree through `UI_TO_WIRE_OPERATOR` rather than leaving it to a comment. @@ -69,7 +70,12 @@ export const LOGICAL_OPERATORS = [ { value: 'or', label: 'or' }, ] as const -export const SORT_DIRECTIONS = [ +/** + * Direction picker options for the sort-builder UI. Distinct from the wire-level + * `SORT_DIRECTIONS` tuple in `lib/table/constants.ts` — both are star-exported + * through `lib/table/index.ts`, and a duplicate name resolves to nothing there. + */ +export const SORT_DIRECTION_OPTIONS = [ { value: 'asc', label: 'ascending' }, { value: 'desc', label: 'descending' }, ] as const diff --git a/apps/sim/lib/table/query-builder/converters.ts b/apps/sim/lib/table/query-builder/converters.ts index 297a244c3e4..5e938c64731 100644 --- a/apps/sim/lib/table/query-builder/converters.ts +++ b/apps/sim/lib/table/query-builder/converters.ts @@ -5,18 +5,24 @@ import { generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import { columnMatchesRef } from '@/lib/table/column-keys' +import { TableQueryValidationError } from '@/lib/table/errors' import { MULTI_SELECT_FILTER_OPERATORS, SINGLE_SELECT_FILTER_OPERATORS, } from '@/lib/table/query-builder/constants' +import { validatePredicateShape } from '@/lib/table/query-builder/validate' import type { ColumnDefinition, Filter, + FilterOp, FilterRule, JsonValue, + Predicate, Sort, SortDirection, SortRule, + SortSpec, + TablePredicate, } from '@/lib/table/types' /** @@ -112,6 +118,47 @@ export function pruneFilterForColumns( return filterRulesToFilter(kept, columns) } +/** + * Predicate-grammar sibling of {@link pruneFilterForColumns}: drops conditions a + * `select` column no longer accepts (operator stranded by a type/`multiple` + * change), so a stale applied filter can't fail every subsequent rows query. + */ +export function prunePredicateForColumns( + predicate: TablePredicate | null, + columns: ColumnDefinition[] +): TablePredicate | null { + if (!predicate) return null + // A malformed value (corrupt persisted state, a stale cast) fails CLOSED to + // "no filter" — throwing here would take down the whole table page render. + if (!('all' in predicate) && !('any' in predicate)) return null + + const rules = predicateToFilterRules(predicate) + const kept = rules.filter((rule) => { + const column = columns.find((c) => columnMatchesRef(c, rule.column)) + if (column?.type !== 'select') return true + const allowed = column.multiple ? MULTI_SELECT_FILTER_OPERATORS : SINGLE_SELECT_FILTER_OPERATORS + return allowed.has(rule.operator) + }) + + if (kept.length === rules.length) return predicate + return filterRulesToPredicate(kept, columns) +} + +/** + * Discriminates the v2 predicate tree from the legacy `$`-object on dual-grammar + * wire fields. Group-first, matching every other discrimination site. + */ +export function isTablePredicate(value: Filter | TablePredicate): value is TablePredicate { + // Require the group value to be an ARRAY: a legacy filter on a real column + // that happens to be named `all`/`any` (allowed by NAME_PATTERN) uses the + // equality shorthand ({ all: "x" }) or an operator object — neither is + // array-valued, so both keep routing to the legacy compiler. A column named + // all/any holding a literal array was already a dropped no-op condition in + // the legacy grammar, so predicate precedence on arrays regresses nothing. + const v = value as Record + return ('all' in v && Array.isArray(v.all)) || ('any' in v && Array.isArray(v.any)) +} + /** Converts a single UI sort rule to a Sort object for API queries. */ export function sortRuleToSort(rule: SortRule | null): Sort | null { if (!rule || !rule.column) return null @@ -280,3 +327,192 @@ function formatValueForBuilder(value: JsonValue): string { function normalizeSortDirection(direction: string): SortDirection { return direction === 'desc' ? 'desc' : 'asc' } + +/* ----------------------------- v2 grammar ----------------------------- */ + +const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isNotNull']) + +function ruleToPredicate(rule: FilterRule, keepAsText = false): Predicate { + const op = rule.operator as FilterOp + if (VALUELESS_OPS.has(op)) return { field: rule.column, op } + return { field: rule.column, op, value: parseValue(rule.value, rule.operator, keepAsText) } +} + +/** + * Converts UI builder rules to a v2 `TablePredicate`. Rules within an `or` + * boundary form an `all` group; multiple groups are combined under `any`. + * Mirrors {@link filterRulesToFilter} but emits the bare-operator grammar. + */ +export function filterRulesToPredicate( + rules: FilterRule[], + columns: ColumnDefinition[] = [] +): TablePredicate | null { + // Tolerate a non-array (the builder value can arrive malformed from an agent + // that doesn't speak the rule shape) instead of throwing "rules is not iterable". + if (!Array.isArray(rules) || rules.length === 0) return null + + const groups: Predicate[][] = [] + let current: Predicate[] = [] + + for (const rule of rules) { + if (rule.logicalOperator === 'or' && current.length > 0) { + groups.push(current) + current = [] + } + if (!rule.column) { + // A predicate-shaped member ({field, op, value}) means the caller mixed the + // two grammars — most likely an agent writing predicate leaves into the + // visual builder. Silently skipping it would DROP that condition and widen + // the result, which on a bulk delete/update is destructive. + if (isRecordLike(rule) && ('field' in rule || 'op' in rule)) { + throw new TableQueryValidationError( + 'Filter looks like a predicate condition but was supplied as a builder rule. ' + + 'Provide the whole filter as a predicate object ({ all | any: [...] }) instead of mixing shapes.', + 'INVALID_FILTER' + ) + } + // A genuinely blank builder row (no column picked yet) contributes nothing. + continue + } + // A select value is an opaque option id — never scalar-coerce it (an id + // that happens to look numeric would silently become a number and match + // nothing). Same rule as filterRulesToFilter. + const isSelect = columns.find((c) => columnMatchesRef(c, rule.column))?.type === 'select' + current.push(ruleToPredicate(rule, isSelect)) + } + if (current.length > 0) groups.push(current) + + if (groups.length === 0) return null + if (groups.length === 1) return { all: groups[0] } + return { any: groups.map((group) => ({ all: group })) } +} + +function predicateLeafToRule(p: Predicate): FilterRule { + return { + id: generateShortId(), + logicalOperator: 'and', + column: p.field, + operator: p.op, + value: VALUELESS_OPS.has(p.op) ? '' : formatValueForBuilder(p.value as JsonValue), + } +} + +/** Flattens a predicate node into builder rules (best-effort for deep nesting). */ +function predicateGroupToRules(node: TablePredicate | Predicate): FilterRule[] { + if ('field' in node) return [predicateLeafToRule(node)] + const members = 'all' in node ? node.all : node.any + return members.flatMap((member) => + 'field' in member ? [predicateLeafToRule(member)] : predicateGroupToRules(member) + ) +} + +/** Converts a v2 `TablePredicate` back to UI builder rules. */ +export function predicateToFilterRules(predicate: TablePredicate | null): FilterRule[] { + if (!predicate) return [] + if ('any' in predicate) { + const groups = predicate.any + .map((node) => predicateGroupToRules(node)) + .filter((g) => g.length > 0) + return applyLogicalOperators(groups) + } + return predicateGroupToRules(predicate) +} + +/** Converts UI sort rules to a v2 `SortSpec` (ordered `{ field, direction }`). */ +export function sortRulesToSortSpec(rules: SortRule[]): SortSpec | null { + const spec: SortSpec = [] + for (const rule of rules) { + if (rule.column) spec.push({ field: rule.column, direction: rule.direction }) + } + return spec.length > 0 ? spec : null +} + +function predicateLeafToFilterValue(p: Predicate): Filter[string] { + if (p.op === 'isEmpty') return { $empty: true } + if (p.op === 'isNotEmpty') return { $empty: false } + // Valueless null checks carry a dummy `true` so the key survives JSON transport. + if (p.op === 'isNull') return { $isNull: true } as Filter[string] + if (p.op === 'isNotNull') return { $isNotNull: true } as Filter[string] + + // Fail loud rather than emit a leaf `buildFilterClause` silently discards. It skips + // an `undefined` condition and any array-valued one, so a dropped leaf WIDENS the + // result — and on the bulk delete/update paths a predicate whose only leaf is + // dropped compiles to no WHERE at all. `op:'eq'` with an array is the realistic + // trigger (an LLM reaching for `in` and writing `eq`). + if (!VALUELESS_OPS.has(p.op) && p.value === undefined) { + throw new TableQueryValidationError( + `Operator "${p.op}" on column "${p.field}" requires a value.`, + 'INVALID_FILTER' + ) + } + if (p.op === 'eq') { + if (Array.isArray(p.value)) { + throw new TableQueryValidationError( + `Operator "eq" on column "${p.field}" does not accept an array — use "in" to match any of several values.`, + 'INVALID_FILTER' + ) + } + return p.value as Filter[string] + } + return { [`$${p.op}`]: p.value } as Filter[string] +} + +/** + * Converts a v2 `TablePredicate` to a legacy `$`-grammar `Filter`. Lossless — + * both compile through the same `fieldPredicate` leaf, so the resulting SQL is + * identical. Lets v2 surfaces author in the predicate grammar while the bulk + * update/delete engine (sync + async-job paths) keeps consuming `Filter`. + */ +export function predicateToFilter(predicate: TablePredicate): Filter { + const nodeToFilter = (node: Predicate | TablePredicate): Filter => { + // A hybrid node (group key AND leaf keys) would convert group-first here, + // silently DROPPING the leaf half — which widens the filter, and on the + // bulk delete/update paths that is destructive. Lossless-or-throw, like + // every other non-representable shape in this converter. + if (('all' in node || 'any' in node) && 'field' in node) { + throw new TableQueryValidationError( + 'A filter node must be either a group ({ all | any: [...] }) or a condition ({ field, op, value }), not both.', + 'INVALID_FILTER' + ) + } + // A node with BOTH group keys would convert `all` and silently drop `any` + // — same widening failure as the hybrid above. Lossless-or-throw. + if ('all' in node && 'any' in node) { + throw new TableQueryValidationError( + 'A filter group must use either "all" or "any", not both — nest one group inside the other instead.', + 'INVALID_FILTER' + ) + } + // Group-first, matching isPredicateGroup/validateNode/buildPredicateNode. A + // leaf-first test would execute a different predicate than the one validated. + if ('all' in node) return { $and: node.all.map(nodeToFilter) } + if ('any' in node) return { $or: node.any.map(nodeToFilter) } + return { [node.field]: predicateLeafToFilterValue(node) } + } + return nodeToFilter(predicate) +} + +/** + * Downgrades a dual-grammar wire filter to the legacy `Filter` the job runners + * and search service still compile. The predicate path throws (via + * `predicateToFilter`) on any leaf the legacy compiler would silently discard, + * so a downgraded filter can never widen. Grammar-only: field keys pass through + * untranslated (session callers already speak column ids). + */ +export function toLegacyFilter(filter: Filter | TablePredicate | undefined): Filter | undefined { + if (!filter) return undefined + if (!isTablePredicate(filter)) return filter + // Shape-validate before converting: the dual-grammar union's legacy branch + // accepts any non-empty object without stripping, so a hybrid or malformed + // tree reaches here Zod-approved. The predicate may be name- OR id-keyed + // (grid vs tools), so only the keying-agnostic checks apply. + validatePredicateShape(filter) + return predicateToFilter(filter) +} + +/** Downgrades a dual-grammar wire sort (ordered spec array or legacy record). */ +export function toLegacySort(sort: Sort | SortSpec | undefined): Sort | undefined { + if (!sort) return undefined + if (!Array.isArray(sort)) return sort + return sort.length > 0 ? Object.fromEntries(sort.map((s) => [s.field, s.direction])) : undefined +} diff --git a/apps/sim/lib/table/query-builder/use-query-builder.ts b/apps/sim/lib/table/query-builder/use-query-builder.ts index 623ec4cd8fd..2d86ae236fd 100644 --- a/apps/sim/lib/table/query-builder/use-query-builder.ts +++ b/apps/sim/lib/table/query-builder/use-query-builder.ts @@ -8,7 +8,7 @@ import { COMPARISON_OPERATORS, type FilterRule, LOGICAL_OPERATORS, - SORT_DIRECTIONS, + SORT_DIRECTION_OPTIONS, type SortRule, } from '@/lib/table/query-builder/constants' import type { ColumnOption } from '@/lib/table/types' @@ -23,7 +23,7 @@ const logicalOptions: ColumnOption[] = LOGICAL_OPERATORS.map((op) => ({ label: op.label, })) -const sortDirectionOptions: ColumnOption[] = SORT_DIRECTIONS.map((d) => ({ +const sortDirectionOptions: ColumnOption[] = SORT_DIRECTION_OPTIONS.map((d) => ({ value: d.value, label: d.label, })) diff --git a/apps/sim/lib/table/query-builder/validate.ts b/apps/sim/lib/table/query-builder/validate.ts new file mode 100644 index 00000000000..f508cd56b57 --- /dev/null +++ b/apps/sim/lib/table/query-builder/validate.ts @@ -0,0 +1,225 @@ +import { isRecordLike } from '@sim/utils/object' +import { getColumnId } from '@/lib/table/column-keys' +import { NAME_PATTERN } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' +import type { + ColumnDefinition, + ColumnType, + FilterOp, + Predicate, + PredicateNode, + SortSpec, + TablePredicate, +} from '@/lib/table/types' + +/** + * Schema-aware validation for the typed predicate/sort wire. The engine + * (`buildPredicateClause`) trusts its input, so this is the boundary gate that + * every caller-supplied filter/sort passes through — the same checks the old + * parser used to do inline, now grammar-agnostic and applied to the object + * directly (predicates are column-NAME-keyed at the boundary, translated to ids + * afterwards). + */ + +/** Equality/containment ops that are meaningless on a `json` column (they never match). */ +const CONTAINMENT_OPS = new Set(['eq', 'ne', 'in', 'nin']) + +/** Ops that legitimately carry no `value`. */ +const VALUELESS_OPS = new Set(['isEmpty', 'isNotEmpty', 'isNull', 'isNotNull']) + +/** + * Cap on `in`/`nin` list length. Each element becomes its own containment clause, + * so an unbounded list is a cheap memory/CPU amplifier from a small request body. + */ +const MAX_IN_LIST_SIZE = 1000 + +/** + * Row-level system columns are filterable/sortable but are not in + * `schema.columns`. Must stay in sync with `SYSTEM_COLUMNS` in `lib/table/sql.ts` + * — a name here with no SQL dispatch there compiles to a `data->>` extraction + * that silently matches nothing. + */ +const SYSTEM_COLUMN_TYPES: ReadonlyArray<[string, ColumnType]> = [ + ['createdAt', 'date'], + ['updatedAt', 'date'], + ['id', 'string'], +] + +function buildTypeByName(columns: ColumnDefinition[]): Map { + const typeByName = new Map(columns.map((c) => [c.name, c.type])) + for (const [name, type] of SYSTEM_COLUMN_TYPES) typeByName.set(name, type) + return typeByName +} + +function validateFieldName(field: string): void { + if (!NAME_PATTERN.test(field)) { + throw new TableQueryValidationError( + `Invalid filter column "${field}". Use a column name (letters, digits, underscore).`, + 'INVALID_FILTER' + ) + } +} + +function validateLeaf(leaf: Predicate, typeByName: Map | null): void { + validateFieldName(leaf.field) + if (typeByName && !typeByName.has(leaf.field)) { + throw new TableQueryValidationError( + `Unknown filter column "${leaf.field}". It is not a column on this table.`, + 'INVALID_FILTER' + ) + } + if (typeByName?.get(leaf.field) === 'json' && CONTAINMENT_OPS.has(leaf.op)) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" is not supported on json column "${leaf.field}" — use like/ilike for text match, or isNull/isNotNull.`, + 'INVALID_FILTER' + ) + } + if (leaf.op === 'in' || leaf.op === 'nin') { + if (!Array.isArray(leaf.value) || leaf.value.length === 0) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" requires a non-empty array value.`, + 'INVALID_FILTER' + ) + } + if (leaf.value.length > MAX_IN_LIST_SIZE) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" accepts at most ${MAX_IN_LIST_SIZE} values, got ${leaf.value.length}.`, + 'INVALID_FILTER' + ) + } + return + } + // A value-taking op with no value, or a scalar op handed an array, compiles to a + // clause the legacy `$`-grammar silently discards — which WIDENS a bulk delete or + // update. Reject here so the copilot path (no Zod) fails the same way the HTTP + // boundary does. + if (!VALUELESS_OPS.has(leaf.op) && leaf.value === undefined) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" requires a value.`, + 'INVALID_FILTER' + ) + } + if (Array.isArray(leaf.value)) { + throw new TableQueryValidationError( + `Operator "${leaf.op}" on column "${leaf.field}" does not accept an array — use "in" to match any of several values.`, + 'INVALID_FILTER' + ) + } +} + +/** + * Structure-only validation: hybrid nodes, group shapes, leaf value rules — + * everything that does not require knowing the table's columns. Used by the + * dual-grammar boundaries where the predicate may be NAME- or ID-keyed, so a + * column-existence check against either keying would be wrong. + */ +export function validatePredicateShape(predicate: TablePredicate): void { + validateNode(predicate, null) +} + +function validateNode(node: PredicateNode, typeByName: Map | null): void { + // Guard before the `in` checks below: an untrusted caller (copilot args, a raw + // block value) can hand us a string/number/null, where `'all' in node` throws + // a raw TypeError. Fail with a clean, actionable message instead. + if (typeof node !== 'object' || node === null) { + throw new TableQueryValidationError( + 'Filter must be a predicate object ({ all | any: [...] }).', + 'INVALID_FILTER' + ) + } + // A node carrying BOTH a group key and `field` is ambiguous: the engine and this + // validator read it group-first while `predicateToFilter`/`predicateNamesToIds` + // read it leaf-first, so the gate would validate one predicate and the bulk-write + // path would execute a different one. Reject rather than pick a winner. + if (('all' in node || 'any' in node) && 'field' in node) { + throw new TableQueryValidationError( + 'A filter node must be either a group ({ all | any: [...] }) or a condition ({ field, op, value }), not both.', + 'INVALID_FILTER' + ) + } + // Same ambiguity with BOTH group keys: every group-first traversal + // (`predicateNamesToIds`, `predicateToFilter`, `buildPredicateNode`) reads + // `all` and silently DROPS `any`, so half the caller's conditions vanish — + // on a bulk delete/update that widens the matched set. Reject rather than + // pick a winner; nesting expresses the intent unambiguously. + if ('all' in node && 'any' in node) { + throw new TableQueryValidationError( + 'A filter group must use either "all" or "any", not both — nest one group inside the other instead.', + 'INVALID_FILTER' + ) + } + if ('all' in node || 'any' in node) { + const members = 'all' in node ? node.all : node.any + if (!Array.isArray(members)) { + throw new TableQueryValidationError( + 'A filter group ({ all | any }) must be an array of conditions.', + 'INVALID_FILTER' + ) + } + // Mirrors the Zod contract's .min(1). An empty group slips the strict union + // (falling to the non-empty-OBJECT legacy branch) and compiles to no WHERE + // clause — which on the run/cancel/delete scopes silently means EVERY row. + if (members.length === 0) { + throw new TableQueryValidationError( + 'A filter group must contain at least one condition.', + 'INVALID_FILTER' + ) + } + for (const child of members) validateNode(child, typeByName) + return + } + // Neither a group nor a leaf. Overwhelmingly this is the legacy `$`-grammar + // (`{ status: { $eq: 'x' } }`) — a shape `validateLeaf` would reject as + // `Unknown filter column "undefined"`, which tells an LLM caller nothing and + // sends it retrying column names forever. Name the actual mistake. + if (!('field' in node)) { + const keys = Object.keys(node) + const looksLegacy = keys.some( + (k) => k.startsWith('$') || isRecordLike((node as Record)[k]) + ) + throw new TableQueryValidationError( + looksLegacy + ? 'Filter uses the legacy operator-object grammar. Use a predicate tree instead: { all: [{ field, op, value }] } (or "any" for OR), with bare operators like eq/gte/contains/in.' + : 'A filter node must be a group ({ all | any: [...] }) or a condition ({ field, op, value }).', + 'INVALID_FILTER' + ) + } + validateLeaf(node as Predicate, typeByName) +} + +/** + * Validates a name-keyed predicate against the table schema: every leaf field + * exists, no equality/containment op targets a `json` column, `in`/`nin` carry a + * non-empty array. Throws {@link TableQueryValidationError} (`INVALID_FILTER`). + */ +export function validatePredicate(predicate: TablePredicate, columns: ColumnDefinition[]): void { + validateNode(predicate, buildTypeByName(columns)) +} + +/** Validates a name-keyed sort spec: every field is a real or system column. */ +export function validateSortSpec(spec: SortSpec, columns: ColumnDefinition[]): void { + const typeByName = buildTypeByName(columns) + for (const { field } of spec) { + validateFieldName(field) + if (!typeByName.has(field)) { + throw new TableQueryValidationError(`Unknown sort column "${field}"`, 'INVALID_ORDER') + } + } +} + +/** + * Validates a STORAGE-keyed predicate — leaf fields are column ids (plus the + * system columns, which keep their names). Runs AFTER wire translation, which + * makes it keying-correct for every caller: a session caller's ids are already + * storage keys, and a workflow tool's names have just been translated — so any + * field left unresolved here is a typo, and on the bulk write paths a typo must + * 400 rather than compile to a filter that silently matches nothing. + */ +export function validateStoragePredicate( + predicate: TablePredicate, + columns: ColumnDefinition[] +): void { + const typeById = new Map(columns.map((c) => [getColumnId(c), c.type])) + for (const [name, type] of SYSTEM_COLUMN_TYPES) typeById.set(name, type) + validateNode(predicate, typeById) +} diff --git a/apps/sim/lib/table/rows/__tests__/cursor.test.ts b/apps/sim/lib/table/rows/__tests__/cursor.test.ts new file mode 100644 index 00000000000..baf9a3434f6 --- /dev/null +++ b/apps/sim/lib/table/rows/__tests__/cursor.test.ts @@ -0,0 +1,96 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { decodeCursor, encodeCursor } from '@/lib/table/rows/cursor' + +describe('cursor codec', () => { + it('encodes a keyset cursor when keyset is valid and round-trips it', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: 'a0' }, + keysetValid: true, + nextOffset: 100, + }) + expect(typeof token).toBe('string') + expect(decodeCursor(token)).toEqual({ after: { orderKey: 'a0', id: 'row-9' } }) + }) + + it('falls back to an offset cursor when keyset is not valid (custom sort / flag off)', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: 'a0' }, + keysetValid: false, + nextOffset: 100, + }) + expect(decodeCursor(token)).toEqual({ offset: 100 }) + }) + + it('emits a compound cursor when the last row is unkeyed but an anchor is known', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: undefined }, + keysetValid: true, + nextOffset: 50, + seekBase: { anchor: { orderKey: 'a5', id: 'row-5' }, offsetFromAnchor: 4 }, + }) + expect(decodeCursor(token)).toEqual({ after: { orderKey: 'a5', id: 'row-5' }, offset: 4 }) + }) + + it('falls back to a whole-view offset when the row has no order key and no anchor', () => { + const token = encodeCursor({ + lastRow: { id: 'row-9', orderKey: undefined }, + keysetValid: true, + nextOffset: 50, + }) + expect(decodeCursor(token)).toEqual({ offset: 50 }) + }) + + it('produces opaque base64url with no raw orderKey/offset leaking', () => { + const token = encodeCursor({ + lastRow: { id: 'r', orderKey: 'zz' }, + keysetValid: true, + nextOffset: 0, + }) + expect(token).not.toContain('orderKey') + expect(token).not.toContain('{') + expect(token).toMatch(/^[A-Za-z0-9_-]+$/) + }) + + it('stamps a version field and rejects any other version', () => { + const token = encodeCursor({ + lastRow: { id: 'r', orderKey: 'a' }, + keysetValid: true, + nextOffset: 0, + }) + expect(JSON.parse(Buffer.from(token, 'base64url').toString('utf8')).v).toBe(1) + // A future/unknown version fails cleanly instead of being misread. + expect(() => decodeCursor(Buffer.from('{"v":2,"o":0}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + // A pre-version token (no `v`) is no longer accepted. + expect(() => decodeCursor(Buffer.from('{"o":0}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) + + it('throws on a malformed cursor', () => { + expect(() => decodeCursor('not-base64-$$$')).toThrow('Invalid cursor') + // Valid base64url but wrong shape. + expect(() => decodeCursor(Buffer.from('{"v":1,"x":1}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) + + it('throws on valid JSON that is not an object (no raw TypeError leak)', () => { + for (const body of ['42', 'null', '"hi"', 'true', '[1,2]']) { + expect(() => decodeCursor(Buffer.from(body).toString('base64url'))).toThrow('Invalid cursor') + } + }) + + it('rejects negative and non-integer offsets', () => { + expect(() => decodeCursor(Buffer.from('{"v":1,"o":-1}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + expect(() => decodeCursor(Buffer.from('{"v":1,"o":1.5}').toString('base64url'))).toThrow( + 'Invalid cursor' + ) + }) +}) diff --git a/apps/sim/lib/table/rows/cursor.test.ts b/apps/sim/lib/table/rows/cursor.test.ts new file mode 100644 index 00000000000..bae39a35c66 --- /dev/null +++ b/apps/sim/lib/table/rows/cursor.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + * + * Opaque cursor encode/decode and the cursor↔sort binding. A cursor encodes a + * position in one specific ordering; replaying it under any other ordering + * silently pages the wrong sequence, so binding violations must throw + * CURSOR_SORT_CONFLICT rather than return wrong rows. + */ +import { describe, expect, it } from 'vitest' +import { TableQueryValidationError } from '@/lib/table/errors' +import { + assertCursorSortBinding, + canonicalSortKey, + decodeCursor, + encodeCursor, +} from '@/lib/table/rows/cursor' + +const ROW = { id: 'row_1', orderKey: 'a1' } + +describe('cursor↔sort binding (bugbot round 2)', () => { + it('stamps an offset cursor with the sort it was minted under', () => { + const token = encodeCursor({ + lastRow: { id: 'row_1', orderKey: null }, + keysetValid: false, + nextOffset: 100, + sort: { col_a: 'desc' }, + }) + const decoded = decodeCursor(token) + expect(decoded.offset).toBe(100) + expect(decoded.sortKey).toBe(canonicalSortKey({ col_a: 'desc' })) + }) + + it('accepts replay under the identical sort', () => { + const decoded = { offset: 100, sortKey: canonicalSortKey({ col_a: 'desc' }) } + expect(() => assertCursorSortBinding(decoded, { col_a: 'desc' })).not.toThrow() + }) + + it('rejects replay under a DIFFERENT sort', () => { + const decoded = { offset: 100, sortKey: canonicalSortKey({ col_a: 'desc' }) } + for (const sort of [{ col_a: 'asc' as const }, { col_b: 'desc' as const }, undefined]) { + expect(() => assertCursorSortBinding(decoded, sort)).toThrow(TableQueryValidationError) + try { + assertCursorSortBinding(decoded, sort) + } catch (e) { + expect((e as TableQueryValidationError).code).toBe('CURSOR_SORT_CONFLICT') + } + } + }) + + it('rejects adding a sort to an unsorted offset cursor', () => { + const token = encodeCursor({ + lastRow: { id: 'row_1', orderKey: null }, + keysetValid: false, + nextOffset: 50, + }) + const decoded = decodeCursor(token) + expect(decoded.sortKey).toBeUndefined() + expect(() => assertCursorSortBinding(decoded, { col_a: 'asc' })).toThrow( + /different sort|sorted query/ + ) + expect(() => assertCursorSortBinding(decoded, undefined)).not.toThrow() + }) + + it('keyset cursors stay default-order only and never carry a sort stamp', () => { + const token = encodeCursor({ lastRow: ROW, keysetValid: true, nextOffset: 10 }) + const decoded = decodeCursor(token) + expect(decoded.after).toEqual({ orderKey: 'a1', id: 'row_1' }) + expect(decoded.sortKey).toBeUndefined() + expect(() => assertCursorSortBinding(decoded, { col_a: 'asc' })).toThrow(/sorted query/) + expect(() => assertCursorSortBinding(decoded, undefined)).not.toThrow() + }) + + it('sort key order is significant (priority is part of the identity)', () => { + expect(canonicalSortKey({ a: 'asc', b: 'desc' })).not.toBe( + canonicalSortKey({ b: 'desc', a: 'asc' }) + ) + }) +}) diff --git a/apps/sim/lib/table/rows/cursor.ts b/apps/sim/lib/table/rows/cursor.ts new file mode 100644 index 00000000000..0d2ab4ab206 --- /dev/null +++ b/apps/sim/lib/table/rows/cursor.ts @@ -0,0 +1,171 @@ +/** + * Opaque pagination cursor for the v2 table-query surface. + * + * The token is a base64url-encoded JSON payload that hides whether paging is + * keyset- or offset-based, so callers (the v2 tools, the agent) only ever echo + * an opaque `cursor` and never juggle `orderKey`/`id`/`offset` themselves. + * + * - Default order → keyset on `(order_key, id)` (`{ k, i }`), an index seek. + * - Sorted views → whole-view offset (`{ o }`), because `(order_key, id)` + * keyset can't seek a data-column ordering. + * - Keyset page whose last row lacks an `orderKey` (rows predating the backfill, + * or forked rows that inherited a NULL key) → compound (`{ k, i, o }`): seek to + * the last keyed anchor, then OFFSET past the unkeyed rows consumed after it. + * This only resolves correctly because the seek admits `order_key IS NULL` + * rows; a bare `(order_key, id) > (…)` excludes them and strands the tail. + */ + +import { TableQueryValidationError } from '@/lib/table/errors' +import type { Sort, TableRow, TableRowsCursor } from '@/lib/table/types' + +/** + * Cursor payload version. Every encoded token carries `v`; decode rejects any + * other value so a future shape change (new `v`) fails cleanly instead of being + * misread against the current field set. + */ +const CURSOR_VERSION = 1 + +type CursorBody = { k: string; i: string } | { o: number } | { k: string; i: string; o: number } +type SortBinding = { s?: string } +type CursorPayload = CursorBody & SortBinding & { v: number } + +/** + * Canonical fingerprint of a sort for cursor binding. Entry order is the sort + * priority (built from the ordered spec upstream), so stringifying entries is + * deterministic for equal sorts and distinct for different ones. + */ +export function canonicalSortKey(sort: Sort | null | undefined): string | undefined { + if (!sort) return undefined + const entries = Object.entries(sort) + return entries.length > 0 ? JSON.stringify(entries) : undefined +} + +/** + * A cursor is only valid for the exact query shape it was minted under: + * keyset/compound cursors encode a position in the DEFAULT `(order_key, id)` + * order, and an offset cursor from a sorted view encodes a position in THAT + * sort. Replaying either against a different ordering silently pages the wrong + * sequence — rows skipped or duplicated with no error. Throws + * `CURSOR_SORT_CONFLICT` so callers restart paging without the cursor. + */ +export function assertCursorSortBinding( + decoded: { after?: TableRowsCursor; offset?: number; sortKey?: string }, + sort: Sort | null | undefined +): void { + const requested = canonicalSortKey(sort) + if (decoded.after && requested !== undefined) { + throw new TableQueryValidationError( + 'Cursor is not valid for a sorted query. Restart paging without the cursor.', + 'CURSOR_SORT_CONFLICT' + ) + } + if ( + decoded.after === undefined && + decoded.offset !== undefined && + decoded.sortKey !== requested + ) { + throw new TableQueryValidationError( + 'Cursor was created under a different sort. Restart paging without the cursor.', + 'CURSOR_SORT_CONFLICT' + ) + } +} + +function invalidCursor(): never { + throw new TableQueryValidationError('Invalid cursor', 'INVALID_CURSOR') +} + +function toBase64Url(json: string): string { + return Buffer.from(json, 'utf8').toString('base64url') +} + +function fromBase64Url(token: string): string { + return Buffer.from(token, 'base64url').toString('utf8') +} + +/** + * Builds the cursor for the page *after* `lastRow`. + * + * Shape selection: + * 1. `keysetValid` and the row carries an `orderKey` → `{ k, i }`. + * 2. `keysetValid` with a known prior anchor (last row unkeyed) → `{ k, i, o }`. + * 3. Otherwise → `{ o: nextOffset }` (whole-view offset). + * + * `keysetValid` must only be true when the `(order_key, id)` index order is + * authoritative for the page: no custom sort AND fractional ordering enabled. + * Passing false forces the offset shape, which is correct under any ordering. + */ +export function encodeCursor(args: { + lastRow: Pick + keysetValid: boolean + nextOffset: number + seekBase?: { anchor: TableRowsCursor; offsetFromAnchor: number } + /** The sort the page was produced under — stamps offset cursors so they can't be replayed against a different ordering. */ + sort?: Sort | null +}): string { + let body: CursorBody + if (args.keysetValid && args.lastRow.orderKey) { + body = { k: args.lastRow.orderKey, i: args.lastRow.id } + } else if (args.seekBase) { + // An anchor is in effect (inbound seek or last keyed row) but a plain + // keyset can't stand alone — resume by seeking the anchor then offsetting + // past the rows consumed after it. Never valid under a custom sort, where + // callers must not pass a seekBase. + body = { + k: args.seekBase.anchor.orderKey, + i: args.seekBase.anchor.id, + o: args.seekBase.offsetFromAnchor, + } + } else { + body = { o: args.nextOffset } + } + const sortKey = canonicalSortKey(args.sort) + const payload: CursorPayload = { + ...body, + // Only the pure-offset shape can exist under a custom sort; keyset and + // compound shapes are default-order by construction and carry no binding. + ...('k' in body || sortKey === undefined ? {} : { s: sortKey }), + v: CURSOR_VERSION, + } + return toBase64Url(JSON.stringify(payload)) +} + +/** Decodes an opaque cursor into the `queryRows` paging inputs it stands for. */ +export function decodeCursor(token: string): { + after?: TableRowsCursor + offset?: number + /** Sort fingerprint an offset cursor was minted under; absent = default order. */ + sortKey?: string +} { + let payload: unknown + try { + payload = JSON.parse(fromBase64Url(token)) + } catch { + invalidCursor() + } + if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) { + invalidCursor() + } + + const record = payload as Record + if (record.v !== CURSOR_VERSION) invalidCursor() + const hasKeyset = typeof record.k === 'string' && typeof record.i === 'string' + const hasOffset = typeof record.o === 'number' && Number.isInteger(record.o) && record.o >= 0 + + if (hasKeyset && hasOffset) { + return { + after: { orderKey: record.k as string, id: record.i as string }, + offset: record.o as number, + } + } + if (hasKeyset) { + return { after: { orderKey: record.k as string, id: record.i as string } } + } + if (hasOffset) { + return { + offset: record.o as number, + ...(typeof record.s === 'string' ? { sortKey: record.s } : {}), + } + } + invalidCursor() +} diff --git a/apps/sim/lib/table/rows/paging.ts b/apps/sim/lib/table/rows/paging.ts deleted file mode 100644 index a96b98c041f..00000000000 --- a/apps/sim/lib/table/rows/paging.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Cuts a fetched page to a byte budget: keeps the longest prefix of rows whose - * serialized `data` fits within `maxBytes`, always keeping at least one row so - * pagination makes forward progress even when a single row exceeds the budget. - * - * The budget counts `data` only — the per-row envelope (`id`, `position`, - * `orderKey`, timestamps, executions) is not measured, so actual response - * payloads run slightly over `maxBytes`. Callers must leave headroom; the - * production SQL-side cut should account for the same overhead. - */ -export function trimRowsToByteBudget( - rows: T[], - maxBytes: number -): T[] { - let total = 0 - let kept = 0 - for (const row of rows) { - total += Buffer.byteLength(JSON.stringify(row.data)) - if (kept > 0 && total > maxBytes) break - kept++ - } - return kept === rows.length ? rows : rows.slice(0, kept) -} diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index 41eedd066f8..dfcc3a4c282 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -25,6 +25,7 @@ import { } from '@/lib/table/billing' import { getColumnId } from '@/lib/table/column-keys' import { getMaxPageBytes, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' import { assertRowDelete, assertRowInsert, @@ -32,7 +33,13 @@ import { patchColumnIds, } from '@/lib/table/mutation-locks' import { nKeysBetween } from '@/lib/table/order-key' -import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner' +import { + type DbExecutor, + type DbTransaction, + withReadGuards, + withSeqscanOff, +} from '@/lib/table/planner' +import { encodeCursor } from '@/lib/table/rows/cursor' import { applyExecutionsPatch, deriveExecClearsForDataPatch, @@ -49,8 +56,13 @@ import { resolveBatchInsertOrderKeys, resolveInsertOrderKey, } from '@/lib/table/rows/ordering' -import { trimRowsToByteBudget } from '@/lib/table/rows/paging' -import { buildFilterClause, buildSortClause, escapeLikePattern } from '@/lib/table/sql' +import { + buildFilterClause, + buildPredicateClause, + buildSortClause, + escapeLikePattern, + fieldPredicate, +} from '@/lib/table/sql' import { fireTableTrigger } from '@/lib/table/trigger' import { scaledStatementTimeoutMs, setTableTxTimeouts } from '@/lib/table/tx' import type { @@ -75,6 +87,7 @@ import type { TableDefinition, TableDeleteJobPayload, TableRow, + TableRowsCursor, UpdateRowData, UpsertResult, UpsertRowData, @@ -421,15 +434,19 @@ export async function replaceTableRowsWithTx( if (uniqueColumns.length > 0 && data.rows.length > 0) { const seen = new Map>() for (const col of uniqueColumns) { - seen.set(col.name, new Map()) + seen.set(getColumnId(col), new Map()) } for (let i = 0; i < data.rows.length; i++) { const row = data.rows[i] for (const col of uniqueColumns) { - const value = row[col.name] + // Coerced rows are keyed by column id, not display name — reading + // `row[col.name]` silently misses renamed columns and lets dupes through. + const colId = getColumnId(col) + const value = row[colId] if (value === null || value === undefined) continue - const normalized = typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value) - const map = seen.get(col.name)! + // Case-sensitive, consistent with the unique-constraint check leaf. + const normalized = typeof value === 'string' ? value : JSON.stringify(value) + const map = seen.get(colId)! if (map.has(normalized)) { throw new Error( `Row ${i + 1}: Column "${col.name}" must be unique. Value "${String(value)}" duplicates row ${map.get(normalized)! + 1} in batch` @@ -568,12 +585,21 @@ export async function upsertRow( throw new Error(`Upsert requires a value for the conflict target column "${targetColumnName}"`) } - // `data->` and `data->>` accept the JSON key as a parameterized text value; - // no need for `sql.raw` interpolation. - const matchFilter = - typeof targetValue === 'string' - ? sql`${userTableRows.data}->>${targetColumnKey}::text = ${String(targetValue)}` - : sql`(${userTableRows.data}->${targetColumnKey}::text)::jsonb = ${JSON.stringify(targetValue)}::jsonb` + // Build the conflict probe through the SAME leaf as the unique-constraint check + // (`fieldPredicate` → case-sensitive JSONB containment). This is what makes + // "find the row to update" and "is this value unique" agree: a value differing + // only in case can no longer slip past the probe and then trip the guard. + // `eq` always yields a clause for a non-null value (guaranteed above). + const matchFilter = fieldPredicate( + USER_TABLE_ROWS_SQL_NAME, + targetColumnKey, + 'eq', + targetValue, + table.schema.columns.find((c) => getColumnId(c) === targetColumnKey) + ) + if (!matchFilter) { + throw new Error('Failed to build upsert conflict predicate') + } // Resolve the plan limit BEFORE the tx (the lookup is a separate pool read; doing // it inside the tx would hold a connection + the row-order lock during it). The @@ -1005,8 +1031,11 @@ export async function queryRows( ): Promise { const { filter, + predicate, sort, - limit = TABLE_LIMITS.DEFAULT_QUERY_LIMIT, + // No default: an undefined limit returns every matching row, bounded only by + // the MAX_QUERY_RESULT_BYTES fail-fast guard below. + limit, offset = 0, after, includeTotal = true, @@ -1026,61 +1055,61 @@ export async function queryRows( deleteMask ) + // v2 predicate takes precedence over the legacy `$`-filter; both compile to a + // WHERE through the same `fieldPredicate` leaf. + const userClause = predicate + ? buildPredicateClause(predicate, tableName, columns) + : filter && Object.keys(filter).length > 0 + ? buildFilterClause(filter, tableName, columns) + : undefined + let whereClause = baseConditions - if (filter && Object.keys(filter).length > 0) { - const filterClause = buildFilterClause(filter, tableName, columns) - if (filterClause) { - whereClause = and(baseConditions, filterClause) - } + if (userClause) { + whereClause = and(baseConditions, userClause) } - // Keyset page: seek past the cursor on the default `(order_key, id)` order instead of paying - // OFFSET's scan-and-discard of every prior row (O(N²) across a deep scroll / full drain). Only - // valid without a custom sort — the contract rejects `after` + `sort` together. The count below - // deliberately excludes the cursor: totals cover the whole view, not the remaining pages. - const pageWhere = - after && !sort - ? and( - whereClause, - sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})` - ) - : whereClause + // Keyset seeks are only authoritative when the page order IS the + // `(order_key, id)` index order: no custom sort. Order keys are NOT guaranteed + // present — unkeyed rows sort last and the seek admits them explicitly. + const keysetValid = !sort - const buildPageQuery = (executor: DbExecutor) => { - const query = executor - .select() - .from(userTableRows) - .where(pageWhere ?? baseConditions) - .orderBy(buildRowOrderBySql(sort, tableName, columns)) - return after ? query.limit(limit) : query.limit(limit).offset(offset) - } - - // Count and page fetch are independent reads — run them concurrently so the + // Count and page drain are independent reads — run them concurrently so the // `includeTotal` hot path doesn't pay two serial round-trips. Filtered counts // go through the tenant-bounded variant (see countRowsTenantBounded); the // unfiltered count already plans an index-only scan on the table_id prefix. - // Custom column sorts order by `data->>'col'` — unestimatable, so left alone - // the planner seq-scans and sorts the whole shared relation on every page - // (9.7s measured on a 1M-row table; 0.76s tenant-bounded). Default-order - // pages already stream the `(table_id, order_key, id)` index. - const hasFilter = Boolean(filter && Object.keys(filter).length > 0) - const rowsPromise = sort ? withSeqscanOff(async (trx) => buildPageQuery(trx)) : buildPageQuery(db) + // The count uses the full-view WHERE (no cursor seek): totals cover the whole + // view, not the remaining pages. + const hasFilter = Boolean(userClause) const countPromise = includeTotal ? hasFilter ? countRowsTenantBounded(whereClause) - : db - .select({ count: count() }) - .from(userTableRows) - .where(whereClause ?? baseConditions) - .then((r) => Number(r[0].count)) + : // Unfiltered count plans an index-only scan on the table_id prefix, but + // still runs under the read timeout so it can't pin a connection. + withReadGuards(async (trx) => { + const [r] = await trx + .select({ count: count() }) + .from(userTableRows) + .where(whereClause ?? baseConditions) + return Number(r.count) + }) : null - const [fetchedRows, totalCount] = await Promise.all([rowsPromise, countPromise]) + // The inbound seek is honored whenever there's no custom sort; `keysetValid` + // only gates re-anchoring and plain-keyset cursor emission. + const drainPromise = fetchRowsBounded({ + baseWhere: whereClause ?? baseConditions, + orderBy: buildRowOrderBySql(sort, tableName, columns), + sorted: Boolean(sort), + keysetValid, + seek: !sort ? after : undefined, + startOffset: offset, + limit, + budgetBytes: TABLE_LIMITS.MAX_QUERY_RESULT_BYTES, + pageCutBytes: getMaxPageBytes() ?? undefined, + }) - // Dev-preview byte cut (TABLE_MAX_PAGE_BYTES, off by default): clients terminate on - // empty page / totalCount, never page fullness, so a short page is safe to return. - const maxPageBytes = getMaxPageBytes() - const rows = maxPageBytes === null ? fetchedRows : trimRowsToByteBudget(fetchedRows, maxPageBytes) + const [fetched, totalCount] = await Promise.all([drainPromise, countPromise]) + const rows = fetched.rows const executionsByRow = withExecutions ? await loadExecutionsByRow( @@ -1090,23 +1119,218 @@ export async function queryRows( : null logger.info( - `[${requestId}] Queried ${rows.length} rows from table ${table.id} (total: ${totalCount})` + `[${requestId}] Queried ${rows.length} rows from table ${table.id} (total: ${totalCount}, bytes: ${fetched.bytes}, more: ${fetched.hasMore})` ) + const mappedRows = rows.map((r) => ({ + id: r.id, + data: r.data as RowData, + executions: executionsByRow?.get(r.id) ?? {}, + position: r.position, + orderKey: r.orderKey ?? undefined, + createdAt: r.createdAt, + updatedAt: r.updatedAt, + })) + + // Opaque next-page cursor: non-null whenever more matching rows exist beyond + // this page — whether the page was cut by `limit` or by the byte budget. The + // drain loop proves `hasMore` with a fetched-but-unreturned witness row. + const lastRow = mappedRows[mappedRows.length - 1] + const nextCursor = + fetched.hasMore && lastRow + ? encodeCursor({ + lastRow, + keysetValid, + nextOffset: offset + mappedRows.length, + seekBase: fetched.anchor + ? { anchor: fetched.anchor, offsetFromAnchor: fetched.anchorOffset } + : undefined, + sort, + }) + : null + return { - rows: rows.map((r) => ({ - id: r.id, - data: r.data as RowData, - executions: executionsByRow?.get(r.id) ?? {}, - position: r.position, - orderKey: r.orderKey ?? undefined, - createdAt: r.createdAt, - updatedAt: r.updatedAt, - })), + rows: mappedRows, rowCount: rows.length, totalCount, - limit, + limit: limit ?? mappedRows.length, offset, + nextCursor, + } +} + +interface BoundedFetchParams { + /** Tenant + delete-mask + user filter — WITHOUT any seek predicate. */ + baseWhere: SQL | undefined + orderBy: SQL + /** Custom sort present → per-batch withSeqscanOff (JSONB order is unestimatable). */ + sorted: boolean + /** `(order_key, id)` order is authoritative → keyset re-anchoring + seeks. */ + keysetValid: boolean + /** Inbound seek anchor (decoded keyset/compound cursor), if any. */ + seek?: TableRowsCursor + /** Inbound offset — the whole-view offset, or the past-anchor offset of a compound cursor. */ + startOffset: number + limit?: number + /** Drain ceiling: sizes batches, and the fail-fast bound for an unbounded query. */ + budgetBytes: number + /** + * Opt-in byte cut for a **bounded** page (`TABLE_MAX_PAGE_BYTES`); `undefined` + * disables it, so a bounded page always returns its full `limit`. + */ + pageCutBytes?: number +} + +interface BoundedFetchResult { + rows: Array + bytes: number + /** Proven by a fetched-but-unreturned witness row — never inferred from page fullness. */ + hasMore: boolean + /** Final keyset anchor, for compound-cursor emission when the last row is unkeyed. */ + anchor?: TableRowsCursor + /** Rows consumed past `anchor` (0 when the anchor is the last returned row). */ + anchorOffset: number +} + +/** Belt-and-braces bound on drain iterations; unreachable in practice. */ +const MAX_QUERY_BATCHES = 1000 + +/** + * Drains rows in adaptively-sized bounded batches until the caller's `limit` + * or the byte ceiling ends the page. Never issues an unbounded SELECT: the + * first batch is capped so its worst-case bytes stay within ~4× the budget at + * the max row size, and later batches are sized from the observed average. + * + * Byte ceiling: an **unbounded** query (no `limit`) always fails fast at + * `budgetBytes` — returning part of a result that promised everything would be + * silent truncation. A **bounded** page cuts short only when `pageCutBytes` is + * set (`TABLE_MAX_PAGE_BYTES`), because a short page is only safe for clients + * that terminate on `nextCursor === null` rather than on page fullness. + * + * Advance strategy: when `keysetValid`, the loop re-anchors on each consumed + * keyed row and seeks `(order_key, id) > (anchor)` — delete-tolerant and an + * index seek. Otherwise (custom sort, flag off) it advances by OFFSET from the + * inbound position; rows deleted mid-drain can skip/duplicate exactly as + * cross-request offset paging already does. + * + * Always returns at least one row when any match exists, even if that row + * alone exceeds the budget. + */ +async function fetchRowsBounded(params: BoundedFetchParams): Promise { + const { baseWhere, orderBy, sorted, keysetValid, limit, budgetBytes, pageCutBytes } = params + + const firstBatchCap = Math.max(1, Math.floor((4 * budgetBytes) / TABLE_LIMITS.MAX_ROW_SIZE_BYTES)) + + // The byte ceiling that ends the drain: an unbounded query fails fast at the + // budget; a bounded page cuts only when the operator opted in. + const cutBytes = limit === undefined ? budgetBytes : pageCutBytes + + const rows: Array = [] + let bytes = 0 + let maxRowBytes = 0 + let hasMore = false + let anchor = params.seek + let anchorOffset = params.startOffset + let consumedSinceAnchor = 0 + + const nextBatchRows = (): number => { + if (rows.length === 0) return Math.min(limit ?? firstBatchCap, firstBatchCap) + const avg = Math.max(1, bytes / rows.length) + // Bytes we may still fetch this batch. When a cut is active it's the + // remainder of that cut; otherwise each batch gets a fresh budget's worth, + // so a large bounded page keeps draining in real steps instead of degrading + // to one row per query once cumulative bytes pass the budget. + const remaining = Math.max(1, cutBytes === undefined ? budgetBytes : cutBytes - bytes) + const byAverage = Math.ceil(remaining / avg) + 1 + const varianceCap = Math.ceil((8 * remaining) / Math.max(maxRowBytes, 1)) + return Math.max(1, Math.min(byAverage, TABLE_LIMITS.QUERY_BATCH_MAX_ROWS, varianceCap)) + } + + const runBatch = (batchSeek: TableRowsCursor | undefined, batchOffset: number, ask: number) => { + const buildQuery = (executor: DbExecutor) => { + // `order_key` is nullable (rows predating the backfill, and forked rows that + // inherit a NULL key). A bare row-constructor comparison evaluates to NULL for + // those rows, so they are dropped by WHERE — and because NULLs sort LAST under + // `ORDER BY order_key, id`, the entire unkeyed tail becomes unreachable and the + // drain terminates early reporting `hasMore: false`. Admitting NULLs keeps the + // seek set exactly "the tail after the anchor", which is also what the compound + // `{k,i,o}` cursor's `offsetFromAnchor` accounting assumes. + const seekWhere = batchSeek + ? and( + baseWhere, + sql`(${userTableRows.orderKey} IS NULL OR (${userTableRows.orderKey}, ${userTableRows.id}) > (${batchSeek.orderKey}, ${batchSeek.id}))` + ) + : baseWhere + const query = executor + .select() + .from(userTableRows) + .where(seekWhere) + .orderBy(orderBy) + .limit(ask) + return batchOffset > 0 ? query.offset(batchOffset) : query + } + // One tx per batch (SET LOCAL dies with it; holding a tx across JS + // accounting between batches would pin a pooled connection). Custom sorts + // order by `data->>'col'` — unestimatable — so they also penalize seq scans + // (9.7s→0.76s on a 1M-row table); default-order pages stream the index and + // just need the read timeout. Either way the batch runs under a statement + // timeout so a pathological filter can't scan unbounded. + return withReadGuards(async (trx) => buildQuery(trx), { seqscanOff: sorted }) + } + + for (let iteration = 0; iteration < MAX_QUERY_BATCHES; iteration++) { + const limitRemaining = limit === undefined ? Number.POSITIVE_INFINITY : limit - rows.length + const target = Math.min(nextBatchRows(), limitRemaining) + const ask = target + 1 // +1 = witness row proving more data exists past a cut + const batch = await runBatch(anchor, anchorOffset + consumedSinceAnchor, ask) + if (batch.length === 0) break + + let cut = false + for (const row of batch) { + const rowBytes = Buffer.byteLength(JSON.stringify(row.data)) + if (cutBytes !== undefined && rows.length > 0 && bytes + rowBytes > cutBytes) { + // Unbounded queries promise the ENTIRE result — a partial page would be + // silent truncation, so fail fast instead (the drain has only fetched + // ~budget bytes at this point, never the whole table). + if (limit === undefined) { + throw new TableQueryValidationError( + `Query result exceeds the ${Math.floor(cutBytes / (1024 * 1024))}MB limit. Add a filter or a limit to narrow the result.`, + 'TABLE_QUERY_RESULT_TOO_LARGE' + ) + } + // Bounded page, byte cut opted in: `row` is the witness. Requires a + // non-empty page so a single over-budget row is still returned alone. + hasMore = true + cut = true + break + } + // Limit cut: `row` is the +1 peek witness. + if (rows.length === limit) { + hasMore = true + cut = true + break + } + rows.push(row) + bytes += rowBytes + consumedSinceAnchor++ + if (rowBytes > maxRowBytes) maxRowBytes = rowBytes + if (keysetValid && row.orderKey) { + anchor = { orderKey: row.orderKey, id: row.id } + anchorOffset = 0 + consumedSinceAnchor = 0 + } + } + if (cut) break + // Short batch = the source is exhausted; hasMore stays false. + if (batch.length < ask) break + } + + return { + rows, + bytes, + hasMore, + anchor, + anchorOffset: anchorOffset + consumedSinceAnchor, } } @@ -1419,17 +1643,22 @@ export async function updateRowsByFilter( eq(userTableRows.workspaceId, table.workspaceId) ) + // A limit selects a SUBSET, so impose the default `(order_key, id)` order — + // without it Postgres returns planner-arbitrary rows and "update the first N" + // is nondeterministic. Sort is irrelevant (and skipped) when every match is updated. // Tenant-bounded: the jsonb filter is unestimatable and otherwise sends the planner to a // whole-shared-relation seq scan (14.4s measured on a 1M-row table). const matchingRows = await withSeqscanOff(async (trx) => { - let query = trx + const base = trx .select({ id: userTableRows.id, data: userTableRows.data }) .from(userTableRows) .where(and(baseConditions, filterClause)) if (data.limit) { - query = query.limit(data.limit) as typeof query + return base + .orderBy(buildRowOrderBySql(undefined, tableName, table.schema.columns)) + .limit(data.limit) } - return query + return base }) if (matchingRows.length === 0) { @@ -1785,16 +2014,20 @@ export async function deleteRowsByFilter( eq(userTableRows.workspaceId, table.workspaceId) ) + // A limit deletes a SUBSET, so order deterministically by `(order_key, id)` — + // see updateRowsByFilter. Unbounded deletes affect every match, so order is moot. // Tenant-bounded for the same reason as updateRowsByFilter — see withSeqscanOff. const matchingRows = await withSeqscanOff(async (trx) => { - let query = trx + const base = trx .select({ id: userTableRows.id, position: userTableRows.position }) .from(userTableRows) .where(and(baseConditions, filterClause)) if (data.limit) { - query = query.limit(data.limit) as typeof query + return base + .orderBy(buildRowOrderBySql(undefined, tableName, table.schema.columns)) + .limit(data.limit) } - return query + return base }) if (matchingRows.length === 0) { diff --git a/apps/sim/lib/table/select-values.test.ts b/apps/sim/lib/table/select-values.test.ts index 7c4d4b456da..041b2f61242 100644 --- a/apps/sim/lib/table/select-values.test.ts +++ b/apps/sim/lib/table/select-values.test.ts @@ -2,7 +2,11 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { resolveFilterSelectValues, selectValueToNames } from '@/lib/table/select-values' +import { + resolveFilterSelectValues, + resolvePredicateSelectValues, + selectValueToNames, +} from '@/lib/table/select-values' import type { ColumnDefinition } from '@/lib/table/types' const status: ColumnDefinition = { @@ -100,3 +104,49 @@ describe('resolveFilterSelectValues', () => { ).toEqual({ $or: [{ col_status: 'opt_open' }, { col_title: 'x' }] }) }) }) + +/** + * The block builder serializes without schema access, so an option NAME that + * looks numeric or boolean arrives scalar-coerced ("123" → 123). Resolution + * must still find the option, or a correctly-authored builder filter compares + * a number against the stored id string and matches nothing. + */ +describe('resolvePredicateSelectValues — scalar-coerced option names', () => { + const numericStatus: ColumnDefinition = { + id: 'col_code', + name: 'code', + type: 'select', + options: [ + { id: 'opt_123', name: '123' }, + { id: 'opt_true', name: 'true' }, + ], + } + const columns = [numericStatus] + + it('resolves a coerced numeric name to its option id', () => { + expect( + resolvePredicateSelectValues({ all: [{ field: 'col_code', op: 'eq', value: 123 }] }, columns) + ).toEqual({ all: [{ field: 'col_code', op: 'eq', value: 'opt_123' }] }) + }) + + it('resolves a coerced boolean name, including inside in/contains', () => { + expect( + resolvePredicateSelectValues( + { any: [{ field: 'col_code', op: 'in', value: [true, 123] }] }, + columns + ) + ).toEqual({ any: [{ field: 'col_code', op: 'in', value: ['opt_true', 'opt_123'] }] }) + expect( + resolvePredicateSelectValues( + { all: [{ field: 'col_code', op: 'contains', value: 123 }] }, + columns + ) + ).toEqual({ all: [{ field: 'col_code', op: 'contains', value: 'opt_123' }] }) + }) + + it('leaves a scalar with no matching option name as-is', () => { + expect( + resolvePredicateSelectValues({ all: [{ field: 'col_code', op: 'eq', value: 999 }] }, columns) + ).toEqual({ all: [{ field: 'col_code', op: 'eq', value: 999 }] }) + }) +}) diff --git a/apps/sim/lib/table/select-values.ts b/apps/sim/lib/table/select-values.ts index 8addc65140d..c120a989ed7 100644 --- a/apps/sim/lib/table/select-values.ts +++ b/apps/sim/lib/table/select-values.ts @@ -5,11 +5,22 @@ * * Row-level id→name translation lives in `cell-format.ts`, which fuses it with * the column key translation. What remains here is the reverse direction: a - * filter operand typed as an option name resolving back to the stored id. + * filter operand typed as an option name resolving back to the stored id — in + * both the legacy `$` grammar and the v2 predicate tree. */ -import { getColumnId } from '@/lib/table/column-keys' -import type { ColumnDefinition, ConditionOperators, Filter, JsonValue } from '@/lib/table/types' +import { buildIdByName, getColumnId, predicateNamesToIds } from '@/lib/table/column-keys' +import type { + ColumnDefinition, + ConditionOperators, + Filter, + FilterOp, + JsonValue, + Predicate, + PredicateNode, + TablePredicate, + TableSchema, +} from '@/lib/table/types' import { resolveSelectOptionId } from '@/lib/table/validation' /** @@ -96,3 +107,67 @@ export function resolveFilterSelectValues(filter: Filter, columns: ColumnDefinit } return walk(filter) } + +/** + * Predicate-grammar sibling of {@link resolveFilterSelectValues}: returns a copy + * of an id-keyed predicate tree with `select` leaf values resolved from option + * name → id. + * + * Without it a v2 filter written against a select column (`{field:'status', + * op:'eq', value:'Open'}`) compares the option NAME against the stored option + * ID and silently matches nothing. + * + * Mirrors the `$`-grammar set exactly: equality/membership on a single select + * (`eq`/`ne`/`in`/`nin`) AND `contains`/`ncontains`, which on a MULTI-select are + * not pattern matches at all — the cell is an array of option ids, so those ops + * express membership and their operand is an option, not a substring. Leaving + * them out is what made a correctly-formed multi-select filter match nothing. + * The remaining pattern ops (`like`, `startsWith`, …) are rejected on select + * columns by `fieldPredicate`'s allowlist, so they never reach here. The + * valueless ops carry nothing to resolve. + */ +export function resolvePredicateSelectValues( + predicate: TablePredicate, + columns: ColumnDefinition[] +): TablePredicate { + const selectById = new Map( + columns.filter((c) => c.type === 'select').map((c) => [getColumnId(c), c]) + ) + if (selectById.size === 0) return predicate + + const RESOLVED_OPS = new Set(['eq', 'ne', 'in', 'nin', 'contains', 'ncontains']) + + const walk = (node: PredicateNode): PredicateNode => { + if ('all' in node) return { all: node.all.map(walk) } + if ('any' in node) return { any: node.any.map(walk) } + + const leaf = node as Predicate + const column = selectById.get(leaf.field) + if (!column || leaf.value === undefined || !RESOLVED_OPS.has(leaf.op)) return leaf + + const options = column.options + const value = Array.isArray(leaf.value) + ? leaf.value.map((v) => resolveOperand(v as JsonValue, options)) + : resolveOperand(leaf.value as JsonValue, options) + return { ...leaf, value: value as Predicate['value'] } + } + + return walk(predicate) as TablePredicate +} + +/** + * The complete name-keyed → storage-keyed translation for a v2 predicate: + * column names become column ids AND select operands become option ids. + * + * Both halves are required and neither is useful alone, but they lived as two + * separate calls that every boundary had to remember to pair — and three of them + * did not, so a filter on a select column compared an option NAME against a + * stored option ID and returned zero rows while reporting success. Call this + * instead of `predicateNamesToIds` so the pair cannot be split again. + */ +export function predicateToStorage(predicate: TablePredicate, schema: TableSchema): TablePredicate { + return resolvePredicateSelectValues( + predicateNamesToIds(predicate, buildIdByName(schema)), + schema.columns + ) +} diff --git a/apps/sim/lib/table/snapshot-cache.ts b/apps/sim/lib/table/snapshot-cache.ts index 0c24eccfb39..e99869b4adf 100644 --- a/apps/sim/lib/table/snapshot-cache.ts +++ b/apps/sim/lib/table/snapshot-cache.ts @@ -103,7 +103,9 @@ async function materialize(table: TableDefinition, key: string): Promise bytes += Buffer.byteLength(header) await handle.write(header) - let after: { orderKey: string; id: string } | null = null + // `order_key` is nullable (rows predating the backfill), and the page query + // seeks NULLs explicitly — so the cursor has to carry a null too. + let after: { orderKey: string | null; id: string } | null = null while (true) { const page = await selectExportRowPage(table, after, SNAPSHOT_BATCH_SIZE) if (page.length === 0) break diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index 3b254ad3b17..7ea5a6de3f0 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -10,25 +10,19 @@ import type { SQL } from 'drizzle-orm' import { sql } from 'drizzle-orm' import { getColumnId } from '@/lib/table/column-keys' import { NAME_PATTERN } from '@/lib/table/constants' +import { TableQueryValidationError } from '@/lib/table/errors' import type { ColumnDefinition, ConditionOperators, Filter, + FilterOp, JsonValue, + Predicate, + PredicateNode, Sort, + TablePredicate, } from '@/lib/table/types' -/** - * Error thrown when caller-supplied filter or sort input is malformed. - * Routes should map this to HTTP 400 with the message preserved. - */ -export class TableQueryValidationError extends Error { - constructor(message: string) { - super(message) - this.name = 'TableQueryValidationError' - } -} - type ColumnType = ColumnDefinition['type'] type ColumnMap = ReadonlyMap @@ -43,6 +37,33 @@ type ColumnMap = ReadonlyMap export const SINGLE_SELECT_OPERATORS = new Set(['$eq', '$ne', '$in', '$nin', '$empty']) export const MULTI_SELECT_OPERATORS = new Set(['$contains', '$ncontains', '$empty']) +/** + * The same allowlists in the v2 bare-operator grammar, applied inside + * `fieldPredicate` so both wire formats gate identically. Not derived from the + * `$` sets above by string surgery because the mapping is not 1:1 — `$empty` + * splits into `isEmpty`/`isNotEmpty`. `isNull`/`isNotNull` have no `$` + * equivalent and are allowed on both: a strict null check is meaningful on any + * column, select included. + */ +const SINGLE_SELECT_OPS = new Set([ + 'eq', + 'ne', + 'in', + 'nin', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +]) +const MULTI_SELECT_OPS = new Set([ + 'contains', + 'ncontains', + 'isEmpty', + 'isNotEmpty', + 'isNull', + 'isNotNull', +]) + /** * Returns the Postgres cast needed to compare a JSONB text value of the given * column type, or `null` when text comparison is correct. Single source of @@ -87,7 +108,13 @@ const ALLOWED_OPERATORS = new Set([ '$ncontains', '$startsWith', '$endsWith', + '$like', + '$ilike', + '$nlike', + '$nilike', '$empty', + '$isNull', + '$isNotNull', ]) /** @@ -169,6 +196,18 @@ function buildFilterClauseInternal( } // Skip arrays for regular fields - arrays are only valid for $or and $and. + // A v2 predicate tree (`{ all | any: [...] }`) that reaches this legacy + // compiler is a VERSION MISMATCH — a caller speaking the newer grammar + // against an older server. Skipping it as "an array on a regular field" + // compiles to no WHERE clause at all, which on a bulk delete means every + // row rather than none. Fail fast and name the mismatch instead. + if ((field === 'all' || field === 'any') && Array.isArray(condition)) { + throw new TableQueryValidationError( + `Filter looks like a v2 predicate tree ("${field}" group) but reached the legacy filter compiler. ` + + 'This usually means a client is sending the predicate grammar to a server that predates it.' + ) + } + // If we encounter an array here, it's likely malformed input (e.g., { name: [filter1, filter2] }) // which doesn't have a clear semantic meaning, so we skip it. if (Array.isArray(condition)) { @@ -191,6 +230,48 @@ function buildFilterClauseInternal( return sql.join(conditions, sql.raw(' AND ')) } +/** + * Builds a WHERE clause from a v2 `TablePredicate` (nestable `all`/`any` groups + * of `{ field, op, value }` leaves). Sibling of `buildFilterClause`: same engine, + * same `fieldPredicate` leaf — only the grammar differs. Returns `undefined` when + * the tree contributes no conditions (empty groups, all-no-op leaves). + * + * @throws {TableQueryValidationError} if a field name is invalid or an operator is not allowed + */ +export function buildPredicateClause( + predicate: TablePredicate, + tableName: string, + columns: ColumnDefinition[] +): SQL | undefined { + return buildPredicateNode(predicate, tableName, buildColumnMap(columns)) +} + +function isPredicateGroup(node: PredicateNode): node is TablePredicate { + return 'all' in node || 'any' in node +} + +function buildPredicateNode( + node: PredicateNode, + tableName: string, + columnMap: ColumnMap +): SQL | undefined { + if (isPredicateGroup(node)) { + const isAll = 'all' in node + const members = isAll ? node.all : node.any + const clauses: SQL[] = [] + for (const member of members) { + const clause = buildPredicateNode(member, tableName, columnMap) + if (clause) clauses.push(clause) + } + if (clauses.length === 0) return undefined + if (clauses.length === 1) return clauses[0] + return sql`(${sql.join(clauses, sql.raw(isAll ? ' AND ' : ' OR '))})` + } + + const leaf = node as Predicate + return fieldPredicate(tableName, leaf.field, leaf.op, leaf.value, columnMap.get(leaf.field)) +} + /** * Builds an ORDER BY clause from a sort object. * @@ -328,7 +409,8 @@ function buildFieldCondition( if (isRecordLike(condition)) { for (const [op, value] of Object.entries(condition)) { - // Validate operator to ensure only allowed operators are used + // Validate against the legacy `$`-whitelist, then normalize onto the shared + // `FilterOp` so v1 and v2 emit byte-identical leaf SQL. validateOperator(op) // Select values are opaque option ids — range/pattern operators are meaningless. if (isSelect) { @@ -340,120 +422,170 @@ function buildFieldCondition( } } - switch (op) { - case '$eq': - conditions.push(buildContainmentClause(tableName, field, value as JsonValue)) - break + if (op === '$empty') { + // `$empty: true/false` maps onto the valueless v2 ops. + const filterOp: FilterOp = coerceEmptyFlag(field, value) ? 'isEmpty' : 'isNotEmpty' + const clause = fieldPredicate(tableName, field, filterOp, undefined, column) + if (clause) conditions.push(clause) + continue + } - case '$ne': - conditions.push( - sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})` - ) - break + // Every other `$op` is `op` minus the leading `$` (e.g. `$gte` → `gte`). + const clause = fieldPredicate(tableName, field, op.slice(1) as FilterOp, value, column) + if (clause) conditions.push(clause) + } + } else { + // Simple value (primitive or null) - shorthand for equality. + // Example: { name: 'John' } is equivalent to { name: { $eq: 'John' } } + // isRecordLike's negation can't structurally exclude ConditionOperators (no index + // signature), so the JsonValue-only shape of this branch is asserted, not inferred. + // Routes through the unified `fieldPredicate` leaf like every other matcher, + // so equality semantics stay defined in exactly one place. + // + // On a multi-select the shorthand reads as "holds this option" — the cell is + // an array of option ids, so scalar equality can never be true. It maps to + // `contains` (membership). An EXPLICIT `$eq` on a multi-select still errors + // via the select allowlist: writing it out is a mistake worth naming, while + // the shorthand has an unambiguous intent. + const shorthandOp: FilterOp = column?.type === 'select' && column.multiple ? 'contains' : 'eq' + const clause = fieldPredicate(tableName, field, shorthandOp, condition as JsonValue, column) + if (clause) conditions.push(clause) + } - case '$gt': - conditions.push( - buildComparisonClause(tableName, field, '>', value as number | string, columnType) - ) - break + return conditions +} - case '$gte': - conditions.push( - buildComparisonClause(tableName, field, '>=', value as number | string, columnType) - ) - break +/** + * The single leaf primitive: compiles one `field op value` into SQL. Every + * matcher routes through here — both filter compilers (`buildFilterClause` for + * the legacy `$`-grammar, `buildPredicateClause` for the v2 grammar), the upsert + * conflict probe, and the unique-constraint checks. Centralizing the leaf means + * equality/case/null/cast semantics are defined exactly once, so "find the row" + * and "is this value unique" can never disagree. + * + * Returns `undefined` when the predicate is a no-op (empty `in`/`nin` array), + * matching the legacy behavior of emitting no clause. + * + * Equality (`eq`/`ne`/`in`/`nin`) uses case-sensitive JSONB containment (GIN + * indexed). Text matches (`contains`/`ncontains`/`startsWith`/`endsWith`) are + * ILIKE (case-insensitive). Ranges cast per column type. + */ +export function fieldPredicate( + tableName: string, + field: string, + op: FilterOp, + value: JsonValue | undefined, + column: ColumnDefinition | undefined +): SQL | undefined { + validateFieldName(field) - case '$lt': - conditions.push( - buildComparisonClause(tableName, field, '<', value as number | string, columnType) - ) - break + // System columns (`createdAt`/`updatedAt`/`id`) are real row columns, not + // JSONB keys — dispatch before the `data->>` builders below, which would + // silently match nothing (the key never exists in `data`). + if (isSystemColumn(field)) { + return buildSystemColumnClause(tableName, field, op, value) + } - case '$lte': - conditions.push( - buildComparisonClause(tableName, field, '<=', value as number | string, columnType) - ) - break - - case '$in': - if (Array.isArray(value) && value.length > 0) { - if (value.length === 1) { - // Single value then use containment clause - conditions.push(buildContainmentClause(tableName, field, value[0])) - } else { - // Multiple values then use OR clause - const inConditions = value.map((v) => buildContainmentClause(tableName, field, v)) - conditions.push(sql`(${sql.join(inConditions, sql.raw(' OR '))})`) - } - } - break - - case '$nin': - if (Array.isArray(value) && value.length > 0) { - const ninConditions = value.map( - (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` - ) - conditions.push(sql`(${sql.join(ninConditions, sql.raw(' AND '))})`) - } - break - - case '$contains': - conditions.push( - isMultiSelect - ? buildArrayMembershipClause(tableName, field, value as JsonValue) - : buildLikeClause(tableName, field, value as string, 'contains') - ) - break - - case '$ncontains': - conditions.push( - isMultiSelect - ? // Mirror the ILIKE negation: rows with no selection at all count - // as "does not contain", so an empty cell isn't silently excluded. - sql`NOT (${buildArrayMembershipClause(tableName, field, value as JsonValue)})` - : buildLikeClause(tableName, field, value as string, 'contains', { negate: true }) - ) - break + const columnType = column?.type + const isSelect = columnType === 'select' + // A multi-select cell holds an ARRAY of option ids, so equality against a + // scalar can never be true; the question is membership. Gating and clause + // choice both live here rather than in `buildFieldCondition` so the v2 + // predicate grammar gets the identical treatment. + const isMultiSelect = isSelect && column?.multiple === true - case '$startsWith': - conditions.push(buildLikeClause(tableName, field, value as string, 'startsWith')) - break + if (isSelect) { + const allowed = isMultiSelect ? MULTI_SELECT_OPS : SINGLE_SELECT_OPS + if (!allowed.has(op)) { + throw new TableQueryValidationError( + `Operator "${op}" is not supported on ${isMultiSelect ? 'multi-select' : 'select'} column "${field}". Allowed: ${Array.from(allowed).join(', ')}` + ) + } + } - case '$endsWith': - conditions.push(buildLikeClause(tableName, field, value as string, 'endsWith')) - break + if (isMultiSelect) { + switch (op) { + case 'contains': + return buildArrayMembershipClause(tableName, field, value as JsonValue) + case 'ncontains': + return sql`NOT (${buildArrayMembershipClause(tableName, field, value as JsonValue)})` + case 'isEmpty': + return buildEmptyClause(tableName, field, true, true) + case 'isNotEmpty': + return buildEmptyClause(tableName, field, false, true) + default: + break + } + } - case '$empty': - conditions.push( - buildEmptyClause(tableName, field, coerceEmptyFlag(field, value), isMultiSelect) - ) - break + switch (op) { + case 'eq': + return buildContainmentClause(tableName, field, value as JsonValue) + + case 'ne': + return sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})` + + case 'gt': + return buildComparisonClause(tableName, field, '>', value as number | string, columnType) + case 'gte': + return buildComparisonClause(tableName, field, '>=', value as number | string, columnType) + case 'lt': + return buildComparisonClause(tableName, field, '<', value as number | string, columnType) + case 'lte': + return buildComparisonClause(tableName, field, '<=', value as number | string, columnType) + + case 'in': { + if (!Array.isArray(value) || value.length === 0) return undefined + if (value.length === 1) return buildContainmentClause(tableName, field, value[0]) + const inConditions = value.map((v) => buildContainmentClause(tableName, field, v)) + return sql`(${sql.join(inConditions, sql.raw(' OR '))})` + } - default: - // This should never happen due to validateOperator, but added for completeness. - // Throw a plain Error (→ 500) since reaching this default means the switch - // and ALLOWED_OPERATORS have drifted — that's a programmer error, not a caller error. - throw new Error(`Unsupported operator: ${op}`) - } + case 'nin': { + if (!Array.isArray(value) || value.length === 0) return undefined + const ninConditions = value.map( + (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})` + ) + return sql`(${sql.join(ninConditions, sql.raw(' AND '))})` } - } else { - // Simple value (primitive or null) - shorthand for equality. - // Example: { name: 'John' } is equivalent to { name: { $eq: 'John' } } - // isRecordLike's negation can't structurally exclude ConditionOperators (no index - // signature), unlike the prior typeof-based narrowing, so the JsonValue-only shape - // of this branch is asserted rather than inferred. - // - // On a multi-select the shorthand reads as "holds this option" — scalar - // equality against an array cell can never be true, so treat it as - // membership rather than compiling a filter that silently matches nothing. - conditions.push( - isMultiSelect - ? buildArrayMembershipClause(tableName, field, condition as JsonValue) - : buildContainmentClause(tableName, field, condition as JsonValue) - ) - } - return conditions + case 'contains': + return buildLikeClause(tableName, field, value as string, 'contains') + case 'ncontains': + return buildLikeClause(tableName, field, value as string, 'contains', { negate: true }) + case 'startsWith': + return buildLikeClause(tableName, field, value as string, 'startsWith') + case 'endsWith': + return buildLikeClause(tableName, field, value as string, 'endsWith') + + case 'like': + return buildPatternClause(tableName, field, value as string, { caseInsensitive: false }) + case 'ilike': + return buildPatternClause(tableName, field, value as string, { caseInsensitive: true }) + case 'nlike': + return buildPatternClause(tableName, field, value as string, { + caseInsensitive: false, + negate: true, + }) + case 'nilike': + return buildPatternClause(tableName, field, value as string, { + caseInsensitive: true, + negate: true, + }) + + case 'isEmpty': + return buildEmptyClause(tableName, field, true) + case 'isNotEmpty': + return buildEmptyClause(tableName, field, false) + + case 'isNull': + return buildNullClause(tableName, field, true) + case 'isNotNull': + return buildNullClause(tableName, field, false) + + default: + throw new TableQueryValidationError(`Invalid operator "${op}"`) + } } /** @@ -497,6 +629,124 @@ function buildLogicalClause( return sql`(${sql.join(clauses, sql.raw(` ${operator} `))})` } +/** + * Row columns that are addressable in filters/sorts but live on the row itself + * rather than inside the JSONB `data` blob. The docs advertise all three as + * filterable and sortable; without this dispatch they compile to a `data->>'…'` + * extraction of a key that never exists, so they silently match nothing. + */ +const SYSTEM_COLUMNS: Readonly> = { + createdAt: { column: 'created_at', kind: 'timestamp' }, + updatedAt: { column: 'updated_at', kind: 'timestamp' }, + id: { column: 'id', kind: 'text' }, +} + +function isSystemColumn(field: string): boolean { + return Object.hasOwn(SYSTEM_COLUMNS, field) +} + +/** + * Builds a predicate against a system column. Timestamp columns bind ISO strings + * normalized to UTC wall clock; the text column (`id`) binds as text and also + * accepts the pattern ops. Anything else is rejected with an actionable error. + */ +function buildSystemColumnClause( + tableName: string, + field: string, + op: FilterOp, + value: JsonValue | undefined +): SQL | undefined { + const spec = SYSTEM_COLUMNS[field] + const col = sql.raw(`${tableName}.${spec.column}`) + // `created_at`/`updated_at` are `timestamp WITHOUT time zone` holding UTC wall + // clock. A bare `::timestamptz` comparison promotes the column using the session + // `TimeZone` GUC, so identical queries return different rows per environment and + // day-boundary ranges land off by the offset. Normalizing the bound to UTC wall + // clock is session-independent and still honors an explicit offset in the input. + const ts = (v: JsonValue | undefined) => sql`${String(v)}::timestamptz AT TIME ZONE 'UTC'` + const bind = spec.kind === 'timestamp' ? ts : (v: JsonValue | undefined) => sql`${String(v)}` + /** + * Mirrors the JSONB pattern builders: `*` is the caller's only wildcard, an + * empty pattern is rejected (it would collapse to `%` and match every row), + * and the negated forms keep NULL cells so "does not contain X" retains them. + */ + const like = ( + v: JsonValue | undefined, + pattern: (escaped: string) => string, + ci: boolean, + negate = false + ) => { + const text = String(v ?? '') + if (text.length === 0) { + throw new TableQueryValidationError( + `Operator "${op}" on column "${field}" requires a non-empty value` + ) + } + const p = pattern(escapeLikePattern(text)) + const match = ci ? sql`${col} ILIKE ${p}` : sql`${col} LIKE ${p}` + return negate ? sql`NOT (${match})` : match + } + + // The text system column (`id`) additionally supports the pattern ops; + // timestamps fall through to the unsupported-operator error below. + if (spec.kind === 'text') { + const star = (e: string) => e.replace(/\*/g, '%') + switch (op) { + case 'like': + return like(value, star, false) + case 'ilike': + return like(value, star, true) + case 'nlike': + return like(value, star, false, true) + case 'nilike': + return like(value, star, true, true) + case 'contains': + return like(value, (e) => `%${e}%`, true) + case 'ncontains': + return like(value, (e) => `%${e}%`, true, true) + case 'startsWith': + return like(value, (e) => `${e}%`, true) + case 'endsWith': + return like(value, (e) => `%${e}`, true) + default: + break + } + } + + switch (op) { + case 'eq': + return sql`${col} = ${bind(value)}` + case 'ne': + return sql`${col} <> ${bind(value)}` + case 'gt': + return sql`${col} > ${bind(value)}` + case 'gte': + return sql`${col} >= ${bind(value)}` + case 'lt': + return sql`${col} < ${bind(value)}` + case 'lte': + return sql`${col} <= ${bind(value)}` + case 'in': { + if (!Array.isArray(value) || value.length === 0) return undefined + return sql`${col} IN (${sql.join(value.map(bind), sql.raw(', '))})` + } + case 'nin': { + if (!Array.isArray(value) || value.length === 0) return undefined + return sql`${col} NOT IN (${sql.join(value.map(bind), sql.raw(', '))})` + } + case 'isNull': + case 'isEmpty': + return sql`${col} IS NULL` + case 'isNotNull': + case 'isNotEmpty': + return sql`${col} IS NOT NULL` + default: + throw new TableQueryValidationError( + `Operator "${op}" is not supported on the built-in column "${field}" — use eq, ne, gt, gte, lt, lte, in, nin, isNull, isNotNull.` + ) + } +} + /** Builds JSONB containment clause: `data @> '{"field": value}'::jsonb` (uses GIN index) */ function buildContainmentClause(tableName: string, field: string, value: JsonValue): SQL { const jsonObj = JSON.stringify({ [field]: value }) @@ -523,11 +773,12 @@ function buildArrayMembershipClause(tableName: string, field: string, value: Jso * to `timestamptz` so date strings compare chronologically and timezone offsets * in ISO strings (e.g. `2024-01-01T00:00:00Z`) are preserved rather than * silently stripped (which would make results depend on the server's TimeZone - * setting). Unknown/other types - * fall back to `numeric` (legacy default — preserves behavior for ad-hoc fields - * with no schema entry). The right-hand value is cast explicitly because - * drizzle parameterizes it as `text`; without the cast, Postgres would compare - * `text text` and silently produce lexicographic results. + * setting). `string` columns compare lexicographically as text. `boolean`/`json` + * columns have no meaningful ordering and are rejected. Columns with no schema + * entry fall back to `numeric` (legacy default — preserves behavior for ad-hoc + * fields). The right-hand value is cast explicitly because drizzle parameterizes + * it as `text`; without the cast, Postgres would compare `text text` and + * silently produce lexicographic results. * * Cannot use the GIN index — falls back to a sequential scan over the table's * rows (bounded by the btree prefix on `table_id`). @@ -540,6 +791,18 @@ function buildComparisonClause( columnType: ColumnType | undefined ): SQL { const escapedField = field.replace(/'/g, "''") + + if (columnType === 'boolean' || columnType === 'json') { + throw new TableQueryValidationError( + `Range operator on column "${field}" (${columnType}) is not supported — ${columnType} values have no ordering.` + ) + } + + if (columnType === 'string') { + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + return sql`${cell} ${sql.raw(operator)} ${String(value)}` + } + const cast = jsonbCastForType(columnType) ?? 'numeric' validateComparisonValue(field, columnType, cast, value) const cell = sql.raw(`(${tableName}.data->>'${escapedField}')::${cast}`) @@ -553,6 +816,36 @@ export function escapeLikePattern(value: string): string { return value.replace(/[\\%_]/g, '\\$&') } +/** + * General LIKE/ILIKE pattern match (the `like`/`ilike` ops). The caller's `*` + * is the only wildcard — it maps to SQL `%`; any literal `%`/`_`/`\` in the + * value is escaped so it matches itself. Empty/exact patterns are allowed (an + * empty pattern matches only the empty string, not every row, so it's not the + * footgun the positional `buildLikeClause` guards against). `negate` inverts + * the match and keeps null cells — "does not match" retains empty rows, + * mirroring `buildLikeClause`'s ncontains semantics. Cannot use the GIN index; + * sequential scan bounded by the `table_id` btree prefix. + */ +function buildPatternClause( + tableName: string, + field: string, + value: string, + options: { caseInsensitive: boolean; negate?: boolean } +): SQL { + const escapedField = field.replace(/'/g, "''") + const pattern = String(value) + .replace(/[\\%_]/g, '\\$&') + .replace(/\*/g, '%') + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + const match = options.caseInsensitive + ? sql`${cell} ILIKE ${pattern}` + : sql`${cell} LIKE ${pattern}` + if (!options.negate) return match + return options.caseInsensitive + ? sql`(${cell} IS NULL OR ${cell} NOT ILIKE ${pattern})` + : sql`(${cell} IS NULL OR ${cell} NOT LIKE ${pattern})` +} + /** * Builds a case-insensitive pattern match against a JSONB cell using ILIKE. * `position` controls wildcard placement: `contains` → `%value%`, `startsWith` @@ -635,6 +928,18 @@ function buildEmptyClause( : sql`(${cell} IS NOT NULL AND ${cell} <> '')` } +/** + * Strict null check on a JSONB cell — distinct from `isEmpty`/`isNotEmpty`, + * which also treat the empty string as empty. `isNull` matches an absent key or + * JSON null (both surfaced as SQL NULL by `->>`); the negation requires the cell + * to be present (an empty string counts as not-null here). + */ +function buildNullClause(tableName: string, field: string, isNull: boolean): SQL { + const escapedField = field.replace(/'/g, "''") + const cell = sql.raw(`${tableName}.data->>'${escapedField}'`) + return isNull ? sql`${cell} IS NULL` : sql`${cell} IS NOT NULL` +} + /** * Builds a single ORDER BY clause for a field. * Timestamp fields use direct column access, others use JSONB text extraction. @@ -654,8 +959,8 @@ function buildSortFieldClause( const escapedField = field.replace(/'/g, "''") const directionSql = direction.toUpperCase() - if (field === 'createdAt' || field === 'updatedAt') { - return sql.raw(`${tableName}.${escapedField} ${directionSql}`) + if (isSystemColumn(field)) { + return sql.raw(`${tableName}.${SYSTEM_COLUMNS[field].column} ${directionSql}`) } const jsonbExtract = `${tableName}.data->>'${escapedField}'` diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 6f38357552f..1c6b5e39913 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -2,7 +2,7 @@ * Type definitions for user-defined tables. */ -import type { COLUMN_TYPES } from '@/lib/table/constants' +import type { COLUMN_TYPES, FILTER_OPS } from '@/lib/table/constants' export type ColumnValue = string | number | boolean | null | Date export type JsonValue = ColumnValue | JsonValue[] | { [key: string]: JsonValue } @@ -63,6 +63,9 @@ export interface ColumnDefinition { multiple?: boolean } +/** The column `type` discriminator, named so callers don't index into the interface. */ +export type ColumnType = ColumnDefinition['type'] + /** One group output → one plain column. */ export interface WorkflowGroupOutput { /** Source block id within the configured workflow. `''` for enrichment groups. */ @@ -283,8 +286,8 @@ export interface TableMetadata { * user resizes, reorders, pins, or hides columns. */ export interface TableViewConfig extends TableMetadata { - filter?: Filter | null - sort?: Sort | null + filter?: TablePredicate | null + sort?: SortSpec | null } /** Async background-job lifecycle state for a table. NULL/undefined = idle (no job). */ @@ -500,6 +503,39 @@ export interface Filter { [key: string]: ColumnValue | ConditionOperators | Filter[] | undefined } +/** + * v2 filter operators (bare, no `$`). Equality and `in`/`nin` are case-sensitive + * (JSONB containment, GIN-indexed); the text ops `contains`/`ncontains`/ + * `startsWith`/`endsWith` are ILIKE (case-insensitive). `isEmpty`/`isNotEmpty` + * match null OR empty string; + * `isNull`/`isNotNull` are strict null checks. The four `is*` ops are valueless. + * This is the canonical operator set the shared `fieldPredicate` leaf + * understands; the legacy `$`-operators normalize onto it. + */ +export type FilterOp = (typeof FILTER_OPS)[number] + +/** A single v2 leaf predicate: `field op value`. `value` is omitted for `isEmpty`/`isNotEmpty`. */ +export interface Predicate { + field: string + op: FilterOp + value?: JsonValue +} + +/** + * v2 nestable filter tree. A group is either `{ all: [...] }` (AND) or + * `{ any: [...] }` (OR); members are leaves or nested groups. Replaces the + * MongoDB-style `Filter` on the v2 surface — same engine, legible grammar. + * + * @example + * { all: [{ field: 'slack_user_id', op: 'in', value: ['U1','U2'] }, { field: 'wins', op: 'gte', value: 10 }] } + * { any: [{ field: 'status', op: 'eq', value: 'active' }, { field: 'status', op: 'eq', value: 'pending' }] } + */ +export type PredicateNode = Predicate | TablePredicate +export type TablePredicate = { all: PredicateNode[] } | { any: PredicateNode[] } + +/** v2 sort specification: an ordered list of `{ field, direction }`. */ +export type SortSpec = Array<{ field: string; direction: SortDirection }> + export interface ValidationResult { valid: boolean errors: string[] @@ -531,11 +567,21 @@ export interface SortRule { export interface QueryOptions { filter?: Filter + /** + * v2 nestable predicate. When set it takes precedence over `filter` — the two + * compile through the same `fieldPredicate` leaf, so callers pick one grammar. + */ + predicate?: TablePredicate sort?: Sort + /** Page row cap. Omitted = return the ENTIRE matching result, failing fast if + * it exceeds the response byte budget (`MAX_QUERY_RESULT_BYTES`). A bounded + * page may byte-cut early with `nextCursor` set. Never an unbounded fetch — + * the drain stops at the budget either way. */ limit?: number offset?: number /** Keyset cursor for the default `(order_key, id)` order — see {@link TableRowsCursor}. - * Mutually exclusive with `sort` and `offset`; takes precedence over `offset` when set. */ + * Mutually exclusive with `sort`. May be combined with `offset` (a compound + * cursor seeks the anchor, then offsets past unkeyed rows consumed after it). */ after?: TableRowsCursor /** * When true (default), runs a `COUNT(*)` and returns `totalCount` as a number. @@ -557,6 +603,13 @@ export interface QueryResult { totalCount: number | null limit: number offset: number + /** + * Opaque cursor for the next page — non-null whenever more matching rows + * exist beyond this page, whether the page was cut by `limit` or by the + * response byte budget. Callers echo it back as `cursor` and never construct + * keyset/offset state themselves. See `rows/cursor.ts`. + */ + nextCursor: string | null } export interface BulkOperationResult { diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index 8b6d58ff0e2..14275ea74eb 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -13,9 +13,11 @@ import { MAX_SELECT_OPTIONS, NAME_PATTERN, TABLE_LIMITS, + USER_TABLE_ROWS_SQL_NAME, } from '@/lib/table/constants' import { normalizeDateCellValue } from '@/lib/table/dates' import { withSeqscanOff } from '@/lib/table/planner' +import { fieldPredicate } from '@/lib/table/sql' import type { ColumnDefinition, JsonValue, @@ -297,12 +299,21 @@ function optionIds(column: ColumnDefinition): Set { * actually fit the target option set. */ export function resolveSelectOptionId(value: JsonValue, options: SelectOption[]): string | null { - if (typeof value !== 'string') return null - const byId = options.find((o) => o.id === value) + // The block builder serializes without schema access, so an option NAME that + // looks numeric or boolean ("123", "true") arrives scalar-coerced. Stringify + // scalars so the name still resolves; arrays/objects stay unresolvable. + const text = + typeof value === 'string' + ? value + : typeof value === 'number' || typeof value === 'boolean' + ? String(value) + : null + if (text === null) return null + const byId = options.find((o) => o.id === text) if (byId) return byId.id const byName = - options.find((o) => o.name === value) ?? - options.find((o) => o.name.toLowerCase() === value.toLowerCase()) + options.find((o) => o.name === text) ?? + options.find((o) => o.name.toLowerCase() === text.toLowerCase()) return byName ? byName.id : null } @@ -469,12 +480,8 @@ export function validateUniqueConstraints( const duplicate = existingRows.find((row) => { if (excludeRowId && row.id === excludeRowId) return false - - const existingValue = row.data[key] - if (typeof value === 'string' && typeof existingValue === 'string') { - return value.toLowerCase() === existingValue.toLowerCase() - } - return value === existingValue + // Case-sensitive, matching the DB unique-check leaf (`fieldPredicate` eq). + return value === row.data[key] }) if (duplicate) { @@ -517,27 +524,13 @@ export async function checkUniqueConstraintsDb( for (const column of uniqueColumns) { const key = getColumnId(column) - if (!NAME_PATTERN.test(key)) { - throw new Error(`Invalid column id: ${key}`) - } - const value = data[key] if (value === null || value === undefined) continue - if (typeof value === 'string') { - conditions.push({ - column, - value, - sql: sql`lower(${userTableRows.data}->>${sql.raw(`'${key}'`)}) = ${value.toLowerCase()}`, - }) - } else { - // For other types, use direct JSONB comparison - conditions.push({ - column, - value, - sql: sql`(${userTableRows.data}->${sql.raw(`'${key}'`)})::jsonb = ${JSON.stringify(value)}::jsonb`, - }) - } + // Same leaf as the upsert conflict probe → case-sensitive JSONB containment + // (GIN-indexed). `eq` always yields a clause for a non-null value. + const clause = fieldPredicate(USER_TABLE_ROWS_SQL_NAME, key, 'eq', value, column) + if (clause) conditions.push({ column, value, sql: clause }) } if (conditions.length === 0) { @@ -545,9 +538,9 @@ export async function checkUniqueConstraintsDb( } // Query for each unique column separately to provide specific error messages. - // Tenant-bounded: `lower(data->>'col') = ...` is unestimatable, so the planner - // otherwise seq-scans the whole shared relation per check — 3.5s on every - // insert/edit when the value is unique (no early exit). With an external + // The predicate is now case-sensitive JSONB containment (`data @> {...}`), + // which can use the GIN index. We still pin `enable_seqscan = off` (tenant- + // bounded) defensively for the small-table / cold-stats case. With an external // transaction the flag is set on it directly — opening our own transaction // inside the caller's would be the nested pool checkout the migration- // hardening work eliminated (self-deadlock under pool exhaustion). @@ -635,8 +628,7 @@ export async function checkBatchUniqueConstraintsDb( const value = rowData[key] if (value === null || value === undefined) continue - const normalizedValue = - typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value) + const normalizedValue = typeof value === 'string' ? value : JSON.stringify(value) // Check for duplicate within batch const columnValueMap = batchValueMap.get(key)! @@ -672,14 +664,22 @@ export async function checkBatchUniqueConstraintsDb( const valueArray = Array.from(values) const valueConditions = valueArray.map((normalizedValue) => { - // Check if the original values are strings (normalized values for strings are lowercase) - // We need to determine the type from the column definition or the first row that has this value - const isStringColumn = column.type === 'string' - - if (isStringColumn) { - return sql`lower(${userTableRows.data}->>${sql.raw(`'${columnId}'`)}) = ${normalizedValue}` + // Reconstruct the original typed value from its normalized key: string + // columns store the raw string; others store JSON.stringify(value). + const originalValue: JsonValue = + column.type === 'string' ? normalizedValue : JSON.parse(normalizedValue) + // Same case-sensitive containment leaf as every other matcher. + const clause = fieldPredicate( + USER_TABLE_ROWS_SQL_NAME, + columnId, + 'eq', + originalValue, + column + ) + if (!clause) { + throw new Error(`Failed to build unique-constraint predicate for column "${column.name}"`) } - return sql`(${userTableRows.data}->${sql.raw(`'${columnId}'`)})::jsonb = ${normalizedValue}::jsonb` + return clause }) const conflictingRows = await ex @@ -697,9 +697,7 @@ export async function checkBatchUniqueConstraintsDb( const conflictData = conflict.data as RowData const conflictValue = conflictData[columnId] const normalizedConflictValue = - typeof conflictValue === 'string' - ? conflictValue.toLowerCase() - : JSON.stringify(conflictValue) + typeof conflictValue === 'string' ? conflictValue : JSON.stringify(conflictValue) // Find which batch rows have this conflicting value for (let i = 0; i < rows.length; i++) { @@ -707,7 +705,7 @@ export async function checkBatchUniqueConstraintsDb( if (rowValue === null || rowValue === undefined) continue const normalizedRowValue = - typeof rowValue === 'string' ? rowValue.toLowerCase() : JSON.stringify(rowValue) + typeof rowValue === 'string' ? rowValue : JSON.stringify(rowValue) if (normalizedRowValue === normalizedConflictValue) { // Check if this row already has errors for this column diff --git a/apps/sim/lib/table/views/service.test.ts b/apps/sim/lib/table/views/service.test.ts index 5b048954313..3405f73bd90 100644 --- a/apps/sim/lib/table/views/service.test.ts +++ b/apps/sim/lib/table/views/service.test.ts @@ -3,7 +3,7 @@ */ import { describe, expect, it } from 'vitest' import type { ColumnDefinition, TableViewConfig } from '@/lib/table/types' -import { pruneViewConfig } from '@/lib/table/views/service' +import { normalizeStoredViewConfig, pruneViewConfig } from '@/lib/table/views/service' const columns: ColumnDefinition[] = [ { id: 'col_a', name: 'Name', type: 'text' }, @@ -28,14 +28,18 @@ describe('pruneViewConfig', () => { }) it('drops a sort on a deleted column and collapses to null when none remain', () => { - expect(pruneViewConfig({ sort: { col_gone: 'asc' } }, columns).sort).toBeNull() - expect(pruneViewConfig({ sort: { col_a: 'desc' } }, columns).sort).toEqual({ col_a: 'desc' }) + expect( + pruneViewConfig({ sort: [{ field: 'col_gone', direction: 'asc' }] }, columns).sort + ).toBeNull() + expect( + pruneViewConfig({ sort: [{ field: 'col_a', direction: 'desc' }] }, columns).sort + ).toEqual([{ field: 'col_a', direction: 'desc' }]) }) it('leaves the filter untouched even when it references a deleted column', () => { // Pruning a predicate would silently widen the view's row set — surfacing a // stale condition the user can see and remove is the safer failure. - const filter = { col_gone: { $eq: 'x' } } + const filter = { all: [{ field: 'col_gone', op: 'eq' as const, value: 'x' }] } expect(pruneViewConfig({ filter }, columns).filter).toEqual(filter) }) @@ -50,3 +54,33 @@ describe('pruneViewConfig', () => { ]) }) }) + +/** + * Reads written before the grammar switch: the feature never released, so + * legacy-shaped configs exist only from pre-refactor testing — but they must + * come back as v2, not render broken. + */ +describe('normalizeStoredViewConfig', () => { + it('converts a legacy $-object filter to a predicate tree', () => { + const out = normalizeStoredViewConfig({ filter: { col_a: { $eq: 'x' } } }) + expect(out.filter).toEqual({ all: [{ field: 'col_a', op: 'eq', value: 'x' }] }) + }) + + it('converts a legacy {col: dir} sort record to an ordered spec', () => { + const out = normalizeStoredViewConfig({ sort: { col_a: 'desc' } }) + expect(out.sort).toEqual([{ field: 'col_a', direction: 'desc' }]) + }) + + it('passes v2-shaped configs through untouched', () => { + const config = { + filter: { all: [{ field: 'col_a', op: 'eq', value: 'x' }] }, + sort: [{ field: 'col_a', direction: 'asc' }], + } + expect(normalizeStoredViewConfig(config)).toEqual(config) + }) + + it('drops an unconvertible legacy filter rather than surfacing it broken', () => { + const out = normalizeStoredViewConfig({ filter: { $bogus: [{ nested: true }] } }) + expect(out.filter).toBeNull() + }) +}) diff --git a/apps/sim/lib/table/views/service.ts b/apps/sim/lib/table/views/service.ts index 1b575bb618a..1c8684a414c 100644 --- a/apps/sim/lib/table/views/service.ts +++ b/apps/sim/lib/table/views/service.ts @@ -15,7 +15,15 @@ import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, asc, eq, ne, sql } from 'drizzle-orm' import { getColumnId } from '@/lib/table/column-keys' -import type { ColumnDefinition, TableViewConfig } from '@/lib/table/types' +import { NAME_PATTERN } from '@/lib/table/constants' +import { filterRulesToPredicate, filterToRules } from '@/lib/table/query-builder/converters' +import type { + ColumnDefinition, + Filter, + Predicate, + PredicateNode, + TableViewConfig, +} from '@/lib/table/types' const logger = createLogger('TableViewsService') @@ -69,22 +77,58 @@ export function pruneViewConfig( pruned.columnWidths = widths } if (config.sort) { - const sort: Record = {} - for (const [id, direction] of Object.entries(config.sort)) { - if (live.has(id)) sort[id] = direction - } - pruned.sort = Object.keys(sort).length > 0 ? sort : null + const sort = config.sort.filter((s) => live.has(s.field)) + pruned.sort = sort.length > 0 ? sort : null } return pruned } +/** + * Migrates a config stored before the grammar switch. The feature never + * released, so legacy-shaped rows exist only from pre-refactor testing: a + * `$`-object filter converts through the builder-rule round-trip (its exact + * authoring domain), and a `{col: dir}` sort record becomes an ordered spec. + * Anything unconvertible is dropped rather than surfaced broken. + */ + +/** Every leaf field in the tree is a plausible column id. */ +function predicateFieldsAreValid(node: PredicateNode): boolean { + if ('all' in node) return node.all.every(predicateFieldsAreValid) + if ('any' in node) return node.any.every(predicateFieldsAreValid) + return NAME_PATTERN.test((node as Predicate).field) +} + +export function normalizeStoredViewConfig(raw: Record): TableViewConfig { + const config = { ...raw } as TableViewConfig + const filter = raw.filter as Record | null | undefined + if (filter && !('all' in filter) && !('any' in filter)) { + try { + const converted = filterRulesToPredicate(filterToRules(filter as Filter)) + // The rule converters don't reject garbage — an unknown `$op` becomes a + // rule on a column literally named `$op`. A converted leaf whose field + // fails the column-name pattern proves the input wasn't builder-authored. + config.filter = converted && predicateFieldsAreValid(converted) ? converted : null + } catch { + config.filter = null + } + } + const sort = raw.sort as Record | unknown[] | null | undefined + if (sort && !Array.isArray(sort)) { + config.sort = Object.entries(sort).map(([field, direction]) => ({ field, direction })) + } + return config +} + function toTableView(row: typeof tableViews.$inferSelect, columns: ColumnDefinition[]): TableView { return { id: row.id, tableId: row.tableId, name: row.name, - config: pruneViewConfig((row.config ?? {}) as TableViewConfig, columns), + config: pruneViewConfig( + normalizeStoredViewConfig((row.config ?? {}) as Record), + columns + ), isDefault: row.isDefault, createdBy: row.createdBy, createdAt: row.createdAt, diff --git a/apps/sim/lib/workspaces/utils.ts b/apps/sim/lib/workspaces/utils.ts index a8fccb2b79c..05dd7a62466 100644 --- a/apps/sim/lib/workspaces/utils.ts +++ b/apps/sim/lib/workspaces/utils.ts @@ -48,6 +48,21 @@ export async function getWorkspaceBilledAccountUserId(workspaceId: string): Prom return settings?.billedAccountUserId ?? null } +/** + * The organization that owns a workspace (null for personal workspaces). Used to + * gate features by org cohort — the flag model allowlists org ids, and API routes + * only carry a `workspaceId`, so this resolves the one from the other. + */ +export async function getWorkspaceOrganizationId(workspaceId: string): Promise { + if (!workspaceId) return null + const rows = await db + .select({ organizationId: workspaceTable.organizationId }) + .from(workspaceTable) + .where(eq(workspaceTable.id, workspaceId)) + .limit(1) + return rows[0]?.organizationId ?? null +} + /** * Workspaces the user administers purely through organization owner/admin role, * with no explicit permission row required. Empty when the user is not an org diff --git a/apps/sim/tools/normalize.test.ts b/apps/sim/tools/normalize.test.ts new file mode 100644 index 00000000000..3f2f845f5ea --- /dev/null +++ b/apps/sim/tools/normalize.test.ts @@ -0,0 +1,40 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { normalizeToolId } from '@/tools/normalize' + +describe('normalizeToolId', () => { + it('strips a resource-id suffix', () => { + expect(normalizeToolId('table_query_rows_tbl_1a2c1741')).toBe('table_query_rows') + expect(normalizeToolId('knowledge_search_5cc56998-3e1d-4d91')).toBe('knowledge_search') + expect(normalizeToolId('workflow_executor_f3d81b32')).toBe('workflow_executor') + expect(normalizeToolId('deployed_block_executor_custom_block_9')).toBe( + 'deployed_block_executor' + ) + }) + + it('leaves a bare tool id alone', () => { + expect(normalizeToolId('table_query_rows')).toBe('table_query_rows') + expect(normalizeToolId('gmail_send')).toBe('gmail_send') + }) + + /** + * Regression: `table_query_rows_v2` starts with `table_query_rows_`, so the + * resource-suffix strip turned it into the v1 tool. The executor then logged + * the v2 id while issuing v1's `GET /rows?filter=` — which reached + * the legacy filter compiler and 400'd on a correctly configured v2 block. + */ + it('does not mistake a version suffix for a resource id', () => { + expect(normalizeToolId('table_query_rows_v2')).toBe('table_query_rows_v2') + }) + + it('still strips a resource id from a VERSIONED op', () => { + expect(normalizeToolId('table_query_rows_v2_tbl_1a2c1741')).toBe('table_query_rows_v2') + }) + + it('is not fooled by a table id that merely starts with v', () => { + expect(normalizeToolId('table_query_rows_v2x')).toBe('table_query_rows') + expect(normalizeToolId('table_query_rows_version')).toBe('table_query_rows') + }) +}) diff --git a/apps/sim/tools/normalize.ts b/apps/sim/tools/normalize.ts index 177c11aa140..c01dceaf33b 100644 --- a/apps/sim/tools/normalize.ts +++ b/apps/sim/tools/normalize.ts @@ -6,6 +6,29 @@ * * Pure string utility — no server dependencies, safe to import in client components. */ + +/** + * A trailing `_v2`-style segment is a VERSION marker, not a resource id, so it + * must not be stripped: `table_query_rows_v2` is its own registered tool, and + * normalizing it to `table_query_rows` silently executes the v1 tool's request + * shape under the v2 tool's name. + */ +const VERSION_SUFFIX = /^v\d+$/ + +/** + * Longest id first, so a versioned op claims its own suffixed ids before the + * unversioned prefix can match them (`table_query_rows_v2_` must + * normalize to `table_query_rows_v2`, not `table_query_rows`). + */ +function stripResourceSuffix(toolId: string, ops: string[]): string | null { + for (const op of [...ops].sort((a, b) => b.length - a.length)) { + if (!toolId.startsWith(`${op}_`) || toolId.length <= op.length + 1) continue + if (VERSION_SUFFIX.test(toolId.slice(op.length + 1))) continue + return op + } + return null +} + export function normalizeToolId(toolId: string): string { // Custom (deploy-as-block) tools: 'deployed_block_executor_custom_block_' -> // 'deployed_block_executor'. Note the id deliberately does NOT start with @@ -23,14 +46,12 @@ export function normalizeToolId(toolId: string): string { } const knowledgeOps = ['knowledge_search', 'knowledge_upload_chunk', 'knowledge_create_document'] - for (const op of knowledgeOps) { - if (toolId.startsWith(`${op}_`) && toolId.length > op.length + 1) { - return op - } - } + const knowledge = stripResourceSuffix(toolId, knowledgeOps) + if (knowledge) return knowledge const tableOps = [ 'table_query_rows', + 'table_query_rows_v2', 'table_insert_row', 'table_batch_insert_rows', 'table_update_row', @@ -41,11 +62,8 @@ export function normalizeToolId(toolId: string): string { 'table_delete_row', 'table_get_schema', ] - for (const op of tableOps) { - if (toolId.startsWith(`${op}_`) && toolId.length > op.length + 1) { - return op - } - } + const table = stripResourceSuffix(toolId, tableOps) + if (table) return table return toolId } diff --git a/apps/sim/tools/registry.minimal.ts b/apps/sim/tools/registry.minimal.ts index bacfae7e566..899124191ce 100644 --- a/apps/sim/tools/registry.minimal.ts +++ b/apps/sim/tools/registry.minimal.ts @@ -71,6 +71,21 @@ import { slackUpdateMessageTool, slackUpdateViewTool, } from '@/tools/slack' +import { + tableBatchInsertRowsTool, + tableCreateTool, + tableDeleteRowsByFilterTool, + tableDeleteRowTool, + tableGetRowTool, + tableGetSchemaTool, + tableInsertRowTool, + tableListTool, + tableQueryRowsTool, + tableQueryRowsV2Tool, + tableUpdateRowsByFilterTool, + tableUpdateRowTool, + tableUpsertRowTool, +} from '@/tools/table' import type { ToolConfig } from '@/tools/types' import { customBlockExecutorTool, workflowExecutorTool } from '@/tools/workflow' @@ -87,6 +102,20 @@ import { customBlockExecutorTool, workflowExecutorTool } from '@/tools/workflow' export const tools: Record = { http_request: httpRequestTool, function_execute: functionExecuteTool, + // Table block operations (v1 + v2 Table blocks are both runnable in minimal mode). + table_query_rows: tableQueryRowsTool, + table_insert_row: tableInsertRowTool, + table_batch_insert_rows: tableBatchInsertRowsTool, + table_upsert_row: tableUpsertRowTool, + table_update_row: tableUpdateRowTool, + table_update_rows_by_filter: tableUpdateRowsByFilterTool, + table_delete_row: tableDeleteRowTool, + table_delete_rows_by_filter: tableDeleteRowsByFilterTool, + table_query_rows_v2: tableQueryRowsV2Tool, + table_get_row: tableGetRowTool, + table_get_schema: tableGetSchemaTool, + table_create: tableCreateTool, + table_list: tableListTool, guardrails_validate: guardrailsValidateTool, // Needed so workflow-as-tool and custom (deploy-as-block) tools resolve their // config in minimal-registry dev mode (both route through `workflow_executor`). diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 1e2facaa404..06fdf3eb441 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -4153,6 +4153,7 @@ import { tableInsertRowTool, tableListTool, tableQueryRowsTool, + tableQueryRowsV2Tool, tableUpdateRowsByFilterTool, tableUpdateRowTool, tableUpsertRowTool, @@ -8796,6 +8797,7 @@ export const tools: Record = { table_delete_row: tableDeleteRowTool, table_delete_rows_by_filter: tableDeleteRowsByFilterTool, table_query_rows: tableQueryRowsTool, + table_query_rows_v2: tableQueryRowsV2Tool, table_get_row: tableGetRowTool, table_get_schema: tableGetSchemaTool, mailchimp_get_audiences: mailchimpGetAudiencesTool, diff --git a/apps/sim/tools/table/index.ts b/apps/sim/tools/table/index.ts index 1d4b84596c0..7b99fecedce 100644 --- a/apps/sim/tools/table/index.ts +++ b/apps/sim/tools/table/index.ts @@ -7,6 +7,7 @@ export * from './get_schema' export * from './insert_row' export * from './list' export * from './query_rows' +export * from './query_rows_v2' export * from './types' export * from './update_row' export * from './update_rows_by_filter' diff --git a/apps/sim/tools/table/query_rows_v2.ts b/apps/sim/tools/table/query_rows_v2.ts new file mode 100644 index 00000000000..6f589de7450 --- /dev/null +++ b/apps/sim/tools/table/query_rows_v2.ts @@ -0,0 +1,106 @@ +import type { TableQueryV2Response, TableRowQueryV2Params } from '@/tools/table/types' +import type { ToolConfig } from '@/tools/types' + +/** + * v2 row query: typed predicate grammar + opaque cursor pagination (no offset). + * Hits POST `/api/table/[tableId]/query`. + */ +export const tableQueryRowsV2Tool: ToolConfig = { + id: 'table_query_rows_v2', + name: 'Query Rows', + description: + 'Query rows with a typed predicate filter and cursor pagination. ' + + 'Filter is a predicate tree: `{"all":[{"field":"wins","op":"gte","value":10},{"field":"status","op":"in","value":["active","pending"]}]}` ' + + '(`all` = AND, `any` = OR; groups nest). Operators: eq, ne, gt, gte, lt, lte, in, nin, like, ilike, ' + + 'nlike, nilike, contains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. ' + + 'Order is a sort spec, e.g. `[{"field":"wins","direction":"desc"}]`. Omit limit to return the entire result — ' + + 'the query fails if it exceeds the 5MB budget (narrow with a filter or set a limit). With a limit, ' + + 'a page can end early at the byte budget: a non-null nextCursor means more rows exist — pass it back ' + + 'as cursor to continue; never infer completion from page size.', + version: '1.0.0', + + params: { + tableId: { + type: 'string', + required: true, + description: 'Table ID', + visibility: 'user-only', + }, + filter: { + type: 'json', + required: false, + description: + 'Predicate tree, e.g. `{"all":[{"field":"wins","op":"gte","value":10}]}`. Omit to match all rows.', + visibility: 'user-or-llm', + }, + order: { + type: 'json', + required: false, + description: 'Sort spec, e.g. `[{"field":"wins","direction":"desc"}]`.', + visibility: 'user-or-llm', + }, + limit: { + type: 'number', + required: false, + description: + 'Maximum rows per page. Omit to return the entire matching result — fails if it exceeds the 5MB budget. With a limit, pages may byte-cut early and set nextCursor when more remain.', + visibility: 'user-or-llm', + }, + cursor: { + type: 'string', + required: false, + description: 'Opaque pagination cursor returned by a prior query. Omit for the first page.', + visibility: 'user-or-llm', + }, + }, + + request: { + url: (params: TableRowQueryV2Params) => `/api/table/${params.tableId}/query`, + method: 'POST', + headers: () => ({ 'Content-Type': 'application/json' }), + body: (params: TableRowQueryV2Params) => { + const workspaceId = params._context?.workspaceId + if (!workspaceId) { + throw new Error('Workspace ID is required in execution context') + } + return { + workspaceId, + ...(params.filter ? { predicate: params.filter } : {}), + ...(params.order ? { sort: params.order } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.cursor ? { cursor: params.cursor } : {}), + } + }, + }, + + transformResponse: async (response): Promise => { + const result = await response.json() + const data = result.data || result + + return { + success: true, + output: { + rows: data.rows, + rowCount: data.rowCount, + totalCount: data.totalCount, + limit: data.limit, + nextCursor: data.nextCursor, + }, + } + }, + + outputs: { + success: { type: 'boolean', description: 'Whether the query succeeded' }, + rows: { type: 'array', description: 'Query result rows' }, + rowCount: { type: 'number', description: 'Number of rows returned' }, + totalCount: { + type: 'number', + description: 'Total rows matching the predicate (computed on the first page only)', + }, + limit: { type: 'number', description: 'Limit used in the query' }, + nextCursor: { + type: 'string', + description: 'Cursor to fetch the next page, or null on the last page', + }, + }, +} diff --git a/apps/sim/tools/table/types.ts b/apps/sim/tools/table/types.ts index ee3e347a590..f5f2633a2e3 100644 --- a/apps/sim/tools/table/types.ts +++ b/apps/sim/tools/table/types.ts @@ -3,7 +3,9 @@ import type { Filter, RowData, Sort, + SortSpec, TableDefinition, + TablePredicate, TableRow, TableSchema, } from '@/lib/table/types' @@ -56,6 +58,26 @@ export interface TableRowGetParams { _context?: WorkflowToolExecutionContext } +/** v2 query params: typed predicate/sort objects + opaque cursor (no offset). */ +export interface TableRowQueryV2Params { + tableId: string + filter?: TablePredicate + order?: SortSpec + limit?: number + cursor?: string + _context?: WorkflowToolExecutionContext +} + +export interface TableQueryV2Response extends ToolResponse { + output: { + rows: TableRow[] + rowCount: number + totalCount: number | null + limit: number + nextCursor: string | null + } +} + export interface TableCreateResponse extends ToolResponse { output: { table: TableDefinition diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 4e7f5f6ca23..8df1ad2a511 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 994, - zodRoutes: 994, + totalRoutes: 997, + zodRoutes: 997, nonZodRoutes: 0, } as const From d95127da672893f1359096bb7444e6674c74f646 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 30 Jul 2026 11:44:24 -0700 Subject: [PATCH 02/21] fix(settings): add bottom padding to settings sidebar (#6097) --- .../sidebar/components/settings-sidebar/settings-sidebar.tsx | 2 +- apps/sim/components/settings/settings-sidebar.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx index 13a3a89f19c..5cc8b0d5d2b 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/settings-sidebar/settings-sidebar.tsx @@ -307,7 +307,7 @@ export function SettingsSidebar({
diff --git a/apps/sim/components/settings/settings-sidebar.tsx b/apps/sim/components/settings/settings-sidebar.tsx index af95658ab5a..585ac03db19 100644 --- a/apps/sim/components/settings/settings-sidebar.tsx +++ b/apps/sim/components/settings/settings-sidebar.tsx @@ -126,7 +126,7 @@ export function SettingsSidebar
({
From 66478087f7e13412f3659bff37d7cb54139f2267 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 30 Jul 2026 11:51:27 -0700 Subject: [PATCH 03/21] fix(sidebar): stop the workspace switcher stranding a phantom hover (#6096) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sidebar): stop the workspace switcher stranding a phantom hover The switcher's keyboard cursor is set from `onMouseMove` and only cleared when the menu closes, and it paints in `--surface-active` — the same token hover uses. So the last row the pointer crossed keeps a background indistinguishable from hover long after the pointer has gone, sitting alongside the equally-`--surface-active` current workspace as a second phantom-hovered row. It shows without any hovering too: the cursor is seeded to row 0 on open, so any user whose current workspace is not first sees two marked rows immediately. Only bites above the 3-workspace search threshold, which is why it went unnoticed. Paint the cursor only while the user is actually navigating by keyboard. Arrow keys enter that mode, any pointer motion leaves it, so in pointer mode the sole mark is real CSS :hover — which follows the pointer and leaves with it. `highlightedId` still tracks the pointer, so Enter keeps targeting the row last touched; only whether it is drawn changes. This is the pattern emcn's own popover already uses (`isKeyboardNav`, commented "prevent dual highlights") and that `tag-dropdown` consumes, and it matches how Headless UI models a combobox: one modality-driven focus marker, separate from the selected value. The list carries no `aria-activedescendant`/`aria-selected`, so the highlight is purely visual and nothing in the a11y contract changes. * test(sidebar): assert the keyboard cursor on a row that isn't selected Review round 1: the keyboard-positive assertions landed on the current workspace, which carries its own `isActive` fill, so they held whether or not the cursor was painted — the tests could not have caught deleting keyboard-cursor rendering. Navigate with ArrowUp instead, wrapping from the seeded first row to the last, and assert on that row. Verified both directions now: removing the modality gate reddens three tests, removing keyboard painting reddens two. * fix(sidebar): do not arm Enter without a visible target Review round 1: gating the cursor's paint on keyboard mode left Enter still acting on the seeded first row while that row was unmarked. The search field takes focus on open, so Enter could switch workspace with nothing shown as the target — emcn's popover instead holds its selection at -1 and ignores Enter until keyboard nav begins. Enter now acts only once a cursor is on screen, and typing counts as keyboard intent so the common "filter, then Enter" flow lands on a visible top result. Every path to Enter therefore has a marked target. --------- Co-authored-by: Waleed Latif --- .../workspace-header.test.tsx | 237 ++++++++++++++++++ .../workspace-header/workspace-header.tsx | 39 ++- 2 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.test.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.test.tsx new file mode 100644 index 00000000000..a315e751edd --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.test.tsx @@ -0,0 +1,237 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockNavigateToSettings } = vi.hoisted(() => ({ mockNavigateToSettings: vi.fn() })) + +const onWorkspaceSwitch = vi.fn() + +vi.mock('@tanstack/react-query', () => ({ + useQueryClient: () => ({ invalidateQueries: vi.fn(), setQueryData: vi.fn() }), +})) +vi.mock('@/lib/auth/auth-client', () => ({ useActiveOrganization: () => ({ data: null }) })) +vi.mock('@/hooks/use-settings-navigation', () => ({ + useSettingsNavigation: () => ({ navigateToSettings: mockNavigateToSettings }), +})) +vi.mock('@/hooks/use-permission-config', () => ({ + usePermissionConfig: () => ({ isInvitationsDisabled: false }), +})) +vi.mock('@/app/workspace/[workspaceId]/providers/workspace-permissions-provider', () => ({ + useWorkspacePermissionsContext: () => ({ + userPermissions: { canAdmin: true, canEdit: true, canRead: true }, + }), +})) +vi.mock('@/hooks/queries/invitations', () => ({ invitationKeys: { all: ['invitations'] } })) +vi.mock('@/hooks/queries/workspace', () => ({ workspaceKeys: { all: ['workspaces'] } })) + +/** Modal/menu siblings are irrelevant to the highlight and drag in heavy trees. */ +vi.mock('@/app/workspace/[workspaceId]/components/invite-modal', () => ({ + InviteModal: () => null, +})) +vi.mock( + '@/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/context-menu/context-menu', + () => ({ ContextMenu: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/w/components/sidebar/components/workflow-list/components/delete-modal/delete-modal', + () => ({ DeleteModal: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/create-workspace-modal/create-workspace-modal', + () => ({ CreateWorkspaceModal: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/pending-invitations/view-invitations-menu-item', + () => ({ ViewInvitationsMenuItem: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/components/pending-invitations/view-invitations-modal', + () => ({ ViewInvitationsModal: () => null }) +) + +import { WorkspaceHeader } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header' + +/** + * `@sim/emcn` is deliberately NOT mocked: the assertion is about the background class + * `chipVariants` produces, so a stubbed chip would only assert the stub. + */ +const ACTIVE_BG = 'bg-[var(--surface-active)]' + +/** + * Above `WORKSPACE_SEARCH_THRESHOLD` (3), so the searchable/keyboard list renders. + * The current workspace is deliberately NOT first: the highlight is seeded to row 0 on + * open, so a current workspace sitting at row 0 would mask the double-mark this guards. + */ +const WORKSPACES = [ + { id: 'ws-rvt', name: 'RVT' }, + { id: 'ws-emir', name: "Emir's Workspace" }, + { id: 'ws-acme', name: 'Acme' }, + { id: 'ws-globex', name: 'Globex' }, +] as unknown as Parameters[0]['workspaces'] + +let container: HTMLDivElement +let root: Root + +function render() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => { + root.render( + {}} + onWorkspaceSwitch={onWorkspaceSwitch} + onCreateWorkspace={async () => {}} + onRenameWorkspace={async () => {}} + onDeleteWorkspace={async () => {}} + isDeletingWorkspace={false} + onUploadLogo={() => {}} + onLeaveWorkspace={async () => {}} + isLeavingWorkspace={false} + /> + ) + }) +} + +function row(name: string): HTMLElement { + const found = [...document.querySelectorAll('[data-workspace-row-idx]')].find((el) => + el.textContent?.includes(name) + ) + if (!found) throw new Error(`No workspace row rendered for "${name}"`) + return found as HTMLElement +} + +/** + * Whether a row is painted with the persistent active fill. + * + * Matches an exact class token, never a substring: the inactive chip carries + * `hover-hover:bg-[var(--surface-active)]`, which *contains* the active class, so a + * substring check reports every row as marked. + */ +function isMarked(name: string): boolean { + return [...row(name).querySelectorAll('*')].some((el) => + el.classList.contains(ACTIVE_BG) + ) +} + +/** + * Types into a React-controlled input. Assigning `.value` directly is ignored: React + * tracks the previous value on the node, so the change must go through the native + * setter for its synthetic `onChange` to fire. + */ +function typeInto(input: HTMLInputElement, value: string) { + const setValue = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set + setValue?.call(input, value) + input.dispatchEvent(new Event('input', { bubbles: true })) +} + +beforeEach(() => { + vi.clearAllMocks() + // jsdom implements neither; the component scrolls the active row into view. + Element.prototype.scrollIntoView = vi.fn() +}) + +afterEach(() => { + act(() => root.unmount()) + container.remove() +}) + +describe('WorkspaceHeader workspace switcher highlight', () => { + it('leaves Enter unarmed until a cursor is on screen', () => { + render() + + const search = document.querySelector('input[placeholder="Search workspaces..."]') + act(() => { + search?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + + // The search field is focused on open, so acting on the seeded row here would + // switch workspace with nothing marked. + expect(onWorkspaceSwitch).not.toHaveBeenCalled() + }) + + it('arms Enter on the top result once the user types', () => { + render() + + const search = document.querySelector( + 'input[placeholder="Search workspaces..."]' + ) as HTMLInputElement | null + act(() => { + if (search) typeInto(search, 'Acme') + }) + // Typing counts as keyboard intent, so the target is visible before Enter fires. + expect(isMarked('Acme')).toBe(true) + + act(() => { + search?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + expect(onWorkspaceSwitch).toHaveBeenCalledWith(expect.objectContaining({ id: 'ws-acme' })) + }) + + it('marks only the current workspace when the menu opens', () => { + render() + + expect(isMarked("Emir's Workspace")).toBe(true) + // Regression: the keyboard cursor used to be seeded to row 0 on open, so a second + // row was marked in the same colour as hover before any interaction. + expect(isMarked('RVT')).toBe(false) + }) + + it('does not leave a highlight behind when the pointer moves across a row', () => { + render() + + act(() => { + row('RVT').dispatchEvent(new MouseEvent('mousemove', { bubbles: true })) + }) + + // The pointer has moved on; nothing should be painted as if still hovered. + // Real CSS :hover handles the row actually under the cursor and leaves with it. + expect(isMarked('RVT')).toBe(false) + expect(isMarked("Emir's Workspace")).toBe(true) + }) + + it('shows the cursor once the user navigates by keyboard', () => { + render() + + const search = document.querySelector('input[placeholder="Search workspaces..."]') + expect(search).not.toBeNull() + act(() => { + search?.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowUp', bubbles: true })) + }) + + // ArrowUp from the seeded first row wraps to the last. Asserting on Globex, not on + // the current workspace: the current one carries its own `isActive` fill, so it + // stays marked either way and would prove nothing about the cursor being painted. + expect(isMarked('Globex')).toBe(true) + }) + + it('drops the keyboard cursor again as soon as the pointer moves', () => { + render() + + const search = document.querySelector('input[placeholder="Search workspaces..."]') + act(() => { + search?.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowUp', bubbles: true })) + }) + expect(isMarked('Globex')).toBe(true) + + act(() => { + row('Acme').dispatchEvent(new MouseEvent('mousemove', { bubbles: true })) + }) + + // Back in pointer mode: the cursor is gone and the row just crossed is unmarked, + // leaving only the current workspace's own fill. + expect(isMarked('Globex')).toBe(false) + expect(isMarked('Acme')).toBe(false) + expect(isMarked("Emir's Workspace")).toBe(true) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.tsx index 8a14799949d..2357a7a4e42 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/workspace-header/workspace-header.tsx @@ -169,6 +169,19 @@ function WorkspaceHeaderImpl({ const [workspaceSearch, setWorkspaceSearch] = useState('') const [highlightedId, setHighlightedId] = useState(null) + /** + * Which input the user is currently driving the list with. The highlight is only + * painted in keyboard mode, because it renders in `--surface-active` — the same + * token hover uses — so a highlight left behind by the pointer is indistinguishable + * from a stuck hover, and sits alongside the equally-`--surface-active` current + * workspace as a second phantom-hovered row. + * + * `highlightedId` itself still tracks the pointer, so Enter always targets the row + * the user last touched; only whether it is *drawn* depends on the mode. Mirrors + * `isKeyboardNav` in emcn's popover ("prevent dual highlights") and the single + * modality-driven focus marker Headless UI's Combobox exposes. + */ + const [isKeyboardNav, setIsKeyboardNav] = useState(false) const showSearch = workspaces.length > WORKSPACE_SEARCH_THRESHOLD const searchQuery = workspaceSearch.trim().toLowerCase() @@ -228,6 +241,7 @@ function WorkspaceHeaderImpl({ if (isWorkspaceMenuOpen) return setWorkspaceSearch('') setHighlightedId(null) + setIsKeyboardNav(false) }, [isWorkspaceMenuOpen]) const [isMounted, setIsMounted] = useState(false) @@ -526,21 +540,33 @@ function WorkspaceHeaderImpl({ icon={Search} placeholder='Search workspaces...' value={workspaceSearch} - onChange={(e) => setWorkspaceSearch(e.target.value)} + onChange={(e) => { + // Typing is keyboard intent, so the cursor appears on the top + // result and Enter has a visible target. + setIsKeyboardNav(true) + setWorkspaceSearch(e.target.value) + }} onKeyDown={(e) => { e.stopPropagation() if (e.nativeEvent.isComposing) return if (filteredWorkspaces.length === 0) return if (e.key === 'ArrowDown') { e.preventDefault() + setIsKeyboardNav(true) const next = (activeIndex + 1) % filteredWorkspaces.length setHighlightedId(filteredWorkspaces[next].id) } else if (e.key === 'ArrowUp') { e.preventDefault() + setIsKeyboardNav(true) const next = (activeIndex - 1 + filteredWorkspaces.length) % filteredWorkspaces.length setHighlightedId(filteredWorkspaces[next].id) } else if (e.key === 'Enter') { + // Only armed once a cursor is actually on screen. The search + // field is focused on open, so acting on the seeded row here + // would switch workspace with nothing marked — emcn's popover + // likewise holds its selection at -1 until keyboard nav starts. + if (!isKeyboardNav) return e.preventDefault() const target = filteredWorkspaces[activeIndex] if (target) onWorkspaceSwitch(target) @@ -562,7 +588,7 @@ function WorkspaceHeaderImpl({ const initial = getWorkspaceInitial(workspace.name) const isActive = workspace.id === workspaceId const isMenuOpen = menuOpenWorkspaceId === workspace.id - const isKeyboardHighlighted = showSearch && idx === activeIndex + const isKeyboardHighlighted = showSearch && isKeyboardNav && idx === activeIndex /** * Hover-highlight is wired to `onMouseMove`, not `onMouseEnter`: a @@ -575,7 +601,14 @@ function WorkspaceHeaderImpl({
setHighlightedId(workspace.id) : undefined} + onMouseMove={ + showSearch + ? () => { + setIsKeyboardNav(false) + setHighlightedId(workspace.id) + } + : undefined + } > {editingWorkspaceId === workspace.id ? (
Date: Thu, 30 Jul 2026 12:55:54 -0700 Subject: [PATCH 04/21] feat(model): sim auto model (#6103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add sim-auto: automatic model routing for the agent block (hosted) - 'Auto' model option (Sim wordmark icon, hosted-only, new-block default; runtime fallback for unset models stays claude-sonnet-5) - Resolver (lib/model-router): classifies each execution via mothership's /api/model-router and routes over two ladders — text: fireworks/glm-5.2 -> fireworks/kimi-k3 (new static hosted Fireworks catalog entries on the platform FIREWORKS_API_KEY); attachments: claude-haiku-4-5 -> gpt-5.5 (Fireworks OSS endpoints reject images). Trivial tasks skip the router; 5-min decision cache; 2s timeout; never fails the workflow - Hidden identity preamble on every auto run (English by default, don't volunteer the underlying model) - Fireworks executor: wire-name map for catalog ids (glm-5.2 -> glm-5p2), pricing keyed on the full catalog id - Billing: routing cost applied to non-streaming output cost only when mothership marks the call billable (bill-model-router flag, default off) - sim-auto special-cases: API-key condition, serialization tool lookup, edit-workflow validation (hosted), VFS model options projection * auto model * agent auto model * fix lint --- .../generated-icon.icon/Assets/border.svg | 3 + .../build/generated-icon.icon/Assets/logo.png | Bin 0 -> 24076 bytes .../build/generated-icon.icon/icon.json | 33 ++ apps/sim/.env.example | 6 +- .../api/providers/fireworks/models/route.ts | 9 +- apps/sim/blocks/blocks/agent.ts | 9 +- apps/sim/blocks/utils.test.ts | 2 + apps/sim/blocks/utils.ts | 19 +- apps/sim/components/icons.tsx | 25 ++ .../handlers/agent/agent-handler.test.ts | 125 ++++++- .../executor/handlers/agent/agent-handler.ts | 185 ++++++++++- apps/sim/lib/api-key/byok.test.ts | 92 +++++- apps/sim/lib/api-key/byok.ts | 29 ++ .../workflow/edit-workflow/validation.ts | 9 +- apps/sim/lib/copilot/vfs/serializers.ts | 17 +- apps/sim/lib/core/config/api-keys.ts | 10 +- apps/sim/lib/core/config/env-flags.ts | 11 +- apps/sim/lib/core/config/env.ts | 5 +- apps/sim/lib/core/utils.test.ts | 26 ++ apps/sim/lib/model-router/resolve.test.ts | 288 ++++++++++++++++ apps/sim/lib/model-router/resolve.ts | 308 ++++++++++++++++++ apps/sim/providers/fireworks/index.test.ts | 22 +- apps/sim/providers/fireworks/index.ts | 19 +- apps/sim/providers/fireworks/utils.test.ts | 18 + apps/sim/providers/fireworks/utils.ts | 20 ++ apps/sim/providers/models.test.ts | 40 +++ apps/sim/providers/models.ts | 93 +++++- .../providers/settled-tool-streams.test.ts | 1 + 28 files changed, 1391 insertions(+), 33 deletions(-) create mode 100644 apps/desktop/build/generated-icon.icon/Assets/border.svg create mode 100644 apps/desktop/build/generated-icon.icon/Assets/logo.png create mode 100644 apps/desktop/build/generated-icon.icon/icon.json create mode 100644 apps/sim/lib/model-router/resolve.test.ts create mode 100644 apps/sim/lib/model-router/resolve.ts create mode 100644 apps/sim/providers/fireworks/utils.test.ts diff --git a/apps/desktop/build/generated-icon.icon/Assets/border.svg b/apps/desktop/build/generated-icon.icon/Assets/border.svg new file mode 100644 index 00000000000..fb9d4f8c789 --- /dev/null +++ b/apps/desktop/build/generated-icon.icon/Assets/border.svg @@ -0,0 +1,3 @@ + + + diff --git a/apps/desktop/build/generated-icon.icon/Assets/logo.png b/apps/desktop/build/generated-icon.icon/Assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..6fce2046095972fb6826093ed9ab7530435da39c GIT binary patch literal 24076 zcmeFZ_g9lk7dCv;L_t9h2T>8wLkJ2|6al5%30)zfHw7u7OYh176)Ad0ic}R5NJv6U z=nzD~h=K?S(xXICssuw1`EJg$-tRp;f5H31!&)wo$;|9Nd#>4=`xa(b_6Qvk0svqS z!tmlX0N{syKI%mOi($#JEJ+0EBQ^~so#NZeI@XXn7#%h0CPKu7l?3xQ3Ni!-;1RZ93`XZTwOjm{jXw3Ahz2y_cb9_tmpB`+ z1b+4ibr`eZ530brRWMDtdUfh+;D!@8h#;5;1q$rcvu*o3^I*l*0p~xV z86CYlNMFZMQ{%bJzhSNA)b(;6dCr4FwQ}`Z)QAlAUu7^sxrp;LVVkmP6;bH(MTFw7 z#C&FN#&>Q}PI-_0I|TP@N1o7*!`3`^l`!Q)?Ay?X;;)kIkPC4L@CLZ?7ZureX2Ch2 zW|k)o!NSAt!Uxr;9f-{NwWrW0DaX7rIP>;pEBb|P_qnfldE)TA2}(Ny1_+|Nq2ZL| z5Ku>hs*TyGyS6E>b@&!0FPEu?D9ClY6~Y6JyZn^rYQrL&!3W+f-Cj)K1yDsU;xhOl z&l4KV&O>?n*J4@R=>pTu$#h{%^hs(p=#%YcD=!HR@2I^-8quzS560we^vDdrSHnR^ zuzViI2T{eCD`dvQ~o-zvLDPiml5X)Ra4`OvQjcU;kP2xohvN8E! z%QeldJyzoEiZ>+WSA9Xrts!h+1Xv3ne0v|tq~hIXVMODgt6+p~cAYuRvn^#=9&(a* zW~ZTf6Mhpge_YI>nC$8AR*nf$U^$I_ZflaVQlZ-e*gPW0Q4~IFdDhRAH3(~j+m5?K zgOoG&_1Bo`CPD2`rj}-0Rid>;LP{eNGonuwyiyJ@HULT%sB@GGca+)E_>WKC!u-JH zlY9nS9|XX1&}|xq^%Q0wiM)p&vxtDr^I`vHW`@9bW0$JZc!KP_2i@%Umdv% zvbp%c0eGjN+^Rh|9H|-zad?ZS@wtJ#wKc^CjhWz@u^M5TY?rTk$cf~4K6PmK{|wnH zQ}qxnh0MY&?%!nRGdEy@aNDA3zAZbB`H1QaBXU@AGi}Tmg$^Vl7Gy>sf9!>Z3|S9h zg7Rna|7bx1wi*p~8gm7=)H8URYKh5H{VNeLH)nOghPA2C$A78d|!??N2Z`sWz(H*06k}84OC>9i3 zrKU`S8>gt7w6QHBZ0@bYBfLP%NYy8W?^iB>vY3s#)mujOAhZM~inRj<*e>T=VOzF_ zU@>7~x4QT%f?i&~(}`5}s&4iV73{b1 zG9}DX@mwUe&_Qn&Dp{YjMupc)6IFp4;kpU zCXR556Jji%S(6hVc5{DOxzn!EHJF7=+Lxd7jw(@qDoY8t-9 z;vzVr^G1vtflxLNiUgODQg=S&*+QSlwnfA1olW06W0XDxGW>CQ_-?Bc=!@C#059Nf7~sTy4Z|epMw{#m zxsUMmYK1caA)qm~V!Q{FXLI{+YxtlnH4}jfy$mj`UrI$bNCTcibUj!lRj~4V2#KSZff}o6q5{EmF#C%QKnU)U?Z&l3K#{i4+Eo+ z6)u=}(l$k?HTx21X~5;4l6BOIHMX)zBNqDazrXrgdZJbpfLccE{OXN2YwLK!!xWR6 zvb;ERyU(0bPRO$CcW1>erLU{%T)Tx0UZCUq8_z)jr!>1aOEjG|>I&FJ+>x@px-&N_-9f&Vte6?6GW5UXuf87s6-W-I^JsZ#wGn2tALlk5%R}oHm%=oE=SVMM1Tof247ZC<7 z!y^1smucbt9QM+(C{CZ7ImO4FT3c>DMvX-*ZFx!*Y&9J=EMM>3ob79+;yWuKav(;r z|3z$3PPT5voM|I;wt^n82&Yl~@8elx+02iy`KD1_ueSBxeFza}M(pqJkVgf%Y-#Tf ztjDY>41}_YQ`7{E4|18R5$I4nozUK@+eI&BnCDjLAEAmMq>(+9CGrtwAlHb+id;JE z&{GlCPYPI>C1kA*yn9vP>~F}z!c?Gh28@}G^cnb8gwgqnN)|V)ixPNgS78t@3H)%A zd(qrPuvByt#b&)Ifkn821O39iSGSz!HLL`-8Uc>O&T3K<5z3}Pmp~lG0mB@E4an7R zhE6mb8lZc|ZL(l-=BzB(BVIL?jnN-a@fjo5Tp7-F+Yij%}304+I8|DT2(%@$~a~+ze3K#T8EVk z=+3Xe3QG|TN$=;iVFLVQ0`+EYf4ykPvW|TvA>)IBVRe*1xo9bsa~F(OrfP1bQ7T(G z(VfU0AL5cew4w--zg9yK9PMr+77kWIXMZMirK2hsByp3WM&6-i-0nN-3UzV^pg2;G zRD84G#QuwuZCC86cvFz}j&m0S9i+PuE!HZhX61g0A~qdVw6X?-*?SOvh#)k0|M-Dh znD^K?|ES6}5AXq)m&L3_#AY&?u<=p}ToGFRC4Qs{Mpc{E)_rkJeE>)0Q$36-yZ@U` z@?LwLbGn&;a@0q)a#qFaocmu)tBC*IiWA z`~O_A0++47Q;etee*dsrXujgEls#hFvsmt)FrrB{Lljs&@gm}Ph2@6FJu8sVHw^0F z?WtVFaHgl8sD&jLIBQxzWxQ*Uo)jd-Ire;-M_DvFs>zhkJhn!4P$s5x#v59!*C98@ zR0S2Y$7|tl*J<97QBO8e&3O)1Dm|K}yr*7GcR6eLTd#i)1u<(xOzf{@RXhH(2$hR}qFF~Cr%>R7w2`s0N+Ndxl>0#*GspbPMAFQo) zZ(pGq({^M8?F#mo9TIE4#nBs{|NXY%O1@Ruh6IY-Yvq2>bh)mcEL9UVZH|e?^Wy!> zINyC=HJG}eEp9^lz5554KTT6fcs_p@-?V8+RSbG|ZX%Bmdn zZ0t>!a8EyLWE?z#+XJj{@pg))qWBn=e_7VgKE)7j$I-|~1<5CCPw{sXgGWV?$#Z_l z1K8Q-zsvGU%dOrVAHY@m-Mz8tVNO9;@c!IAS zkm5g#3z0CKjdL|!ODDvnd-P*pkz_ZXpZ+1#rLU;F%v89!GG`=}U+6vfDc!bC+uAg7 z-6{~D>H0|mH{db!Qs_LP?~BmjP>7Ol5&83bqR1F6+&Cu9!^%5mJ=)Rs*${5l2FDQ` zh{3wcatQFZ)lmV&?CwBQfwQsKqWZ>dZFw4L^Ek(@jn;{ZOXBaMoW$qXgT=4w;68o7 zqGVW8XFx$BqrUdP3qR9z=V%mCK4m{s>>_Xc$%^xA@>so{sFY)3nKE9ygy8J3R&X!? zS!^$D@B1_qTjQW|jq8V-5~}nUKQsKyNvqMs)u|HosJ3j^x7$d^^#ZAzp+0tnU;5Oa zH8)II2CEYYJqu+DF9%T98Y}Dsyb3HX^|UxD)@iaCP9ML$o~wE%v=YD}lgl*9yHWxT z*3D>D^MfS!S$UGF@8?fzC5sL9Ro-JY*PDr@2Gq;QX+hw?Nk!TVt}AWmhbcRVn=rH< z8zSmtk>vG$T>4kL5Pwq(ya6x%9GMX}Sc!V9kMuo~UUq}s^zLbR04>iZ>=0@^t4zGD z(Q3!5G!l!wfkiKTQD|72z`km6ysJ2B-~aksuCT+6+K@BfMb^6fCs62hk4>X`(a} zWKg*y7qAQ_IYed$aG8>v@pS~n8oQc!M;Yt=%CR*iZsNZnypU%%ve42#Q1Ec5%UC(Hp>mDTirSAgWuJ#de z;l6x7jDe@36R!`70Ob~rS3+@0JIC>e)+pN7p1urP5D6cxPa>31Hk5kr;)$25^(j89UxVr$dd*d&k%_ZjwaFSJm20;Y zLe^?>Or)*6y6jU|H%AEE+cWF$;e-~2EN0&^gi>TcxnC3qxp(@!x(^48>xCc`^OQ!r>5WMP;5gd zWZSbsS~GU~hN_7z!nbgJ7il?WzJXhF{dSSthX#omOVVDuhf_0an~sGEu7_(;&&Dk! zjvc6ra!36*1OOLah<}I-Zr)jNE$V3Th3@ylw2_>V*h+Y{=#1?`FDREs#v>3qB2pOpzUpWw$Uj%Y$jZo zQtkapx3?(pI=+HTr35!z0p$l)^%@C@gw}P{bUN?C%38h;^5DrLmwcfeEe&@fvwkyd z(m&O;#Ky_s#yPs3M=<6d*98g5ydX!O0wp?>aVKLW6HpSFj2 zpOjeP{u2OntGrD&m+^o8UiwSvL+kpk*S;A+u|eYYLtgL6{3pDCrw@(-s*61F-_Q>} zS$dNy1tLwpzuX*ZA(5{}m)&`YKk&W6wm5m9<=I@(HIB#J*vjNPw_)NOPZ2pE;`q}$ z@%*@;*$BPLC3_XBQ0YldCnI^NZ{v~Oo1st>ULIZ`;b4gpfVOsoPMd`~g}q+1xX7Lx zT`8+(0!PE}-3Qtq_L4(5%6b;I2G+kBnkW58zMI#UZ);jlW|EsN9vJ{XJ~m_Dj(beY z`G<-+9ZI<*lhCCY!w#WUH{eO|&ZJqflR{{NHp6^fA z>6>s##>f=*6R+BE3Sz$`5bXEF?h-mNLeDPBqj_WZ{Mbkj?M106aODw%j9k_QKAy@b z$Ypj#l9sPKtHXaB0aV9$;_0z|jSVfWSAC@$H_bY@ighdL4a-f> z=X4!W5&qoqFruFT-t65Iay5HR%a?Q@)+&T2lt9@7sG3Io#Vz^NsWRVY9Dg!o5YR7E z7!W;JDK5?vzaRqmShJOfbi4K!Z>FNXwL5U2&yee0#vJnPk{am}c6#Fa?$F)cw?ubJLH$F~g0#n61SwhTKs ziP~2}kXh^aEtTLb7hw(NN803}Pw@jCvOvJP3q?fyy;if}cw61XjQ~gd_m&3Kgonb; ziU>z%yvFN2-L#KRI1g*n=emma`+)2h>bOr_S41qqddVh5Qeo$>G$G!amf(|)p^LKRX#6!STv)#2=!GiZt-;gz|V3aUI=PDiO^^o zU6n5O^#xRY>E&4!Vb;&;&gzE0l^k{BHJN;{5-bKBAOZ%)NA>H-)xn&>R5|?}*47oP z-_-Bt67CO=Vlf{k@f+#=A#3X}8IvmebS-0mVqrr{o~p^#ly= zMtG8MhK+n~uT%-rI+yMQd0mRTV&c-%smds0QEBGo+D--*rfN88AKHtWWQuMRa_6LK3iUu>vxNAS?d9_lSU3-)LI58 z=W8T>@@=6mB;D|PlAJR;l=l9912e}qSGQjTTx$4$Y1&ymIF+rt($vu}o#1ROm4kZV z=2eph&$S9zA5l0o>5#f~(sS@Z=K!ZjfCq?AGyw`M#UITU&Tf7yo(^26RW3pltQIND{Dl=ABU^&1ju%^H6_TqDD~xn-r+JnSb$lbzxVy>Uf{dDJ_n%c%+ZI%-m_} zKQ?Z}j`vB-s<|+=SzO$6ydGcosX3{1pbDUP?f`~Di&N2;meWzeS0ZafPD5DGV_2@) zSLu&;c0^eRtiO_umbq~={Tz-Ra1mVJWBcn6@bbeUFXOo^H|&G+J3r+8|f zquUJtm(zT}Y}~x%xvt#EUuRQ)>h4hD`#FJFDvxJBMVYhWVe{oQmsy`qwcOb&8JDU) z{NbPts8&g5Gc(xnhnKVd-oKG;uX0~l^44_Ed&vpjYtV%gpYwRn1y?zq_+DY4hQ{qg zO4cq${W_TXQ-8-TG;f^Y>^J`b%8~D3cRg<#PkJ>~1F;sv25vlnDxF>)!yPwc^+@Zy zC_%q3d!7_Wq$lCce;pM6z2amVnMOPxR%p!nnd5NSe!`MHpl2QT9mnp8CMxWy)x~-s zcU+3$=~B`b}E%^3p*2SGyrSJXzFX zB%#MYIBNHhcDEon^Uw{C&@%^IO7V74U(*ivK98$KiZgKDpW-g#9pcE52CeGU+fX>Y z-!n$Su4F5mpbuTrt-8EE-w2Bc<~(VhE*xzABs8AMY!^R#rdSAcymmYO>gct=YhV_v z-ha&$k~S~kOACuF0p($Qf=%CdGez9Ra zrQ=RhwV`hA3sB%ld7{5VlUDD){JRlmdvo8wqOL+f@T;Apw^EDscW_G+<30AATRH@o zl+hs&6Dc0?Cw8-^?9ZM;t9P4{-k&Ni6ekm|W<1Txe|DqM+tg-gYDF61RKP5?Q*U*- zsE|31vR4qnXU&qo(y#CW?NY#*XLn2_Q46&ktmX#fcs`dkI~kcnnIvO@n-VYG(HwtJ zaIUiV|I`9h=gSZLoF&{=1e+Qg22UW)A%FvRT}Z9jX*4N%>0!&;ZmjNd34L>;*6Mbl zE*5uA@enyQW}HxCK)H;VbG8-a+IDg#kPl0JP3!6fJA=GZv;ewnsS4&M+aD12&YV@we0?5Io5CXdJHVie;4Q|Uz;Ys z8CeMQ>HLw4Si-MM11U{=6dXBy7j#w&Et^}1^%&LzMjuC$o+@yHNMQX9cY$aEs?_gi z+sXWyl^B|pCr_H*AHab*z+m|N;){ls0?UIuVi$5wRx|kLGB25Y{hn?=rKd?0ENmtU zjM`nMRr`mHJ2#WvV~$bx=&s~pq!y>N>dU-;*UU#XGnT^7HYh>ZK5 zEEHRVriT2U@2qXskN%4~*Y|sbC*En0fO=*=-Dcbkra>_DOQBV%$ISb*jsQ-Y^;%Z{ z5GkkL8XU0FntsCOz>|Dxvib_GadEV z_(1>uIWOnp%@@x747pg*JoMN_?KfFws&21|4Y;JU#Wk%CwlQUqwExi%-sO!6xTOJD$5; zhS*_OyPT7?@OM$jp*YA3EESu?bfv4OoARw^JzFi7KMq)VuT4P8^QYMiR`7PB ztuouaYb9riqh)sMWAz2;_V5UcQNBWu`G$m?`=Ri6*i8Z6o2dcALvK^o@@Yw;C$(R{4D&p#0ha z_~Gxrn?|3H2{b2njo9JHll|C@EcCa{R2q+4Vb0^ZYhtVGw1v?ZWEZ?6hD5_AbSQ|Q zu`ixVDt8hCLvTrkIAY+u&~BY3yA1NK(M7ayziWolul$^T$5VeLUKTv)(f=YeS1WWK zm3eV|X|opnonHD+TZdjRuak@(jFaw2$dsPQ zH@@tHri{EZSVMPSvLa&F_zDTh9P{!c?;9@IRNus79T#3q)h#9p5mK%ndCJ+}a&7do z9iAUJ@PxzVTCV?GcRD;ZG%XlFAIJTp=N$N4&oDO&>NE?2f7zE@x&{4 zl;;@6)}st+iRZoBHE!E~{@X7%QY6oYCykKKK9<^rGkY_v@S!y%?4W>Re^^A2KzsM$ zUOJ}4`XqIK(~kH@;qm@X0R?k$Em22>l zG0=Y5fug{Btkt9#_SBlCw?OdYp@h-0ssjaft-Y4m5Z8gu*z8Vm^#y%wvg(|L{l&|b z8XtFrl9^ktFoIsm1$3 zO8e%5sUmXo^N`u+yLTQ~l49__x;m6~L7>QG(fd*G^&T9T`}~u9F+*u&iF7I=QnZ`) z^|IpUrC}^~Wa&s4lpLu}90KZUuA_4YAo$0ds+7MliazJB}?rx z_O}NG)2|Awep4`5Vk+c?b?o&~JRIy``{bnkQqx;`VC2~>_DHXJhl5t?)zGOt3!m|X_3aZFojOQ5>}2mA45xr{meG!4Tl!P|2FBvD%_T03Ai|k({#v7zO^w6xSP==-RX2=R>8a1hu-|< zZ<#UL&->G#NZ2d6bc8ZktgaCLo35o?^l;b5s@qO^Zqo!&W#B|=Gp>R*9{DT4@pD~i z^8CywMt34+`LofWZmr;?-_;4SH?Duk#rM;~0+dUSH)v}r7bt$aT zTYISC$WGpkb!O}AzBo(v1DVPA-a9D#!~*>t{{d)ge{Xbqx1)>=V3Q*P^bo%W#wh&` zj%>EleeES}h$b$-f{o8cnBKL-iWm%((Y64Xy8xvLYrgPosJ*E-o5)Z;vGdpG9Doz~ z8r!7uq3$^JVtY7``324u-osn7KBGMqq4$2+a1Hf(KLF2ozWgrv{c&?OStXs>Ni^Sa~gwBy&}Z!=0>a8BBZV!T#}aPUzqsL-zuswOzy~CC)OV1x7k6PMgQ<& z?>#@GsO6*!!42~OMRfX*Ppd@ch*`#^(3_&&-%L`^($0Dm!g3ou;qHw?FPHkH^LnT$ zOMX@UV3ZaY*p1nvc!!f%fVx%CdL=rYAlm=1AquIW*HSVW_I&9YRLPYfk`6(<;X8ou z{R)1H7gvvzHDSoHAv?~5np6RPAE+b& zhYeH3jDKHvi7ip>?YnN&*>CJ(e`)3pKv@w0l<{=mYjq`NU0)+2LE;`)t>q&(LPuo3XyaAkH}jw}FzG_K zM1ttjPFpzjvbfF9>091c2hqNg6k|*|oqdi!{@kcOwF~`Rx3_(Pt94G6Dmoi(N z%G#9@>x6lM2{FL)Ttzx6@I9$ywc(JpQto6s@`V5Oi~ghjXE9`UJ*oh{wNZ z#dr2#htp9+#snhtPb+?xY+fK{j5oj=9DqewZ1JzFLG4AvW-*hnHPNP`9eHay^7OHx z_2A&>f56NmMV{Y%9x@-oB+?D8#QV!fIE8$0jl^FY-t7JnI6X8xz_a2&qWBEx0J}~1 zQDlBI&NWo@3Jo;tQrjE!EMy;o!%LrAHK<<@X{AP^j_J#^$yK7=Ij`eS9U&-|15oym zl7~?*2lBp8-;WdL)A4=#fM?8ty_;-Yl_aJuYfqQ99^Jn=lCY@%)-zGK@UffNfFRKR zblw*jj&&@{vaH5JRh*5eXU!TKvu&Tt^}3SSLCx12p!b9!p7?r%vLUNsO-fa52=%~R ze$`TxaGRQPR(+SRt><)|=gs#Cn5JiUu83VwZLv)Z9p>x0F><{Oi=&e(4R(fzW<|3HZNJL`?0&h;Fk>?S8N zA{777Z1)!^i#s;UeBE8Dt{}`PLb~}spDYr;s#jrmk=I~;6u(4>Aqk51CZ^hBHq@zY zQgfmnzjtkssBlQ_#vo~Ryk=D_J zWw>{R^Tdy#F;kte)${3=vwqmWOz>iq{02Rj6c~ia?Ez|HMmjzFX!pexe|$<#a~e-T zyCR6W`|HAAH0;ZF=c(a+(<*ZZ1-6>7MBXIx9N0gtTyb8lL{Rk8CDP#PO+4dxReLa` z;8B3`jb^>!N2+@J-i|2L;ApkXx_|VbHJ$wE16E&G4V^fo7Ze0`MF!}e@Y{mf6>a!| z=GvRHq;*56YVp?*$9_*7AOj^_1A-}!V1Q~4Pkipw*up0m&U8`z^(%Z1Q;t-)nI!EX zmXJc8-$+L|9i1+>jFMX&*;~W6aRI8mi?FJ_D-7)D6Pg|^xpsPe1h+cnz3caL82W6z z$w`RU-M~h3)!hQ8QFwMl7foGlw_?5U&D6YWiXRENb2?RHrrk4b>XmhYW?(uk)Cl;| z><*z%_SkQ|7kd%_nNp)hZywbQ2tQN8TLAr6z%vMMm*!Q$~XA{|Jw+j}+)NcX|d4yAzo4tO9 z*(_@KefQMXS}T`msh@j8`pTlDr`c{uTf4y&k|eYec~1tahxZmGnupMl?4axK!Br!{9wa0pXPVF(61Ai=z76{ z*IzpX5PHt)!hnk>nMVijG^O&htvxvKh7((0^P;Yg=F}SFPvPjIGz|7<`=gfRc<{TW8^=OMCf5eqVnF^x-%ExdM@yfH1ee}s4Z>YY)&PghU zTr`p=o}B;l2W)siL%yLJ334#eM8SbKQmJRvzl_umF+b(g>nCoLV%$P|_yDLd?#=^& zc-K!(peTa12+2&)cRnOkTwnw$QFfFuNXh{yKLh~3(yx%=PUL$d4P#G6`cLIRwV-)s8d^^U0mLNFghFZ7lA|`-7H+ocz4WWnu9UzV?0|=+bZtkr~-FVA< zz;1zFuUn_LU;U=F_K5Y5Q+<-t$mwx?c6i`sVz4Llvg|uq&u} z#@%CIU&li#R0IOtk$K~_OZ)opnTZPwg%Pb1!GX@NQ1%zp-KL(?-J03u&m>o}iD0eLs^cdh)wfrAzL zOTSaC$hxg4OYWZ{c zc*`0{>S*HkE8hCd*E>^|UO}E@iFIe3Q@`(pX^FB%#VhenKzC0_1#u=pCSa%8Vn{pV z(#?4clY{V^YGYTH213c9T*Py+Vcg6k`j^Q`1mS0!dR4_gqO%X%iJdcp;6~BudFx7w zgcXVcXM^v;p$ixupxBDP9`Vf~O75t>CSadFv!FMp;g8{8KCSV<=>s#V+k;BKsx*jDrQwH zHT4W5ia%O})}Vj$RKar%gJMBvt9@-M@YMjQVH4FL_FuTPMxJ2#(c3%eKse1vJDySa%sjTn(T2jWlF$Evz@dO0BB7ImvHNX@U z1~*Pn7Z7@nnITXKsF0AK?d|xbKbp&(DK*fk{%_hl?RlUBMM>Qj`Deh?9iCf?djqb5 z`mj|mk{#Mta2&&Y3gaS&cSz?mk0Ntj6JWSCNtK%%#pEsjs zZ_P)_#XRoJ8wy$&@YY{RRX7oDC@wytM2%6@z9D{)7Xt@W5AXo+3lIx`K-Mg6Q@u7S zlfewT1E~G6vyQ~elV#P_mU(+&)?0h!Hq76I1;Tm^S=rd5AEz|Riz^rg|0O`NY5BqW zJ3y8YZ}6c5b(Oq)tz}OsonVt5)Kj5zn%Y6$HF9)2wZBV`G1h3(6?g-#0#!-Z@QKXi zojaOVsxRGlLZR$6?23E@+Fh_lgiq8wi0G~Twsq%riF?oyRTsc^%tqPTSeWiXjm?nV zL#rzu?t*z1N`i%1Zk@2OvMfc2Ou7GSpdb5=PzZ>zJmxxrpt@_`aqn|?!3`zq6Ot+8 z2Uhae2KHdUxVQS}Q`#Mmc5f5y?|uauaZ?--`xt^2Od?aEcf?$rAH`hpz^_j*ZDeyi zeJ*h1zX+UEiGXbG6;x(Yvq)z7{53Y-z+){3l@L=AdRJ!!mdliQp~~#@9SIQP!VhHc zhSXsUX)(zFty>!f%3{wjMx;$bQ?>PW;O|W&l2RvTk&M9VlhmEtEr;^(vno{dDhjXfVA%6- z7BC;f{1H|5w4DboL15E9zy->9vQO|pJyxfbVxo(3T%;@nChgMK;dgI?l*gl;6;3;Y!R*zC9 z&i`fzh>Y-uN-y7TE&!;`iuKmJXU>9?Hc>oTvC|K$DE9B%@nR${4S||d)0Zr3hn<>oA=N+Y!{@h`t27dhr}_&!y|n6x=fqcdfrzZzulM%npDTpaybt${#{}WTT)-7zBmpzzPwpBA5ite$q>#nO^9W;0#~As2*|nke+gv&t4!! z7I>Nn#G|0FZgX7&_k04{D4!XDfa>+z5iJfkes-7d3lU=dbWTuI`7krrWRFD2 z0|%}E2Esf51nmPAf8(>5I}o8}L0ni5R%z0?yf;|F^YG^nS~%XER^ycM%U>PA=tOjrwOzcAQUYf63z%LAzv0zeivQk-laN{P|`n_>a+{!*av((4;UyyhFwALxNUE} zH(&s^jJp}!gJ5BEjs=*2J#wozL79HH+rsMN4!`y?5)jh4Zk_1SYNs52p>6ZcG)gbu z8J4TJOym`WMIb;~?;$-8p4?l2*6y1-bk8Wk59zh~Li(*g;2CJp6)K zgDEbIqJ){rahuOM@v9%EV#Zq2(ojvQ)K;d}Vm8XVtDJ|~%uFE18kes( zv1}tH0TOa&Pf_Qjqy)AZl?&FD(t#l35qx20LG#`|%a4WlH;+@VA~w~j5()DyZ&a}z*RDihWIdI5PHotvHiM(8k+muR#nHg(%>SzG_k6k=^wnYd0HNLIOZZpjH z!^fy}gkJ$Ojdq1`6;V*Y6h}ZM#tQ*vn>8?rP>XVv{zIYqhYQ?#j*p9N<8i;Q%71w?wHATBf@&jp^RJxR$r6wHAasvjz}Z4>}>D7 z_umuDBPYBpgy~7ACKNDJu0R>y~Z1A;kk$|KTx#xJ_!XL2TO2jTkVUUKyXE z)?{YNS{nMMx@VhSEHsFk-$n61c{)*h1CW9OoCX5ai+Ped4zrCD3bXmQ>j0=ed6yD} zg7GY-wOTlgQ3caCV$HyEO}~f;|JySu-VQ3%4;URCDjIXR(Cw07BBvpZKiZU|1#cEM z*tR15dLIm`DT)9sL5KcxuC2;_0MtM?hal(ua@_y#NEkm|fdHdmP}d-lzO_zx#d}%!KZ=D0OKG{qZ!s!WfQmtQ%p@=B3oDh*mY}@nl&@n0pY$A2c;PY;WTm$2< zQoAv*ZL3T(E<+7fLBXK0*0EniGzUhF!i>bd3bWm|5IyiBBNYRk^Y;<4*m;`Xm-45SLoeL6sxEc@n`D=wk@Jrj)I^>hZ+=tN1VX{PoIO=$D z@oPG|)DE$xzHQ3-=#l`_hMzq_9hCZW4QX`|3wqxsg=uQ{oM*I1gnwh^u>sA(9q44cQEk8&?b z1=0tV}}(&=kl2a zQsrDEdKsoIKlo0{$)XEaKlTjKJ@xdn^JqiWA@MnlwyW{m=(zQHE#@xF+-R#e9>Ho5 zp-WCLHl3h~VP?LHm@{q9oPnCQ9B5vE>fx)OWvOiW-#R=0ZP};S9>C)AnWj)9)(wW) zluQ1#!bxMqSdN>2Sa_Q+$Q(q(<}q1jeDPtzz$UZ7!AV;delW!$_y0eAp7y38mP}bc zMTDAY_-f4IEN120uZt}fy1G#3y~D_YNci3%i5Ib0-=;^Be)Pk^edHp7xr8 zI!CBiAWKHocAK(Du=RngOZzo|GZhm61f~bLvC9vnDVp4o1}4t%{+|Tmqu!#qL)daP?>F-&&^IiJCIi`3!9AwWwE z6n9PKSAcEy^S*5lcz9%CaNafFCI* zN{MWbL>YJec!O#u&lN@ewnI z#EJWiS4C{!GPnq~jK7T5O}lL$l8=EPf>5)nN80)&)m%|?R%LgTzC$+imrz;K9anI> za6EsM$2CGRuIhQh^ZACcopF^N^gTx@Soq-4SbA&t_+L~-aHD2Tv3w?Ak88gQco?c_ zIPM_7O*0Wg0;<1xR*)c0``9L(oYWKU6GD_}*;(PYd5Q{w*bTfH8To{9Bp-$E<*%i< zZPuzPnl$@c5LMK?_@BAY*(ospvg$9Ib#&qzvv~Dk9cmGH7{i>D8tEe8T~D>GV{|-? zmaVp_s$jS5lHO^)#4qa8+6bpyrgTC4RSn~y5-rQPn}W;p4fI`5?WbGq@logPbRQNx z^QVJ+sKrLAE%c`YLcuKgL3Jv;;BV4WL|quzKDD_tHnV_*$(Xb5!L&_&wrtDS-(#e_ zqgLd#j{yV478{pA3G1Dd!b5i+L0$XqVBuEovmWX={T$|x?a{@1iEh)^qRp|fXWew? z{&V~42p_(R;Gg>{n!lFu&1G(~zjId(9^X_khOSgS_NH-n5r1qmk}DV5*rMvvb5Np` zFkRJRKRLWy@YQ}|46eO<;42?W5# zYv*KPnko)Sn%$CIg(;a=xr6Jlh)W<5Zrj_&vB?T?Pd(P;3`&}>MsAl~buKKzC6zyq zbOUH_(4r2~4*!}(6kL`16;`ey^58>7m+R!#U+I{EPFOSUWN3g2o%C!X4jCR;4VyM! zZTPRn*f=1Hmh%3Av;E!xSMl}>H`-efL0zz1<>%|CXZ$`ZQ7x?p=i*}jzjm(ktEp^@ z6Bq?Sz_kQyltCiQvw(`A1_YIn%$24{LNE!TjqwqXCPRr30}30AbYqwa0NH%2GA>Nl9ZHC(Wfj0$ZTf;}%`!KEhdU zk=LbfcRCv@t~isA;mC|K0xybmee~&qlEw0aX@>rrlmI*R_c)?P(x z=8Y3jzy->Jh!gCieA16!K%EL-|9tGBeik5Ff$(j;|#GsVEp`TeC+yi3C;d-&Y+awxRfoI?RM60 zMsRa!ja8Zop4SIRx&*J|;5)lIXRRleO#IGt*Qa=5OQw=7>Y*1rX z=Ic6{y&ymp^fZ|uh{{6GH}TMznOV3A_Sb}miQlZFeL)vgUm*s(H_A}#(znNXn!ZBG z!G_2!mG7o@8{&{A@$qYpN=F9cw}+46Bv*tt)g_NbT$aossuB~MvK_0EsQ!uTR;=pI zG7~U5U+73r>|Bw{>KZi__g${%d)k>SPCbK_C~eeRUr|VyyR<0w*_E~3I8SZurSdnF zQ_4Ex&s;C>)g|+jrOA7{b2>=j?nbGx!J+Gf_AOdbO8fNi87z8qF%8k+APwlwX(G)C zat=nMc2Xvk4w!3KSD6$lgD(`jm(X*3vwr6<+Hac{QyA zdp;9Zxkx&!2eMqA5^0~TXByB-N?X(RJjBP)N@rvDmq;pWQYfY9Sqck|<+aN&N^1IF z^=BYHi#V6G*6peq8t5G=d6#?4rgO#d)(Z_Nr7rK(oS*B0Bl)-()0^A;1jj)U3%}QC zwnZAe3M{xPHGO*xgKbEyZESIiyuY{aB6Yt)ghQUY@}&c`QJT=>5TDyBy5&hSJHT?g z>cRePBS~=4`A7_HjbC--u2NIP)D||3-QfW%} zLVUs!QdMWPfKJpctL$xK3vdWtNu&+YLNfZz|EWp3MhTrx7iYf02`-VpslP6-4j1}O zu7sl@>LGgvuDg$+@-M7M#;ifaIdL9oNqe5ZJn(9V(B?$#*@EI8u%u$A;q-cdlJV_& zBC&|OK&wT0#|EnI;2|3BfXSI3O}U?aaLV1rj8AFJyo484Y^G8si*8mAFkhSmntcLI z3iHIBfNjT^QlR3nS;|>|5nOgYGRF*Nz3X%yv~l28KqT+nozBjFW6o)efWHquY&_E4GmJI8sxL8l-?z#zaOAWhw4EdX{} z_CV0bIR%Y;1Om7q0x7!PvP;4d27T(eqGr+ zvC90KCDt;+-K}Pr@b&JR_7$hAjG%#GPh>j!x92JNt1|aFVNbK)H|RA7E}D*AYG+Ag zeWNn8il0$vc2E{Py;Azcgn@@mOzIz=y4%i<8tV@J8si!2a>EDD8$3jfNMuZBZ@^t8 zkn^Z-??%QyeVa9`MKjERd^vYwT^vNg@^1Q(yxwy9Djrox($EEua0zZ-5VNH)jyvFs zr92>1zpYA$P?x-ozt)UBGqOzsk|LWzvl5C(S!hwwT|107e_~hC_|m>O&NC|mC`CJs zL|VNtX@8IlGOgyFST~{gEib+*A#oxs)BH(F#vP)?%Ef#Mc-_tC#4#u^X8$*Hd@&mD z(}d-hz0K;~H4=k*a!MK<#2B4#WDGa!o?CXl6f#ziP5Z-;*+WLuIn$Xjf{A$Gwy)C% z?t*sPf39Jcg8H&&J5dEvGXzFfIq~_EV0^51LvSB9>x-Ly4hH`4q+W!Zk{?_oE?rpFhwQS-|@FRF>uTMj0WJ>g_ zBuR}Az6xK#89K#f8Zkz#hxS1S&o)0ml;X%Tv#Xt4KuCQHB{e_|o%DL)5S>V@vxs4m z`GYdnP{qv#A6iFg!i(jSPx|)LI9?~7BwzUWvG^X+O>)&h<={ZA3f9}=OM@U+Y9Y|0 zwvu8vrL-H5YMOP&H}PDbKCH>@49K7Wp;_zajFNW_&w%9HCP?(QqYJLFINBPT3e|Ty zkA{HHeOgpA-ALqh1Jn--Pnj$Bdjv$h?Q5%XA$E^6;jPFvAV4}Wu z;!4$O6ep@D{Z~1$)7;3xj_z=Lk8nv(#gE#e7jkLPCKXUkhi*@Zck7B^uf?HY_zAu4 z9lg2p2ki2WhIj1Z7e|QVfX=dq6pdpJo*h-bIJ+OG31MezSXRb&5}^3@gIn4p>>T^l z>Y|`a3ad6!lmj~F=PD3nL|*^eX~T-h`iCQZmJrn%g?* zCttO&6l%gskubf`8~oTWsGRiHci5W$h>{1TeOP#x)Q7=WJQGsV1w=%J!??fAW7(Hz zVvBs7z&gIzCP1$T((6FKvqAN z%4p7hFacZ@6<#3DEivv1ogzREMBn(-+vaE^HgvSb%|>T{>ChvWSPyh>u!L=&wq`f> z#!J}Dm6M2>|FQOM4EX)|E`je7_%4C(68L|Vz-e;{2)jR6;(6E)fy F{tLfV1Ev4~ literal 0 HcmV?d00001 diff --git a/apps/desktop/build/generated-icon.icon/icon.json b/apps/desktop/build/generated-icon.icon/icon.json new file mode 100644 index 00000000000..f53f60fc7e4 --- /dev/null +++ b/apps/desktop/build/generated-icon.icon/icon.json @@ -0,0 +1,33 @@ +{ + "fill": { + "solid": "srgb:1.00000,1.00000,1.00000,1.00000" + }, + "groups": [ + { + "layers": [ + { + "image-name": "logo.png", + "is-glass": false, + "name": "Sim" + }, + { + "image-name": "border.svg", + "is-glass": false, + "name": "Dev Border" + } + ], + "shadow": { + "kind": "neutral", + "opacity": 0 + }, + "specular": false, + "translucency": { + "enabled": false, + "value": 0 + } + } + ], + "supported-platforms": { + "squares": ["macOS"] + } +} diff --git a/apps/sim/.env.example b/apps/sim/.env.example index ef123188499..3c2dcf8f229 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -63,7 +63,11 @@ API_ENCRYPTION_KEY=your_api_encryption_key # Use `openssl rand -hex 32` to gener # VLLM_API_KEY= # Optional bearer token if your vLLM instance requires auth # LITELLM_BASE_URL=http://localhost:4000 # Base URL for your LiteLLM proxy (OpenAI-compatible) # LITELLM_API_KEY= # Optional bearer token if your LiteLLM proxy requires auth -# FIREWORKS_API_KEY= # Optional Fireworks AI API key for model listing +# NEXT_PUBLIC_FORCE_HOSTED=true # Dev only: treat this instance as hosted Sim (sim-auto pool, platform keys); ignored in production builds +# FIREWORKS_API_KEY= # Optional Fireworks AI API key for model listing and inference +# FIREWORKS_API_KEY_1= # Optional Fireworks API key for rotation (hosted deployments) +# FIREWORKS_API_KEY_2= # Additional Fireworks API key for load balancing +# FIREWORKS_API_KEY_3= # Additional Fireworks API key for load balancing # NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS=true # Set when using AWS default credential chain (IAM roles, ECS task roles, IRSA). Hides credential fields in Agent block UI. # AZURE_OPENAI_ENDPOINT= # Azure OpenAI endpoint (hides field in UI when set alongside NEXT_PUBLIC_AZURE_CONFIGURED) # AZURE_OPENAI_API_KEY= # Azure OpenAI API key diff --git a/apps/sim/app/api/providers/fireworks/models/route.ts b/apps/sim/app/api/providers/fireworks/models/route.ts index ef3c000d960..82754eb53fb 100644 --- a/apps/sim/app/api/providers/fireworks/models/route.ts +++ b/apps/sim/app/api/providers/fireworks/models/route.ts @@ -10,6 +10,7 @@ import { validationErrorResponse } from '@/lib/api/server' import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' +import { isHosted } from '@/lib/core/config/env-flags' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' @@ -54,7 +55,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } } - if (!apiKey) { + /** + * On hosted Sim the platform key is the sim-auto pool's inference key, not a + * workspace credential: enumerating the whole Fireworks catalog from it would + * offer every serverless model as selectable when no key can actually run it + * (`getApiKeyWithBYOK` serves the platform key to catalog models only). + */ + if (!apiKey && !isHosted) { apiKey = env.FIREWORKS_API_KEY } diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index fb2e9756c38..29619bff5b7 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -20,6 +20,7 @@ import { getReasoningEffortValuesForModel, getThinkingLevelsForModel, getVerbosityValuesForModel, + isAutoModel, supportsTemperature, } from '@/providers/models' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' @@ -522,7 +523,13 @@ Return ONLY the JSON array.`, if (!model) { throw new Error('No model selected') } - const tool = getBaseModelProviders()[model] + // sim-auto resolves to a concrete pool model at execution time, where + // the agent handler derives the provider from the resolved model and + // never reads this serialized value. Serialization still needs the + // same provider-id shape every other model stores, so look up the + // runtime fallback model's provider. + const lookupModel = isAutoModel(model) ? 'claude-sonnet-5' : model + const tool = getBaseModelProviders()[lookupModel] if (!tool) { throw new Error(`Invalid model selected: ${model}`) } diff --git a/apps/sim/blocks/utils.test.ts b/apps/sim/blocks/utils.test.ts index bf90c25d656..9d5a3adf24f 100644 --- a/apps/sim/blocks/utils.test.ts +++ b/apps/sim/blocks/utils.test.ts @@ -40,6 +40,8 @@ vi.mock('@/providers/models', () => ({ getProviderModels: mockGetProviderModels, getProviderIcon: mockGetProviderIcon, getBaseModelProviders: mockGetBaseModelProviders, + SIM_AUTO_MODEL_ID: 'sim-auto', + isAutoModel: (model: string) => model.trim().toLowerCase() === 'sim-auto', })) vi.mock('@/providers/utils', () => ({ diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index f0c7d11f13a..848b8d36363 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -1,4 +1,5 @@ import { toError } from '@sim/utils/errors' +import { SimAutoIcon } from '@/components/icons' import { isAzureConfigured, isCohereConfigured, @@ -14,7 +15,9 @@ import { getModelSunsetStatus, getProviderIcon, getProviderModels, + isAutoModel, orderModelIdsByReleaseDate, + SIM_AUTO_MODEL_ID, } from '@/providers/models' import { isPiSupportedModel } from '@/providers/pi-providers' import { getProviderFromModel } from '@/providers/utils' @@ -75,12 +78,21 @@ export function getModelOptions() { ]) ) - return allModels + const options = allModels .filter((model) => getModelSunsetStatus(model) !== 'deprecated') .map((model) => { const icon = getProviderIcon(model) return { label: model, id: model, ...(icon && { icon }) } }) + + // Hosted-only automatic model. Deliberately LAST in the list (limited + // visibility for the initial release): available to anyone who scrolls or + // searches for it, but never the first thing the dropdown offers. + if (isHosted) { + options.push({ label: 'Auto', id: SIM_AUTO_MODEL_ID, icon: SimAutoIcon }) + } + + return options } /** @@ -183,6 +195,11 @@ function shouldRequireApiKeyForModel(model: string): boolean { const normalizedModel = model.trim().toLowerCase() if (!normalizedModel) return false + // On hosted Sim the auto pseudo-model resolves server-side to a hosted pool + // model. On self-hosted it exists only via imported workflows and always + // falls back to the default Anthropic model, so the key field must show. + if (isAutoModel(normalizedModel)) return !isHosted + if (isHosted) { const hostedModels = getHostedModels() if (hostedModels.some((m) => m.toLowerCase() === normalizedModel)) return false diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index efedba2000d..5a2e8424b26 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -7558,6 +7558,31 @@ export function SixtyfourIcon(props: SVGProps) { ) } +/** + * The "sim" brand wordmark (v1.0 brand guide simLogotype paths — the same mark + * the navbar/login header renders), inked with the theme-adaptive + * `--text-body`. Used as the icon for the Auto model option; the wide viewBox + * letterboxes itself inside square icon slots. + */ +export function SimAutoIcon(props: SVGProps) { + return ( + + ) +} + export function SimTriggerIcon(props: SVGProps) { return ( { expect(result).toEqual(expectedOutput) }) + it('reports a sim-auto run under the sim-auto identity, not the model that served it', async () => { + mockExecuteProviderRequest.mockResolvedValue({ + content: 'Mocked response content', + model: AGENT.DEFAULT_MODEL, + tokens: { input: 10, output: 20, total: 30 }, + toolCalls: [], + cost: { input: 0.001, output: 0.002, total: 0.003 }, + timing: { + total: 100, + timeSegments: [ + { type: 'model', name: AGENT.DEFAULT_MODEL, provider: 'anthropic', duration: 100 }, + ], + }, + }) + + const result = (await handler.execute(mockContext, mockBlock, { + model: SIM_AUTO_MODEL_ID, + userPrompt: 'Hello!', + })) as { + model: string + cost: unknown + tokens: unknown + providerTiming: { timeSegments: Array<{ name?: string; provider?: string }> } + } + + expect(result.model).toBe(SIM_AUTO_MODEL_ID) + expect(result.providerTiming.timeSegments[0].name).toBe(SIM_AUTO_MODEL_ID) + expect(result.providerTiming.timeSegments[0].provider).toBeUndefined() + // Only the label changes: tokens and the already-priced cost are untouched. + expect(result.tokens).toEqual({ input: 10, output: 20, total: 30 }) + expect(result.cost).toEqual({ input: 0.001, output: 0.002, total: 0.003 }) + }) + + /** Reaches the private signal builder; routing depends on nothing else. */ + const buildAutoRoutingSignalsFor = (inputs: Record) => + ( + handler as unknown as { + buildAutoRoutingSignals: (i: unknown, rf: unknown) => { mediaKind: string } + } + ).buildAutoRoutingSignals(inputs, undefined) + + const png = { id: 'f1', type: 'image/png' } + const pdf = { id: 'f2', type: 'application/pdf' } + + it('reports no media when neither the files input nor any message carries one', async () => { + const signals = buildAutoRoutingSignalsFor({ + messages: [{ role: 'user' as const, content: 'Summarize this text' }], + }) + + expect(signals.mediaKind).toBe('none') + }) + + it('detects media carried on inbound messages, not just the files input', async () => { + const signals = buildAutoRoutingSignalsFor({ + messages: [{ role: 'user' as const, content: 'What is in this image?', files: [png] }], + }) + + expect(signals.mediaKind).toBe('image') + }) + + it('classifies an all-image attachment set as image', async () => { + expect(buildAutoRoutingSignalsFor({ files: [png, png] }).mediaKind).toBe('image') + }) + + it('classifies a mixed image + document set as file', async () => { + expect(buildAutoRoutingSignalsFor({ files: [png, pdf] }).mediaKind).toBe('file') + }) + + it('treats an unknown MIME type as file rather than assuming it is an image', async () => { + expect(buildAutoRoutingSignalsFor({ files: [{ id: 'f3' }] }).mediaKind).toBe('file') + }) + + it('overlays the routing charge on a streaming cost written after the fact', async () => { + // Mirrors the real streaming shape: the policy accessor is installed at + // provider-return time, the drain writes the final cost long after the + // handler returned, and consumers read it at log time. + const output: Record = { cost: { input: 0, output: 0, total: 0 } } + installStreamingCostPolicy(output as never, { billable: true, multiplier: 1 }) + const streaming = { stream: new ReadableStream(), execution: { output } } + + ;( + handler as unknown as { applyRoutingCost: (r: unknown, c: number) => void } + ).applyRoutingCost(streaming, 0.002) + + // The drain settles the model cost afterwards. + + ;(output as { cost: unknown }).cost = { input: 0.01, output: 0.02, total: 0.03 } + + expect(output.cost).toEqual({ + input: 0.01, + output: 0.02, + total: expect.closeTo(0.032, 10), + routing: 0.002, + }) + }) + + it('adds the routing charge to a settled non-streaming cost', async () => { + const result: Record = { cost: { input: 0.01, output: 0.02, total: 0.03 } } + + ;( + handler as unknown as { applyRoutingCost: (r: unknown, c: number) => void } + ).applyRoutingCost(result, 0.002) + + expect(result.cost).toEqual({ + input: 0.01, + output: 0.02, + total: expect.closeTo(0.032, 10), + routing: 0.002, + }) + }) + + it('leaves the reported model alone for an explicitly selected model', async () => { + const result = (await handler.execute(mockContext, mockBlock, { + model: 'gpt-4o', + userPrompt: 'Hello!', + apiKey: 'test-api-key', + })) as { model: string } + + expect(result.model).toBe('mock-model') + }) + it('should attach files to the last user message only', async () => { const inputs = { model: 'gpt-4o', diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index fc721b990f2..a126b7e00b4 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -7,7 +7,18 @@ import { truncate } from '@sim/utils/string' import { and, eq, inArray, isNull } from 'drizzle-orm' import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records' import { createMcpToolId } from '@/lib/mcp/utils' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' +import { + type AutoMediaKind, + type AutoRoutingResult, + type AutoRoutingSignals, + resolveAutoModel, + SIM_AUTO_SYSTEM_PREAMBLE, +} from '@/lib/model-router/resolve' +import { + MODEL_SUPPORTED_IMAGE_MIME_TYPES, + processFilesToUserFiles, + type RawFileInput, +} from '@/lib/uploads/utils/file-utils' import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' import { resolveCustomBlockToolBinding } from '@/lib/workflows/custom-blocks/operations' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' @@ -46,6 +57,7 @@ import { shouldUseLargeFilePath, supportsFileAttachments, } from '@/providers/attachments' +import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' import { getProviderFromModel, transformBlockTool } from '@/providers/utils' import type { SerializedBlock } from '@/serializer/types' import { filterSchemaForLLM, type ToolSchema } from '@/tools/params' @@ -77,7 +89,32 @@ export class AgentBlockHandler implements BlockHandler { await this.validateToolPermissions(ctx, filteredInputs.tools || []) const responseFormat = parseResponseFormat(filteredInputs.responseFormat) - const model = filteredInputs.model || AGENT.DEFAULT_MODEL + const configuredModel = filteredInputs.model || AGENT.DEFAULT_MODEL + + let model = configuredModel + let autoRouting: AutoRoutingResult | null = null + if (isAutoModel(configuredModel)) { + autoRouting = await resolveAutoModel({ + ctx, + blockId: block.id, + signals: this.buildAutoRoutingSignals(filteredInputs, responseFormat), + fallbackModel: AGENT.DEFAULT_MODEL, + }) + model = autoRouting.model + logger.info('Resolved sim-auto model', { + blockId: block.id, + model, + tier: autoRouting.tier, + decidedBy: autoRouting.decidedBy, + }) + // Hidden identity preamble for every auto execution (fallback included): + // keeps pool models in English by default and off the topic of which + // underlying model they are. Applied after signal building so the + // preamble never influences classification. + filteredInputs.systemPrompt = [SIM_AUTO_SYSTEM_PREAMBLE, filteredInputs.systemPrompt] + .filter(Boolean) + .join('\n\n') + } await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx) @@ -126,6 +163,14 @@ export class AgentBlockHandler implements BlockHandler { const result = await this.executeProviderRequest(ctx, providerRequest, block, responseFormat) + if (autoRouting && autoRouting.billableRoutingCost > 0) { + this.applyRoutingCost(result, autoRouting.billableRoutingCost) + } + + if (autoRouting) { + this.applyAutoModelLabel(result, model) + } + if (this.isStreamingExecution(result)) { if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { return this.wrapStreamForMemoryPersistence( @@ -144,6 +189,142 @@ export class AgentBlockHandler implements BlockHandler { return result } + /** + * Derives the compact routing signals for sim-auto resolution from the + * block's resolved inputs. Excerpts only — the resolver truncates further + * and mothership re-clamps server-side. + */ + private buildAutoRoutingSignals(inputs: AgentInputs, responseFormat: any): AutoRoutingSignals { + const lastMessage = + typeof inputs.userPrompt === 'string' + ? inputs.userPrompt + : inputs.userPrompt != null + ? stringifyJSON(inputs.userPrompt) + : (inputs.messages?.at(-1)?.content ?? '') + const systemPrompt = inputs.systemPrompt ?? '' + const normalizedFiles = normalizeFileInput(inputs.files) + const approxChars = + systemPrompt.length + + lastMessage.length + + (inputs.messages ? stringifyJSON(inputs.messages).length : 0) + + return { + systemPrompt, + lastMessage, + messageCount: (inputs.messages?.length ?? 0) + (inputs.userPrompt ? 1 : 0), + toolNames: (inputs.tools ?? []).map((t) => t.title || t.type || 'tool'), + mediaKind: this.resolveMediaKind(inputs, normalizedFiles), + hasResponseFormat: Boolean(responseFormat), + approxInputTokens: Math.ceil(approxChars / 4), + } + } + + /** + * Classifies what the block attaches, which decides the sim-auto pool column. + * + * Media reaches the provider by two routes — the block's `files` input and + * files already carried on inbound messages (chat deployments, memory, an + * upstream block feeding `messages`) — and both count. A file whose MIME type + * is missing or unrecognized counts as `file`, the column served by the + * providers that accept the most input types, because the alternative is + * handing a document to a model whose API models no such content part. + */ + private resolveMediaKind(inputs: AgentInputs, normalizedFiles: unknown): AutoMediaKind { + const attached: Array<{ type?: string }> = [ + ...(Array.isArray(normalizedFiles) ? (normalizedFiles as Array<{ type?: string }>) : []), + ...(inputs.messages ?? []).flatMap((message) => message.files ?? []), + ] + + if (attached.length === 0) return 'none' + + return attached.every((file) => + MODEL_SUPPORTED_IMAGE_MIME_TYPES.has((file.type ?? '').toLowerCase()) + ) + ? 'image' + : 'file' + } + + /** + * Reports a completed auto run under the `sim-auto` identity everywhere the + * run is observed — the block output, the trace span, and the usage-ledger + * row keyed on the model name — so the pool model that served the request + * stays an implementation detail (matching the block's configured model and + * the hidden identity preamble the models themselves run under). + * + * Runs after `executeProviderRequest`, whose billability gate and pricing + * key on the concrete pool model: tokens and cost are already settled here, + * only the label changes. + */ + private applyAutoModelLabel( + result: BlockOutput | StreamingExecution, + resolvedModel: string + ): void { + const output = this.isStreamingExecution(result) + ? (result as StreamingExecution).execution?.output + : (result as BlockOutput) + if (!output || typeof output !== 'object') return + + const target = output as { + model?: string + providerTiming?: { + timeSegments?: Array<{ type?: string; name?: string; provider?: string }> + } + } + target.model = SIM_AUTO_MODEL_ID + + // Model segments name themselves after the model (every provider does) and + // carry the serving provider, which the log detail renders as that + // provider's icon — the same leak by two other routes. + for (const segment of target.providerTiming?.timeSegments ?? []) { + if (segment.type !== 'model') continue + if (segment.name?.toLowerCase() === resolvedModel.toLowerCase()) { + segment.name = SIM_AUTO_MODEL_ID + } + segment.provider = undefined + } + } + + /** + * Adds the billable sim-auto routing charge to the output's cost breakdown + * as a distinct `routing` component. + * + * A non-streaming cost is final, so it is mutated in place. A streaming + * output's cost settles at stream end — written through the accessor + * `installStreamingCostPolicy` installed — so the charge is layered as a + * read-time overlay on the property instead: whatever the drain writes, + * every later read (trace spans, cost summary, usage ledger) sees the + * routing component on top. + */ + private applyRoutingCost(result: BlockOutput | StreamingExecution, routingCost: number): void { + const output = this.isStreamingExecution(result) + ? (result as StreamingExecution).execution?.output + : (result as BlockOutput) + if (!output || typeof output !== 'object') return + const target = output as { cost?: Record } + + const withRouting = (cost: Record | undefined): Record => + cost && typeof cost.total === 'number' + ? { ...cost, routing: routingCost, total: cost.total + routingCost } + : { input: 0, output: 0, routing: routingCost, total: routingCost } + + if (!this.isStreamingExecution(result)) { + target.cost = withRouting(target.cost) + return + } + + const prior = Object.getOwnPropertyDescriptor(target, 'cost') + let raw = prior?.get ? undefined : (target.cost as Record | undefined) + Object.defineProperty(target, 'cost', { + get: () => withRouting(prior?.get ? (prior.get.call(target) as never) : raw), + set: (value: Record | undefined) => { + if (prior?.set) prior.set.call(target, value) + else raw = value + }, + configurable: true, + enumerable: true, + }) + } + private async validateToolPermissions(ctx: ExecutionContext, tools: ToolInput[]): Promise { if (!Array.isArray(tools) || tools.length === 0) return diff --git a/apps/sim/lib/api-key/byok.test.ts b/apps/sim/lib/api-key/byok.test.ts index 7bd905b16c5..bcc8d81bfcb 100644 --- a/apps/sim/lib/api-key/byok.test.ts +++ b/apps/sim/lib/api-key/byok.test.ts @@ -13,11 +13,24 @@ vi.mock('@/lib/core/security/encryption', () => ({ })) vi.mock('@/lib/core/config/api-keys', () => ({ - getRotatingApiKey: vi.fn(), + getRotatingApiKey: mockGetRotatingApiKey, +})) + +const { mockEnv, mockGetRotatingApiKey, mockGetHostedModels, mockIsHosted } = vi.hoisted(() => ({ + mockEnv: {} as Record, + mockGetRotatingApiKey: vi.fn(), + mockGetHostedModels: vi.fn(() => [] as string[]), + mockIsHosted: { value: true }, })) vi.mock('@/lib/core/config/env', () => ({ - env: {}, + env: mockEnv, +})) + +vi.mock('@/lib/core/config/env-flags', () => ({ + get isHosted() { + return mockIsHosted.value + }, })) vi.mock('@/providers/models', () => ({ @@ -25,7 +38,7 @@ vi.mock('@/providers/models', () => ({ .fn() .mockReturnValue({ maxBytes: 10 * 1024 * 1024, strategy: 'inline' }), INLINE_ATTACHMENT_MAX_BYTES: 10 * 1024 * 1024, - getHostedModels: vi.fn(() => []), + getHostedModels: mockGetHostedModels, })) vi.mock('@/providers/utils', () => ({ @@ -36,7 +49,8 @@ vi.mock('@/stores/providers/store', () => ({ useProvidersStore: { getState: vi.fn() }, })) -import { getBYOKKey } from '@/lib/api-key/byok' +import { getApiKeyWithBYOK, getBYOKKey } from '@/lib/api-key/byok' +import { useProvidersStore } from '@/stores/providers/store' /** * Rotation counters in the module under test are keyed by @@ -152,3 +166,73 @@ describe('getBYOKKey', () => { expect(await getBYOKKey(uniqueWorkspaceId(), 'openai')).toBeNull() }) }) + +describe('getApiKeyWithBYOK for Fireworks', () => { + const HOSTED_POOL_MODEL = 'fireworks/glm-5.2' + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockIsHosted.value = true + mockEnv.FIREWORKS_API_KEY = 'platform-fireworks-key' + mockGetHostedModels.mockReturnValue([HOSTED_POOL_MODEL, 'fireworks/kimi-k3']) + mockGetRotatingApiKey.mockReturnValue('rotated-fireworks-key') + ;(useProvidersStore.getState as ReturnType).mockReturnValue({ + providers: { + ollama: { models: [] }, + vllm: { models: [] }, + litellm: { models: [] }, + fireworks: { models: [] }, + together: { models: [] }, + baseten: { models: [] }, + 'ollama-cloud': { models: [] }, + }, + }) + }) + + it('serves the rotating platform key for a hosted catalog model', async () => { + const result = await getApiKeyWithBYOK('fireworks', HOSTED_POOL_MODEL, uniqueWorkspaceId()) + + expect(mockGetRotatingApiKey).toHaveBeenCalledWith('fireworks') + expect(result).toEqual({ apiKey: 'rotated-fireworks-key', isBYOK: false }) + }) + + it('prefers a workspace BYOK key over the platform key, as hosted models do', async () => { + dbChainMockFns.orderBy.mockResolvedValue([storedKey('key-1')]) + + const result = await getApiKeyWithBYOK('fireworks', HOSTED_POOL_MODEL, uniqueWorkspaceId()) + + expect(result).toEqual({ apiKey: 'decrypted-key-1', isBYOK: true }) + expect(mockGetRotatingApiKey).not.toHaveBeenCalled() + }) + + it('never serves the platform key to a dynamic model on hosted', async () => { + await expect( + getApiKeyWithBYOK('fireworks', 'fireworks/accounts/acme/models/custom', uniqueWorkspaceId()) + ).rejects.toThrow('API key is required for Fireworks') + expect(mockGetRotatingApiKey).not.toHaveBeenCalled() + }) + + it('serves a user-provided key for a dynamic model on hosted', async () => { + const result = await getApiKeyWithBYOK( + 'fireworks', + 'fireworks/accounts/acme/models/custom', + uniqueWorkspaceId(), + 'user-key' + ) + + expect(result).toEqual({ apiKey: 'user-key', isBYOK: false }) + }) + + it('falls back to the env key for any model when self-hosted', async () => { + mockIsHosted.value = false + + const result = await getApiKeyWithBYOK( + 'fireworks', + 'fireworks/accounts/acme/models/custom', + uniqueWorkspaceId() + ) + + expect(result).toEqual({ apiKey: 'platform-fireworks-key', isBYOK: false }) + }) +}) diff --git a/apps/sim/lib/api-key/byok.ts b/apps/sim/lib/api-key/byok.ts index aa456962264..ba7c9f7e611 100644 --- a/apps/sim/lib/api-key/byok.ts +++ b/apps/sim/lib/api-key/byok.ts @@ -122,6 +122,35 @@ export async function getApiKeyWithBYOK( return byokResult } } + + /** + * On hosted Sim the platform Fireworks key backs the static catalog (the + * sim-auto pool) and nothing else, exactly as the platform Anthropic and + * OpenAI keys back only their catalogued models. A dynamic `fireworks/*` + * id a workspace configured itself carries no catalog pricing, so + * `shouldBillModelUsage` would return false for it — serving it on Sim's + * key would be unmetered inference. + */ + if (isHosted) { + const isModelHosted = getHostedModels().some((m) => m.toLowerCase() === model.toLowerCase()) + if (isModelHosted) { + try { + const serverKey = getRotatingApiKey('fireworks') + return { apiKey: serverKey, isBYOK: false } + } catch (_error) { + if (userProvidedKey) { + return { apiKey: userProvidedKey, isBYOK: false } + } + throw new Error(`No API key available for fireworks ${model}`) + } + } + + if (userProvidedKey) { + return { apiKey: userProvidedKey, isBYOK: false } + } + throw new Error(`API key is required for Fireworks ${model}`) + } + if (userProvidedKey) { return { apiKey: userProvidedKey, isBYOK: false } } diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts index 23b8de61ba7..36bc722a0f8 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { omit } from '@sim/utils/object' import { validateSelectorIds } from '@/lib/copilot/validation/selector-validator' +import { isHosted as isHostedDeployment } from '@/lib/core/config/env-flags' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' import type { PermissionGroupConfig } from '@/lib/permission-groups/types' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' @@ -16,7 +17,7 @@ import { getBlock } from '@/blocks/registry' import type { SubBlockConfig } from '@/blocks/types' import { getModelOptions } from '@/blocks/utils' import { BlockType, EDGE, normalizeName } from '@/executor/constants' -import { isKnownModelId, suggestModelIdsForUnknownModel } from '@/providers/models' +import { isAutoModel, isKnownModelId, suggestModelIdsForUnknownModel } from '@/providers/models' import { isPiByokOnlyMode } from '@/providers/pi-providers' import { getTool } from '@/tools/utils' import { TRIGGER_RUNTIME_SUBBLOCK_IDS, TRIGGER_WEBHOOK_URL_FIELD } from '@/triggers/constants' @@ -557,6 +558,12 @@ export function validateValueForSubBlockType( if (usesProviderCatalog) { const stringValue = typeof value === 'string' ? value : String(value) const trimmed = stringValue.trim() + // sim-auto is a valid model value on hosted Sim only (mirrors the + // options array the agent reads: it is absent from self-hosted + // snapshots, so writes of it there are rejected as unknown). + if (trimmed !== '' && isAutoModel(trimmed) && isHostedDeployment) { + return { valid: true, value: trimmed.toLowerCase() } + } if (trimmed !== '' && !isKnownModelId(trimmed)) { const suggestions = suggestModelIdsForUnknownModel(trimmed) const suggestionText = diff --git a/apps/sim/lib/copilot/vfs/serializers.ts b/apps/sim/lib/copilot/vfs/serializers.ts index d394ed07347..0972b7a0d1f 100644 --- a/apps/sim/lib/copilot/vfs/serializers.ts +++ b/apps/sim/lib/copilot/vfs/serializers.ts @@ -12,7 +12,11 @@ import { getBlock } from '@/blocks' import { isCustomBlockType } from '@/blocks/custom/build-config' import type { BlockConfig, SubBlockConfig } from '@/blocks/types' import { isHiddenUnder } from '@/blocks/visibility/context' -import { DYNAMIC_MODEL_PROVIDERS, PROVIDER_DEFINITIONS } from '@/providers/models' +import { + DYNAMIC_MODEL_PROVIDERS, + PROVIDER_DEFINITIONS, + SIM_AUTO_MODEL_ID, +} from '@/providers/models' import type { ToolConfig, ToolHostingCondition } from '@/tools/types' /** The service-account alternative to OAuth for a service, when it offers one. */ @@ -500,6 +504,17 @@ function getStaticModelOptionsForVFS(): StaticModelOption[] { const models: StaticModelOption[] = [] + // Hosted-only automatic model. Deliberately not `recommended` and given no + // prompt guidance (limited-visibility release): the build agent can write it + // when a user explicitly asks for the auto model, but is never steered to it. + if (isHosted) { + models.push({ + id: SIM_AUTO_MODEL_ID, + provider: 'sim', + hosted: true, + }) + } + for (const [providerId, def] of Object.entries(PROVIDER_DEFINITIONS)) { if (dynamicProviders.has(providerId)) continue for (const model of def.models) { diff --git a/apps/sim/lib/core/config/api-keys.ts b/apps/sim/lib/core/config/api-keys.ts index c8812aa05e4..839df609a1a 100644 --- a/apps/sim/lib/core/config/api-keys.ts +++ b/apps/sim/lib/core/config/api-keys.ts @@ -14,7 +14,8 @@ export function getRotatingApiKey(provider: string): string { provider !== 'cohere' && provider !== 'zai' && provider !== 'xai' && - provider !== 'kimi' + provider !== 'kimi' && + provider !== 'fireworks' ) { throw new Error(`No rotation implemented for provider: ${provider}`) } @@ -49,6 +50,13 @@ export function getRotatingApiKey(provider: string): string { if (env.KIMI_API_KEY_1) keys.push(env.KIMI_API_KEY_1) if (env.KIMI_API_KEY_2) keys.push(env.KIMI_API_KEY_2) if (env.KIMI_API_KEY_3) keys.push(env.KIMI_API_KEY_3) + } else if (provider === 'fireworks') { + if (env.FIREWORKS_API_KEY_1) keys.push(env.FIREWORKS_API_KEY_1) + if (env.FIREWORKS_API_KEY_2) keys.push(env.FIREWORKS_API_KEY_2) + if (env.FIREWORKS_API_KEY_3) keys.push(env.FIREWORKS_API_KEY_3) + // The platform Fireworks key predates the rotation slots and ships as a + // single secret; it stands in as a one-key pool until slots are populated. + if (keys.length === 0 && env.FIREWORKS_API_KEY) keys.push(env.FIREWORKS_API_KEY) } if (keys.length === 0) { diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index 115242cbfbc..29792c6af1a 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -34,7 +34,16 @@ try { } catch { // invalid URL — isHosted stays false } -export const isHosted = appHostname === 'sim.ai' || appHostname.endsWith('.sim.ai') +/** + * Local-development escape hatch for exercising hosted-only paths (the sim-auto + * pool, platform keys, hosted-only UI) without pointing `NEXT_PUBLIC_APP_URL` at + * a sim.ai hostname, which would break local callback URLs. Ignored in + * production builds, so a self-hosted deployment can never claim to be Sim's + * hosted environment. + */ +const forceHosted = !isProd && isTruthy(getEnv('NEXT_PUBLIC_FORCE_HOSTED')) + +export const isHosted = forceHosted || appHostname === 'sim.ai' || appHostname.endsWith('.sim.ai') /** * Enables the strict attributed-v1 Sim/Copilot billing protocol after the Go diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 8c28ba643f8..b6a8f6d2469 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -173,7 +173,10 @@ export const env = createEnv({ VLLM_API_KEY: z.string().optional(), // Optional bearer token for vLLM LITELLM_BASE_URL: z.string().url().optional(), // LiteLLM proxy base URL (OpenAI-compatible) LITELLM_API_KEY: z.string().optional(), // Optional bearer token for LiteLLM - FIREWORKS_API_KEY: z.string().optional(), // Optional Fireworks AI API key for model listing + FIREWORKS_API_KEY: z.string().optional(), // Platform Fireworks AI key backing the hosted sim-auto pool (and self-hosted model listing/inference) + FIREWORKS_API_KEY_1: z.string().min(1).optional(), // Primary Fireworks API key for load balancing + FIREWORKS_API_KEY_2: z.string().min(1).optional(), // Additional Fireworks API key for load balancing + FIREWORKS_API_KEY_3: z.string().min(1).optional(), // Additional Fireworks API key for load balancing TOGETHER_API_KEY: z.string().optional(), // Optional Together AI API key for model listing and inference BASETEN_API_KEY: z.string().optional(), // Optional Baseten API key for model listing and inference COHERE_API_KEY: z.string().min(1).optional(), // Cohere API key for reranker (rerank-v4.0-pro, rerank-v4.0-fast, rerank-v3.5) diff --git a/apps/sim/lib/core/utils.test.ts b/apps/sim/lib/core/utils.test.ts index 62753f61204..1c882094c52 100644 --- a/apps/sim/lib/core/utils.test.ts +++ b/apps/sim/lib/core/utils.test.ts @@ -24,6 +24,9 @@ beforeAll(() => { XAI_API_KEY_1: 'test-xai-key-1', XAI_API_KEY_2: 'test-xai-key-2', XAI_API_KEY_3: 'test-xai-key-3', + FIREWORKS_API_KEY_1: 'test-fireworks-key-1', + FIREWORKS_API_KEY_2: 'test-fireworks-key-2', + FIREWORKS_API_KEY_3: 'test-fireworks-key-3', }) }) @@ -338,6 +341,29 @@ describe('getRotatingApiKey', () => { expect(result).toMatch(/^test-xai-key-[1-3]$/) }) + it.concurrent('should return Fireworks API key based on current minute', () => { + const result = getRotatingApiKey('fireworks') + expect(result).toMatch(/^test-fireworks-key-[1-3]$/) + }) + + it('falls back to the single platform Fireworks key when no rotation slot is set', () => { + setEnv({ + FIREWORKS_API_KEY_1: undefined, + FIREWORKS_API_KEY_2: undefined, + FIREWORKS_API_KEY_3: undefined, + FIREWORKS_API_KEY: 'test-fireworks-platform-key', + }) + + expect(getRotatingApiKey('fireworks')).toBe('test-fireworks-platform-key') + + setEnv({ + FIREWORKS_API_KEY: undefined, + FIREWORKS_API_KEY_1: 'test-fireworks-key-1', + FIREWORKS_API_KEY_2: 'test-fireworks-key-2', + FIREWORKS_API_KEY_3: 'test-fireworks-key-3', + }) + }) + it.concurrent('should throw error for unsupported provider', () => { expect(() => getRotatingApiKey('unsupported')).toThrow('No rotation implemented for provider') }) diff --git a/apps/sim/lib/model-router/resolve.test.ts b/apps/sim/lib/model-router/resolve.test.ts new file mode 100644 index 00000000000..d4f65f7b6e5 --- /dev/null +++ b/apps/sim/lib/model-router/resolve.test.ts @@ -0,0 +1,288 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockFetchGo, + mockValidateModelProvider, + mockGetMothershipBaseURL, + mockGetProviderFromModel, +} = vi.hoisted(() => ({ + mockFetchGo: vi.fn(), + mockValidateModelProvider: vi.fn(), + mockGetMothershipBaseURL: vi.fn(), + mockGetProviderFromModel: vi.fn(), +})) + +vi.mock('@/lib/core/config/env-flags', () => ({ + isHosted: true, + getCostMultiplier: () => 2, +})) + +vi.mock('@/lib/core/config/env', () => ({ + env: { COPILOT_API_KEY: 'test-copilot-key' }, +})) + +vi.mock('@/lib/copilot/request/go/fetch', () => ({ + fetchGo: mockFetchGo, +})) + +vi.mock('@/lib/copilot/server/agent-url', () => ({ + getMothershipBaseURL: mockGetMothershipBaseURL, +})) + +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + validateModelProvider: mockValidateModelProvider, +})) + +vi.mock('@/providers/utils', () => ({ + getProviderFromModel: mockGetProviderFromModel, +})) + +import { type AutoRoutingSignals, resolveAutoModel } from '@/lib/model-router/resolve' +import type { ExecutionContext } from '@/executor/types' + +const ctx = { + userId: 'user-1', + workspaceId: 'ws-1', + workflowId: 'wf-1', + executionId: 'exec-1', +} as unknown as ExecutionContext + +/** Distinct-by-default signals so the module-level decision cache never collides across tests. */ +let signalSeq = 0 +function makeSignals(overrides: Partial = {}): AutoRoutingSignals { + return { + systemPrompt: `analyze the quarterly report ${++signalSeq}`, + lastMessage: 'here is the data to reconcile against the ledger', + messageCount: 1, + toolNames: ['exa_search'], + mediaKind: 'none', + hasResponseFormat: false, + approxInputTokens: 5000, + ...overrides, + } +} + +function routerResponse(body: unknown, ok = true, status = 200) { + return { ok, status, json: () => Promise.resolve(body) } +} + +describe('resolveAutoModel', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetMothershipBaseURL.mockResolvedValue('https://copilot.test') + mockValidateModelProvider.mockResolvedValue(undefined) + mockGetProviderFromModel.mockReturnValue('fireworks') + }) + + /** The full pool, exactly as specified: media kind → tier → model. */ + const POOL_CASES: Array<{ mediaKind: AutoRoutingSignals['mediaKind']; byTier: string[] }> = [ + { mediaKind: 'none', byTier: ['fireworks/glm-5.2', 'fireworks/glm-5.2', 'fireworks/kimi-k3'] }, + { mediaKind: 'image', byTier: ['gemini-3.6-flash', 'fireworks/kimi-k3', 'fireworks/kimi-k3'] }, + { mediaKind: 'file', byTier: ['gemini-3.6-flash', 'claude-sonnet-5', 'gpt-5.6-sol'] }, + ] + + for (const { mediaKind, byTier } of POOL_CASES) { + byTier.forEach((expected, index) => { + const tier = String(index + 1) + it(`routes ${mediaKind} tier ${tier} to ${expected}`, async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: tier })) + + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals({ mediaKind }), + fallbackModel: 'claude-sonnet-5', + }) + + expect(result.model).toBe(expected) + expect(result.tier).toBe(tier) + }) + }) + } + + it('offers all three tiers and no model name to the router', async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: '2' })) + + await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals({ mediaKind: 'image' }), + fallbackModel: 'claude-sonnet-5', + }) + + const body = JSON.parse(mockFetchGo.mock.calls[0][1].body as string) + expect(body.candidates.map((c: { id: string }) => c.id)).toEqual(['1', '2', '3']) + // Media kind is Sim's business: the wire carries only its presence. + expect(body.signals.hasMedia).toBe(true) + expect(body.signals.mediaKind).toBeUndefined() + for (const model of ['glm', 'kimi', 'gemini', 'gpt-5.6-sol', 'sonnet']) { + expect(JSON.stringify(body)).not.toContain(model) + } + }) + + it('never crosses media kinds when walking down from a denied tier', async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: '3' })) + // gpt-5.6-sol denied; the file column must drop to sonnet, never to a + // Fireworks model that cannot accept the attachment at all. + mockValidateModelProvider.mockImplementation(async (_u, _w, model: string) => { + if (model === 'gpt-5.6-sol') throw new Error('provider blocked') + }) + + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals({ mediaKind: 'file' }), + fallbackModel: 'claude-sonnet-5', + }) + + expect(result.model).toBe('claude-sonnet-5') + expect(result.tier).toBe('3') + }) + + it('classifies even a tiny toolless prompt instead of assuming the lowest tier', async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: '3' })) + + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals({ approxInputTokens: 20, toolNames: [], hasResponseFormat: false }), + fallbackModel: 'claude-sonnet-5', + }) + + expect(mockFetchGo).toHaveBeenCalledTimes(1) + expect(result.model).toBe('fireworks/kimi-k3') + expect(result.decidedBy).toBe('llm') + }) + + it('uses the router choice and applies the cost multiplier when billable', async () => { + mockFetchGo.mockResolvedValue( + routerResponse({ + choice: '2', + decidedBy: 'llm', + usage: { + model: 'glm-5.2', + inputTokens: 900, + cachedInputTokens: 0, + outputTokens: 2, + cost: 0.001, + }, + billable: true, + }) + ) + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals(), + fallbackModel: 'claude-sonnet-5', + }) + expect(result.model).toBe('fireworks/glm-5.2') + expect(result.tier).toBe('2') + expect(result.decidedBy).toBe('llm') + expect(result.billableRoutingCost).toBeCloseTo(0.002) + }) + + it('does not bill when the response omits billable (fail-safe)', async () => { + mockFetchGo.mockResolvedValue( + routerResponse({ + choice: '1', + usage: { + model: 'glm-5.2', + inputTokens: 900, + cachedInputTokens: 0, + outputTokens: 2, + cost: 0.001, + }, + }) + ) + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals(), + fallbackModel: 'claude-sonnet-5', + }) + expect(result.model).toBe('fireworks/glm-5.2') + expect(result.billableRoutingCost).toBe(0) + }) + + it('falls back when the router call fails', async () => { + mockFetchGo.mockRejectedValue(new Error('timeout')) + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals(), + fallbackModel: 'claude-sonnet-5', + }) + expect(result.model).toBe('claude-sonnet-5') + expect(result.decidedBy).toBe('fallback') + }) + + it('falls back when the router returns an unknown choice', async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: 'banana' })) + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals(), + fallbackModel: 'claude-sonnet-5', + }) + expect(result.model).toBe('claude-sonnet-5') + expect(result.decidedBy).toBe('fallback') + }) + + it('falls back when every pool model is denied by workspace permissions', async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: '1' })) + mockValidateModelProvider.mockRejectedValue(new Error('provider blocked')) + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals(), + fallbackModel: 'claude-sonnet-5', + }) + expect(result.model).toBe('claude-sonnet-5') + expect(result.decidedBy).toBe('fallback') + }) + + it('falls back when every pool model resolves to a blacklisted provider', async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: '2' })) + mockGetProviderFromModel.mockImplementation(() => { + throw new Error('Provider "fireworks" is not available') + }) + + const result = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals: makeSignals(), + fallbackModel: 'claude-sonnet-5', + }) + + expect(result.model).toBe('claude-sonnet-5') + expect(result.decidedBy).toBe('fallback') + }) + + it('serves repeated identical signals from the decision cache without re-calling the router', async () => { + mockFetchGo.mockResolvedValue(routerResponse({ choice: '1' })) + const signals = makeSignals() + + const first = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals, + fallbackModel: 'claude-sonnet-5', + }) + const second = await resolveAutoModel({ + ctx, + blockId: 'b1', + signals, + fallbackModel: 'claude-sonnet-5', + }) + + expect(first.decidedBy).toBe('llm') + expect(second.decidedBy).toBe('cache') + expect(second.model).toBe('fireworks/glm-5.2') + expect(second.tier).toBe('1') + expect(second.billableRoutingCost).toBe(0) + expect(mockFetchGo).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/model-router/resolve.ts b/apps/sim/lib/model-router/resolve.ts new file mode 100644 index 00000000000..9a4a1921570 --- /dev/null +++ b/apps/sim/lib/model-router/resolve.ts @@ -0,0 +1,308 @@ +import { createHash } from 'crypto' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { fetchGo } from '@/lib/copilot/request/go/fetch' +import { getMothershipBaseURL } from '@/lib/copilot/server/agent-url' +import { env } from '@/lib/core/config/env' +import { getCostMultiplier, isHosted } from '@/lib/core/config/env-flags' +import { validateModelProvider } from '@/ee/access-control/utils/permission-check' +import type { ExecutionContext } from '@/executor/types' +import { getProviderFromModel } from '@/providers/utils' + +const logger = createLogger('ModelRouter') + +/** + * The difficulty scale the classifier chooses from. These are the ONLY routing + * facts that cross the wire — the router never sees a model name, so the pool + * below can change without touching mothership. + */ +const TIERS = [ + { + id: '1', + hint: 'low — short mechanical work: extraction, formatting, classification, simple lookups', + }, + { + id: '2', + hint: 'standard — typical multi-step work: tool use, drafting, moderate reasoning over the inputs', + }, + { + id: '3', + hint: 'max — hardest reasoning, synthesis across many inputs, long context, high-stakes output', + }, +] as const + +type AutoTier = (typeof TIERS)[number] +type AutoTierId = AutoTier['id'] + +/** + * What the task attaches. Capability, not difficulty: it decides which models + * can serve a tier at all, so it is resolved on Sim's side and never asked of + * the classifier. + */ +export type AutoMediaKind = 'none' | 'image' | 'file' + +/** + * The sim-auto pool: media kind → tier → candidate models, ordered by + * preference. Every entry is a hosted-billable catalog model, filtered through + * the deployment blacklist and workspace model permissions at resolve time. + * + * Text and pure-image tasks ride the Fireworks OSS models on the platform key + * — glm-5.2 beats the small proprietary models per dollar, and kimi-k3 is + * natively multimodal (text + `image_url` parts only; the Fireworks chat API + * models no other content part, and glm-5.2 rejects images outright). + * Anything else attached — PDFs, audio, video, text documents — must ride the + * native providers, which are the only ones that accept those parts. + */ +const POOL: Record> = { + none: { + '1': ['fireworks/glm-5.2'], + '2': ['fireworks/glm-5.2'], + '3': ['fireworks/kimi-k3'], + }, + image: { + '1': ['gemini-3.6-flash'], + '2': ['fireworks/kimi-k3'], + '3': ['fireworks/kimi-k3'], + }, + file: { + '1': ['gemini-3.6-flash'], + '2': ['claude-sonnet-5'], + '3': ['gpt-5.6-sol'], + }, +} + +/** + * Hidden identity preamble prepended to the system prompt of every sim-auto + * execution (including fallback runs), so pool models behave consistently + * regardless of which one routing picked. + */ +export const SIM_AUTO_SYSTEM_PREAMBLE = `You are the Sim auto model on sim.ai. Respond in English unless the user writes in another language or explicitly asks for one. Do not volunteer information about which underlying model powers you; if asked directly, say you are the Sim auto model.` + +const MODEL_ROUTER_TIMEOUT_MS = 2000 +const MAX_SIGNAL_CHARS = 2000 +const DECISION_CACHE_TTL_MS = 5 * 60 * 1000 +const DECISION_CACHE_MAX_ENTRIES = 500 + +/** Compact facts about the pending agent-block task; never the full payload. */ +export interface AutoRoutingSignals { + systemPrompt?: string + lastMessage?: string + messageCount: number + toolNames: string[] + /** + * What the task attaches, which selects the pool column. Only its presence + * (`!== 'none'`) is sent to the router as `hasMedia`: the classifier picks a + * difficulty tier, and Sim owns which model can actually serve it. + */ + mediaKind: AutoMediaKind + hasResponseFormat: boolean + approxInputTokens: number +} + +/** Mirrors mothership's model-router-v1 response usage block. */ +interface ModelRouterUsage { + model: string + inputTokens: number + cachedInputTokens: number + outputTokens: number + cost: number +} + +interface ModelRouterResponse { + choice?: string + decidedBy?: string + usage?: ModelRouterUsage + billable?: boolean +} + +export interface AutoRoutingResult { + model: string + tier: AutoTierId | null + decidedBy: 'llm' | 'cache' | 'fallback' + /** + * Cost of the routing call itself in USD with the hosted cost multiplier + * applied — non-zero only when mothership marked the call billable. Absent + * `billable` in the response is treated as not billable (fail-safe). + */ + billableRoutingCost: number + usage?: ModelRouterUsage +} + +const decisionCache = new Map() + +function cacheKey(signals: AutoRoutingSignals): string { + const tokenBucket = Math.round(signals.approxInputTokens / 500) + return createHash('sha256') + .update( + JSON.stringify([ + signals.systemPrompt ?? '', + (signals.lastMessage ?? '').slice(0, 500), + [...signals.toolNames].sort(), + signals.hasResponseFormat, + signals.mediaKind, + tokenBucket, + ]) + ) + .digest('hex') +} + +function readDecisionCache(key: string): AutoTierId | null { + const entry = decisionCache.get(key) + if (!entry) return null + if (entry.expires < Date.now()) { + decisionCache.delete(key) + return null + } + return entry.tier +} + +function writeDecisionCache(key: string, tier: AutoTierId): void { + if (decisionCache.size >= DECISION_CACHE_MAX_ENTRIES) { + const oldest = decisionCache.keys().next().value + if (oldest !== undefined) decisionCache.delete(oldest) + } + decisionCache.set(key, { tier, expires: Date.now() + DECISION_CACHE_TTL_MS }) +} + +/** + * Picks the first model of the chosen tier — then of each lower tier for the + * same media kind — that is actually usable: its provider resolves and is not + * blacklisted by the deployment, and it passes workspace model permissions. + * Never crosses into another media kind's column, so a text-only model can + * never be handed an image. Returns null when nothing in the column is usable + * and the caller falls back. + */ +async function pickModelForTier( + mediaKind: AutoMediaKind, + tier: AutoTierId, + ctx: ExecutionContext +): Promise { + const startIndex = TIERS.findIndex((t) => t.id === tier) + const tried = new Set() + + for (let i = startIndex; i >= 0; i--) { + for (const model of POOL[mediaKind][TIERS[i].id]) { + // Tiers may share a model; a second permission round-trip buys nothing. + if (tried.has(model)) continue + tried.add(model) + try { + // Throws when the provider or model is blacklisted, which would + // otherwise surface as a hard block failure after resolution. + getProviderFromModel(model) + await validateModelProvider(ctx.userId, ctx.workspaceId ?? undefined, model, ctx) + return model + } catch { + logger.info('sim-auto candidate unavailable, trying the next one', { model }) + } + } + } + return null +} + +async function callModelRouter( + signals: AutoRoutingSignals, + ctx: ExecutionContext, + blockId: string +): Promise { + const baseURL = await getMothershipBaseURL({ userId: ctx.userId ?? '' }) + + const res = await fetchGo(`${baseURL}/api/model-router`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}), + }, + body: JSON.stringify({ + signals: { + systemPrompt: (signals.systemPrompt ?? '').slice(0, MAX_SIGNAL_CHARS), + lastMessage: (signals.lastMessage ?? '').slice(0, MAX_SIGNAL_CHARS), + messageCount: signals.messageCount, + toolNames: signals.toolNames, + hasMedia: signals.mediaKind !== 'none', + hasResponseFormat: signals.hasResponseFormat, + approxInputTokens: signals.approxInputTokens, + }, + candidates: TIERS.map((t) => ({ id: t.id, hint: t.hint })), + workspaceId: ctx.workspaceId, + userId: ctx.userId, + workflowId: ctx.workflowId, + blockId, + executionId: ctx.executionId, + }), + signal: AbortSignal.timeout(MODEL_ROUTER_TIMEOUT_MS), + spanName: 'sim → go /api/model-router', + operation: 'model_router', + }) + + if (!res.ok) { + logger.warn('Model router returned non-OK status', { status: res.status }) + return null + } + return (await res.json().catch(() => null)) as ModelRouterResponse | null +} + +/** + * Resolves the sim-auto pseudo-model to a concrete model for one agent-block + * execution. Never throws and never fails the workflow: any error, timeout, + * non-hosted deployment, or fully unavailable pool column falls back to + * `fallbackModel` (the block's standard default). + */ +export async function resolveAutoModel(args: { + ctx: ExecutionContext + blockId: string + signals: AutoRoutingSignals + fallbackModel: string +}): Promise { + const { ctx, blockId, signals, fallbackModel } = args + const fallback: AutoRoutingResult = { + model: fallbackModel, + tier: null, + decidedBy: 'fallback', + billableRoutingCost: 0, + } + + if (!isHosted) return fallback + + try { + // Every execution is classified: a short prompt is not a simple task, and + // a local size rule can only ever route DOWN, which is the expensive + // mistake. The cache below replays a prior router decision, never a + // locally derived one. + const key = cacheKey(signals) + const cachedTier = readDecisionCache(key) + if (cachedTier) { + const model = await pickModelForTier(signals.mediaKind, cachedTier, ctx) + if (!model) return fallback + return { model, tier: cachedTier, decidedBy: 'cache', billableRoutingCost: 0 } + } + + const response = await callModelRouter(signals, ctx, blockId) + const tier = TIERS.find((t) => t.id === response?.choice)?.id + if (!tier) { + logger.warn('sim-auto: router returned no usable choice, using fallback model', { + blockId, + choice: response?.choice, + }) + return fallback + } + + writeDecisionCache(key, tier) + const model = await pickModelForTier(signals.mediaKind, tier, ctx) + if (!model) return fallback + + const billable = response?.billable === true && (response.usage?.cost ?? 0) > 0 + return { + model, + tier, + decidedBy: 'llm', + billableRoutingCost: billable ? (response!.usage!.cost ?? 0) * getCostMultiplier() : 0, + usage: response?.usage, + } + } catch (error) { + logger.warn('sim-auto: routing failed, using fallback model', { + blockId, + error: getErrorMessage(error, 'unknown routing error'), + }) + return fallback + } +} diff --git a/apps/sim/providers/fireworks/index.test.ts b/apps/sim/providers/fireworks/index.test.ts index f20d8568630..563117f4b27 100644 --- a/apps/sim/providers/fireworks/index.test.ts +++ b/apps/sim/providers/fireworks/index.test.ts @@ -9,11 +9,13 @@ const { mockSupportsNativeStructuredOutputs, mockPrepareToolsWithUsageControl, mockExecuteTool, + mockResolveFireworksWireModel, } = vi.hoisted(() => ({ mockCreate: vi.fn(), mockSupportsNativeStructuredOutputs: vi.fn(), mockPrepareToolsWithUsageControl: vi.fn(), mockExecuteTool: vi.fn(), + mockResolveFireworksWireModel: vi.fn(), })) vi.mock('openai', () => ({ @@ -45,6 +47,7 @@ vi.mock('@/providers/fireworks/utils', () => ({ () => new ReadableStream({ start: (controller) => controller.close() }) ), checkForForcedToolUsage: vi.fn(() => ({ hasUsedForcedTool: false, usedForcedTools: [] })), + resolveFireworksWireModel: mockResolveFireworksWireModel, })) vi.mock('@/providers/trace-enrichment', () => ({ @@ -109,6 +112,7 @@ describe('fireworksProvider', () => { forcedTools: [], })) mockExecuteTool.mockResolvedValue({ success: true, output: { ok: true } }) + mockResolveFireworksWireModel.mockImplementation((stripped: string) => stripped) }) const baseRequest = { @@ -131,11 +135,27 @@ describe('fireworksProvider', () => { expect(result).toMatchObject({ content: 'hi there', - model: 'llama-v3p1-70b-instruct', + model: 'fireworks/llama-v3p1-70b-instruct', tokens: { input: 10, output: 5, total: 15 }, }) }) + it('sends the resolved wire model name while reporting the catalog id', async () => { + mockCreate.mockResolvedValueOnce(textResponse('ok')) + mockResolveFireworksWireModel.mockReturnValueOnce('accounts/fireworks/models/glm-5p2') + + const result = await fireworksProvider.executeRequest({ + ...baseRequest, + model: 'fireworks/glm-5.2', + }) + + expect(mockResolveFireworksWireModel).toHaveBeenCalledWith('glm-5.2') + expect(callBody(0).model).toBe('accounts/fireworks/models/glm-5p2') + // The catalog id is the billing/logging identity: the central cost policy, + // the usage ledger row, and the trace span all key on it. + expect(result).toMatchObject({ model: 'fireworks/glm-5.2' }) + }) + it('wraps API errors in a ProviderError', async () => { mockCreate.mockRejectedValueOnce(new Error('boom')) diff --git a/apps/sim/providers/fireworks/index.ts b/apps/sim/providers/fireworks/index.ts index 0ec540a8c2b..c7965c64072 100644 --- a/apps/sim/providers/fireworks/index.ts +++ b/apps/sim/providers/fireworks/index.ts @@ -9,6 +9,7 @@ import { formatMessagesForProvider } from '@/providers/attachments' import { checkForForcedToolUsage, createReadableStreamFromOpenAIStream, + resolveFireworksWireModel, supportsNativeStructuredOutputs, } from '@/providers/fireworks/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -89,7 +90,7 @@ export const fireworksProvider: ProviderConfig = { baseURL: 'https://api.fireworks.ai/inference/v1', }) - const requestedModel = request.model.replace(/^fireworks\//, '') + const requestedModel = resolveFireworksWireModel(request.model.replace(/^fireworks\//, '')) logger.info('Preparing Fireworks request', { model: requestedModel, @@ -165,7 +166,10 @@ export const fireworksProvider: ProviderConfig = { ) const streamingResult = createStreamingExecution({ - model: requestedModel, + // Echo the catalog id, never the wire name: it is the billing and + // logging identity (cost policy, ledger row, trace span, model + // breakdown) the way every other Sim-hosted provider reports it. + model: request.model, providerStartTime, providerStartTimeISO, timing: { kind: 'simple', segmentName: request.model }, @@ -181,8 +185,10 @@ export const fireworksProvider: ProviderConfig = { total: usage.total_tokens, } + // Pricing keys on the catalog id (fireworks/), not the wire + // name — static hosted entries price; dynamic ids stay unpriced. const costResult = calculateCost( - requestedModel, + request.model, usage.prompt_tokens, usage.completion_tokens ) @@ -547,7 +553,8 @@ export const fireworksProvider: ProviderConfig = { } if (request.stream) { - const accumulatedCost = calculateCost(requestedModel, tokens.input, tokens.output) + // Pricing keys on the catalog id (fireworks/), not the wire name. + const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output) const toolCost = sumToolCosts(toolResults) const finalCost = { input: accumulatedCost.input, @@ -557,7 +564,7 @@ export const fireworksProvider: ProviderConfig = { } const streamingResult = createStreamingExecution({ - model: requestedModel, + model: request.model, providerStartTime, providerStartTimeISO, timing: { @@ -591,7 +598,7 @@ export const fireworksProvider: ProviderConfig = { return { content, - model: requestedModel, + model: request.model, tokens, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, toolResults: toolResults.length > 0 ? toolResults : undefined, diff --git a/apps/sim/providers/fireworks/utils.test.ts b/apps/sim/providers/fireworks/utils.test.ts new file mode 100644 index 00000000000..79f51b4b866 --- /dev/null +++ b/apps/sim/providers/fireworks/utils.test.ts @@ -0,0 +1,18 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { resolveFireworksWireModel } from '@/providers/fireworks/utils' + +describe('resolveFireworksWireModel', () => { + it('maps the static hosted catalog ids to their serverless resource paths', () => { + expect(resolveFireworksWireModel('glm-5.2')).toBe('accounts/fireworks/models/glm-5p2') + expect(resolveFireworksWireModel('kimi-k3')).toBe('accounts/fireworks/models/kimi-k3') + }) + + it('passes user-configured dynamic ids through untouched', () => { + expect(resolveFireworksWireModel('accounts/acme/models/custom')).toBe( + 'accounts/acme/models/custom' + ) + }) +}) diff --git a/apps/sim/providers/fireworks/utils.ts b/apps/sim/providers/fireworks/utils.ts index c4abd3111bd..23793f26bfe 100644 --- a/apps/sim/providers/fireworks/utils.ts +++ b/apps/sim/providers/fireworks/utils.ts @@ -4,6 +4,26 @@ import { createOpenAICompatibleAgentEventStream } from '@/providers/openai-compa import type { AgentStreamEvent } from '@/providers/stream-events' import { checkForForcedToolUsageOpenAI } from '@/providers/utils' +/** + * Wire model names for the static hosted Fireworks catalog entries (the + * sim-auto pool). Catalog ids are the short `fireworks/` form; the + * Fireworks API requires the full serverless resource path. User-configured + * dynamic ids are sent verbatim after prefix-stripping, exactly as before — + * only ids in this map are rewritten. + */ +const FIREWORKS_WIRE_NAMES: Record = { + 'glm-5.2': 'accounts/fireworks/models/glm-5p2', + 'kimi-k3': 'accounts/fireworks/models/kimi-k3', +} + +/** + * Resolves the wire model name Fireworks expects from a prefix-stripped + * catalog id. + */ +export function resolveFireworksWireModel(strippedModel: string): string { + return FIREWORKS_WIRE_NAMES[strippedModel] ?? strippedModel +} + /** * Checks if a model supports native structured outputs (json_schema). * Fireworks AI supports structured outputs across their inference API. diff --git a/apps/sim/providers/models.test.ts b/apps/sim/providers/models.test.ts index e8deb2a9311..e58b4b65f6d 100644 --- a/apps/sim/providers/models.test.ts +++ b/apps/sim/providers/models.test.ts @@ -5,12 +5,15 @@ import { describe, expect, it } from 'vitest' import { getBaseModelProviders, getHostedModels, + getModelPricing, getModelsWithPromptCaching, getPromptCachingMinimumTokens, + getProviderModels, getThinkingStreamVisibility, isModelDeprecated, orderModelIdsByReleaseDate, PROVIDER_DEFINITIONS, + updateFireworksModels, } from '@/providers/models' import { supportsPromptCaching } from '@/providers/utils' @@ -370,6 +373,43 @@ describe('xai provider definition', () => { }) }) +describe('fireworks static catalog (the sim-auto pool)', () => { + const poolModels = ['fireworks/glm-5.2', 'fireworks/kimi-k3'] + + it('is included in getHostedModels since Sim provides the Fireworks key server-side', () => { + for (const model of poolModels) { + expect(getHostedModels()).toContain(model) + } + }) + + it('prices every pool model so hosted usage is billable', () => { + for (const model of poolModels) { + const pricing = getModelPricing(model) + expect(pricing?.input).toBeGreaterThan(0) + expect(pricing?.output).toBeGreaterThan(0) + } + }) + + it('survives a dynamic model sync, which merges rather than replaces', () => { + updateFireworksModels(['fireworks/accounts/acme/models/custom']) + + for (const model of poolModels) { + expect(getHostedModels()).toContain(model) + expect(getModelPricing(model)?.input).toBeGreaterThan(0) + } + expect(getProviderModels('fireworks')).toContain('fireworks/accounts/acme/models/custom') + }) + + it('does not duplicate a pool model the dynamic listing also returns', () => { + updateFireworksModels(['fireworks/glm-5.2']) + + expect(getProviderModels('fireworks').filter((id) => id === 'fireworks/glm-5.2')).toHaveLength( + 1 + ) + expect(getModelPricing('fireworks/glm-5.2')?.input).toBeGreaterThan(0) + }) +}) + describe('isModelDeprecated', () => { it('returns true for a catalogued deprecated model (case-insensitive)', () => { const id = firstDeprecatedModelId() diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index aac911d4d5b..98be92cd78d 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -164,7 +164,47 @@ export const PROVIDER_DEFINITIONS: Record = { toolUsageControl: true, }, contextInformationAvailable: false, - models: [], + /** + * Static Fireworks serverless entries: the sim-auto routing pool. These are + * hosted-billable (platform FIREWORKS_API_KEY) and priced at the Fireworks + * serverless rate, which differs from the vendors' native rates. Text-only: + * the Fireworks serving endpoints reject image inputs. User-configured + * `fireworks/` ids remain dynamic/BYO-key as before. + */ + models: [ + { + id: 'fireworks/glm-5.2', + pricing: { + input: 1.4, + cachedInput: 0.14, + output: 4.4, + updatedAt: '2026-07-29', + }, + capabilities: { + toolUsageControl: true, + maxOutputTokens: 131072, + }, + contextWindow: 1048576, + releaseDate: '2026-06-13', + recommended: true, + }, + { + id: 'fireworks/kimi-k3', + pricing: { + input: 3.0, + cachedInput: 0.3, + output: 15.0, + updatedAt: '2026-07-29', + }, + capabilities: { + toolUsageControl: true, + maxOutputTokens: 1048576, + }, + contextWindow: 1048576, + releaseDate: '2026-07-16', + recommended: true, + }, + ], }, together: { id: 'together', @@ -4037,6 +4077,19 @@ export function suggestModelIdsForUnknownModel(_modelId: string, limit = 5): str .slice(0, limit) } +/** + * Pseudo-model id for the agent block's automatic model. Not a real catalog + * entry: at execution time the resolver (lib/model-router) classifies the task + * via mothership and swaps in a concrete model from the hosted Fireworks pool + * before any provider/pricing lookup runs. Hosted Sim only. + */ +export const SIM_AUTO_MODEL_ID = 'sim-auto' + +/** True when the configured model is the sim-auto pseudo-model. */ +export function isAutoModel(model: string): boolean { + return model.trim().toLowerCase() === SIM_AUTO_MODEL_ID +} + export function getBaseModelProviders(): Record { return Object.entries(PROVIDER_DEFINITIONS) .filter( @@ -4166,6 +4219,11 @@ export function getHostedModels(): string[] { ...getProviderModels('zai'), ...getProviderModels('xai'), ...getProviderModels('kimi'), + // The STATIC Fireworks catalog only (the sim-auto pool, platform key) — + // deliberately not the live provider list, which `updateFireworksModels` + // merges a workspace's own dynamic ids into. Those must never enter the + // hosted set: it gates both billing and the platform-key handout. + ...STATIC_FIREWORKS_MODELS.map((model) => model.id), ] } @@ -4231,16 +4289,31 @@ export function updateLiteLLMModels(models: string[]): void { })) } +/** + * The static Fireworks catalog (the hosted sim-auto pool), captured at module + * load before any dynamic sync runs. Hosted billing, pricing, and the agent + * block's API-key condition all key on these ids, so a workspace's own + * Fireworks models are merged on top of them rather than replacing them — + * unlike the other dynamic providers, whose static list is empty. + */ +const STATIC_FIREWORKS_MODELS = PROVIDER_DEFINITIONS.fireworks.models + export function updateFireworksModels(models: string[]): void { - PROVIDER_DEFINITIONS.fireworks.models = models.map((modelId) => ({ - id: modelId, - pricing: { - input: 0, - output: 0, - updatedAt: new Date().toISOString().split('T')[0], - }, - capabilities: {}, - })) + const staticIds = new Set(STATIC_FIREWORKS_MODELS.map((model) => model.id.toLowerCase())) + PROVIDER_DEFINITIONS.fireworks.models = [ + ...STATIC_FIREWORKS_MODELS, + ...models + .filter((modelId) => !staticIds.has(modelId.toLowerCase())) + .map((modelId) => ({ + id: modelId, + pricing: { + input: 0, + output: 0, + updatedAt: new Date().toISOString().split('T')[0], + }, + capabilities: {}, + })), + ] } export function updateTogetherModels(models: string[]): void { diff --git a/apps/sim/providers/settled-tool-streams.test.ts b/apps/sim/providers/settled-tool-streams.test.ts index d7d1ea5c4c9..0cba631abe4 100644 --- a/apps/sim/providers/settled-tool-streams.test.ts +++ b/apps/sim/providers/settled-tool-streams.test.ts @@ -104,6 +104,7 @@ vi.mock('@/providers/fireworks/utils', () => ({ })), createReadableStreamFromOpenAIStream: vi.fn(() => createEmptyStream()), supportsNativeStructuredOutputs: vi.fn(() => true), + resolveFireworksWireModel: vi.fn((stripped: string) => stripped), })) vi.mock('@/providers/openrouter/utils', () => ({ checkForForcedToolUsage: vi.fn(() => ({ From 811a39ec05eec3a3f485e2f526c1089c6ed5b66d Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 30 Jul 2026 13:40:40 -0700 Subject: [PATCH 05/21] fix(integrations): show family service accounts on every product they authenticate (#6102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(integrations): show family service accounts on every product they authenticate An Atlassian API token authenticates Jira, Jira Service Management, and Confluence alike, so it is modeled as an `atlassian` pseudo-provider whose only service is named "Atlassian Service Account". Every credential display surface resolved through `getServiceConfigByProviderId`, which walks OAUTH_PROVIDERS in declaration order — so the credential resolved to that pseudo-service instead of to any product. The result: adding a service account from the Jira page, through a modal titled "Add Jira service account", produced a credential that appeared under neither Jira, JSM, nor Confluence, was titled "Atlassian Service Account" on its detail page, and lost its brand tile and category on the list. The same bug hid a Google service account everywhere except Gmail. - match credentials with `credentialProviderMatchesService`, which accepts a service's OAuth id or its service-account id - add `lib/integrations/credential-display.ts` as the single resolver for catalog join, mark, and copy, replacing three duplicated lookups that keyed the catalog by OAuth service *display name* — the reason the pseudo-service fell off the map - derive "family service account" from the catalog (a service-account id serving >1 integration) rather than hardcoding vendors, so a new integration joining a family needs no edit - title service-account detail pages by credential name, subtitle them with their reach, and state that reach up front on the connect form - keep the service description as the detail subtitle for every non-family credential, unchanged No schema, migration, contract, or persisted value changes; resolution is computed at render time from static config. Coverage for all 22 service-account provider ids is pinned in tests, including that the index and the predicate the Connected list filters on cannot drift apart. * chore(icons): use Atlassian's gradient marks for Jira and Confluence Replaces the flat #1868DB Jira and Confluence marks with Atlassian's gradient versions, matching the Atlassian mark added alongside them. - gradient ids go through `useId()` rather than the source SVGs' static ids, which would collide wherever two of these icons render on one page — the integrations list and the landing loops both do - pads the Atlassian viewBox so its artwork fills ~78% of the box, matching the inset Atlassian ships on the Jira and Confluence marks; without it the mark renders ~30% heavier than its siblings in the same tile Visual-only, but these marks render in ~60 files, so it is split from the credential fix to stay independently revertable. * fix(integrations): route the editor's service-account setup modal through the shared target The workflow editor's credential selector passed the OAuth service's own name and icon straight to ConnectServiceAccountModal, so opening the setup form from a Jira block titled it "Add Jira service account" while the integrations page and the chat — both of which already resolve through `useServiceAccountConnectTarget` — titled the same form "Add Atlassian service account". That is the exact confusion this branch set out to remove, surviving on the one surface that bypassed the shared resolver. * docs(atlassian): correct the service-account setup path and cover all three products The setup section could not be followed. It sent readers to a "Settings → Integrations tab" that does not exist (Integrations is a top-level workspace module) and told them to search the integrations list for "Atlassian Service Account", which matches no catalog entry — the catalog lists Jira, Jira Service Management, and Confluence. The page also described the credential as covering "Jira and Confluence" while listing Jira Service Management scopes, and the product now spells the coverage out in the connect form. - correct the path: Integrations -> Jira/JSM/Confluence -> Add to Sim -> Add service account - name all three products consistently, and state that one service account covers them - match the real button label ("Add service account") --- .../atlassian-service-account.mdx | 27 +- .../[block]/integration-block-detail.tsx | 25 +- .../connect-service-account-modal.tsx | 12 + .../use-service-account-connect.ts | 28 +- .../connected-credential-detail.tsx | 53 ++- .../integrations/integrations.tsx | 40 ++- .../credential-selector.tsx | 10 +- .../search-modal/integration-search-items.ts | 20 +- apps/sim/components/icons.tsx | 138 +++++++- .../integrations/credential-display.test.ts | 237 +++++++++++++ .../lib/integrations/credential-display.ts | 310 ++++++++++++++++++ apps/sim/lib/integrations/index.ts | 9 + apps/sim/lib/integrations/oauth-service.ts | 2 +- apps/sim/lib/oauth/oauth.ts | 7 +- apps/sim/lib/oauth/utils.ts | 36 ++ 15 files changed, 854 insertions(+), 100 deletions(-) create mode 100644 apps/sim/lib/integrations/credential-display.test.ts create mode 100644 apps/sim/lib/integrations/credential-display.ts diff --git a/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx b/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx index de38fe8a69a..5312576ca84 100644 --- a/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx +++ b/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx @@ -1,6 +1,6 @@ --- title: Atlassian Service Accounts -description: Set up an Atlassian service account with a scoped API token to use Jira and Confluence in Sim workflows +description: Set up an Atlassian service account with a scoped API token to use Jira, Jira Service Management, and Confluence in Sim workflows --- import { Callout } from 'fumadocs-ui/components/callout' @@ -8,9 +8,11 @@ import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' import { FAQ } from '@/components/ui/faq' -Atlassian service accounts let your workflows authenticate to Jira and Confluence as a non-human bot user — independent of any individual employee's account. Each service account has its own email, its own permissions, and its own API tokens, all managed centrally in admin.atlassian.com. +Atlassian service accounts let your workflows authenticate to Jira, Jira Service Management, and Confluence as a non-human bot user — independent of any individual employee's account. Each service account has its own email, its own permissions, and its own API tokens, all managed centrally in admin.atlassian.com. -This is the recommended way to use Jira and Confluence in production workflows: no one person's OAuth consent expires, the bot's permissions are auditable, and access can be revoked without touching anyone's personal account. +This is the recommended way to use Atlassian products in production workflows: no one person's OAuth consent expires, the bot's permissions are auditable, and access can be revoked without touching anyone's personal account. + +One service account covers all three products. You add it once, and it appears as a connected credential on the Jira, Jira Service Management, and Confluence integration pages alike — there is no separate credential to create per product. ## Prerequisites @@ -124,12 +126,17 @@ Your Atlassian site domain is the URL you use to access Jira or Confluence in yo - Open your workspace **Settings** and go to the **Integrations** tab + Open **Integrations** in your workspace sidebar - Search for "Atlassian Service Account" and click it + Open **Jira**, **Jira Service Management**, or **Confluence** — any of the three works, since they share one service account - {/* TODO(screenshot): Integrations page with "Atlassian Service Account" in the service list */} + {/* TODO(screenshot): Integrations page with Jira in the list */} + + + Click **Add to Sim** and choose **Add service account** + + {/* TODO(screenshot): Jira integration page with the "Add to Sim" dropdown open */} Paste the API token, enter the site domain (e.g. `your-team.atlassian.net`), and optionally set a display name and description @@ -145,15 +152,17 @@ Your Atlassian site domain is the URL you use to access Jira or Confluence in yo
- Click **Add Service Account**. Sim verifies the token by calling Atlassian's `/myself` endpoint through the gateway — if it fails, you'll see a specific error explaining what went wrong. + Click **Add service account**. Sim verifies the token by calling Atlassian's `/myself` endpoint through the gateway — if it fails, you'll see a specific error explaining what went wrong. The token, domain, and discovered cloudId are encrypted before being stored. +Once added, the credential is listed under **Connected** on all three Atlassian integration pages. It is named after the service account's own Atlassian display name, so several service accounts on the same site stay easy to tell apart. + ## Using the Service Account in Workflows -Add a Jira or Confluence block to your workflow. In the credential dropdown, your Atlassian service account appears alongside any OAuth credentials. Select it and configure the block as you normally would. +Add a Jira, Jira Service Management, or Confluence block to your workflow. In the credential dropdown, your Atlassian service account appears alongside any OAuth credentials. Select it and configure the block as you normally would.
{ if (!oauthService) return [] return credentials.filter( (c) => (c.type === 'oauth' || c.type === 'service_account') && c.providerId && - getServiceConfigByProviderId(c.providerId)?.providerId === oauthService.providerId + credentialProviderMatchesService(c.providerId, oauthService) ) }, [credentials, oauthService]) const [serviceAccountOpen, setServiceAccountOpen] = useState(false) @@ -112,7 +121,7 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration { value: CONNECT_MODE.serviceAccount, label: serviceAccountConnectLabel, - icon: oauthService.serviceIcon, + icon: serviceAccountTarget?.serviceIcon ?? oauthService.serviceIcon, }, ] : [] @@ -170,14 +179,14 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration serviceIcon={oauthService.serviceIcon} /> )} - {hasServiceAccount && oauthService?.serviceAccountProviderId && ( + {hasServiceAccount && serviceAccountTarget && ( )}
- {credential.description || oauthService?.serviceName} + {credential.description || resolveCredentialDisplay(credential).subtitle}
diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx index bb83405f7f6..22c36c6a71b 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx @@ -22,6 +22,7 @@ import { getTokenServiceAccountDescriptor, type TokenServiceAccountProviderId, } from '@/lib/credentials/token-service-accounts/descriptors' +import { getServiceAccountCoverageSentence } from '@/lib/integrations/credential-display' import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, SLACK_CUSTOM_BOT_PROVIDER_ID, @@ -60,6 +61,16 @@ function openDocs(url: string): void { */ const ATLASSIAN_DOMAIN_HINT_REGEX = /^[a-z0-9-]+\.atlassian\.net$/i +/** + * States the site-wide reach of the token up front. Users reaching this modal + * from the Jira page were left unsure whether they had connected Jira or Jira + * Service Management; the credential covers both, plus Confluence. Derived from + * the catalog so it cannot drift as Atlassian integrations are added. + */ +const ATLASSIAN_COVERAGE_HINT = getServiceAccountCoverageSentence( + ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID +) + /** * Maps server `error.code` values returned by the Atlassian service-account * route to user-facing messages. Falls back to {@link FALLBACK_ERROR_MESSAGE} @@ -524,6 +535,7 @@ function AtlassianServiceAccountModal({ ? 'Atlassian sites usually look like your-team.atlassian.net.' : undefined } + hint={ATLASSIAN_COVERAGE_HINT} /> /** @@ -71,6 +88,15 @@ export function useServiceAccountConnectTarget({ ? 'Set up a custom bot' : `Add ${getServiceAccountConnectNoun(serviceAccountProviderId)}` - return { serviceAccountProviderId, serviceName, serviceIcon, label, hidden } + const familyName = getServiceAccountFamilyName(serviceAccountProviderId) + const familyIcon = getServiceAccountFamilyIcon(serviceAccountProviderId) + + return { + serviceAccountProviderId, + serviceName: familyName ?? serviceName, + serviceIcon: familyIcon ?? serviceIcon, + label, + hidden, + } }, [serviceAccountProviderId, serviceName, serviceIcon, isSlackBot, hidden]) } diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx index 11735375f5f..4ddfb4a5024 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx @@ -16,8 +16,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useRouter } from 'next/navigation' import { writeOAuthReturnContext } from '@/lib/credentials/client-state' -import { INTEGRATIONS, resolveOAuthServiceForIntegration } from '@/lib/integrations' -import { getServiceConfigByProviderId } from '@/lib/oauth' +import { resolveCredentialDisplay } from '@/lib/integrations' import { AddPeopleModal, CredentialDetailHeading, @@ -97,27 +96,18 @@ export function ConnectedCredentialDetail({ [oauthServiceNameByProviderId] ) - const serviceConfig = useMemo(() => { - if (!credential?.providerId) return null - return getServiceConfigByProviderId(credential.providerId) - }, [credential]) - /** - * Resolve the integration block type from the credential's OAuth service so - * the header tile can render with the same brand background used by the rows - * on the integrations list page. Several integrations can share one service - * (e.g. Jira and Jira Service Management); the one named after the service - * is preferred since it is the service's canonical integration. + * Service, brand tile, and copy all come from the shared resolver so this + * page, the integrations list, and the Cmd-K search agree on how a credential + * is named and branded — a family service account reads as its family + * ("Atlassian"), not as whichever product the provider walk happened to hit. */ - const integrationBlockType = useMemo(() => { - if (!serviceConfig) return '' - const candidates = INTEGRATIONS.filter( - (i) => resolveOAuthServiceForIntegration(i)?.providerId === serviceConfig.providerId - ) - const serviceName = serviceConfig.name.toLowerCase() - const canonical = candidates.find((i) => i.name.toLowerCase() === serviceName) - return (canonical ?? candidates[0])?.type ?? '' - }, [serviceConfig]) + const display = useMemo( + () => (credential ? resolveCredentialDisplay(credential) : null), + [credential] + ) + const serviceConfig = display?.service ?? null + const integrationBlockType = display?.blockType ?? '' const handleReconnectOAuth = async () => { if (!credential || credential.type !== 'oauth' || !credential.providerId || !workspaceId) return @@ -206,7 +196,7 @@ export function ConnectedCredentialDetail({ : handleReconnectOAuth } disabled={connectOAuthService.isPending} - leftIcon={serviceConfig?.icon} + leftIcon={display?.icon ?? undefined} > Reconnect @@ -242,19 +232,16 @@ export function ConnectedCredentialDetail({ ) } - const serviceLabel = - serviceConfig?.name || resolveProviderLabel(credential.providerId) || 'Unknown service' + const headingTitle = + display?.detailTitle || resolveProviderLabel(credential.providerId) || 'Unknown service' return ( <> } - /> + display?.icon ? ( + ) : (
@@ -263,8 +250,8 @@ export function ConnectedCredentialDetail({
) } - title={serviceLabel} - subtitle={serviceConfig?.description || 'Connected service'} + title={headingTitle} + subtitle={display?.detailSubtitle ?? 'Connected service'} /> @@ -335,8 +322,8 @@ export function ConnectedCredentialDetail({ onOpenChange={setReconnectOpen} workspaceId={workspaceId} serviceAccountProviderId={credential.providerId as ServiceAccountProviderId} - serviceName={serviceConfig?.name || credential.displayName} - serviceIcon={serviceConfig?.icon as ComponentType<{ className?: string }>} + serviceName={display?.familyName || serviceConfig?.name || credential.displayName} + serviceIcon={display?.icon as ComponentType<{ className?: string }>} credentialId={credential.id} credentialDisplayName={credential.displayName} credentialDescription={credential.description ?? undefined} diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx index 26c2aa0b3a7..2f04ac50d33 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx @@ -20,8 +20,8 @@ import { formatIntegrationType, INTEGRATIONS, type Integration, + resolveCredentialDisplay, } from '@/lib/integrations' -import { getServiceConfigByProviderId } from '@/lib/oauth' import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/integrations/components/integration-tabs-header' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' @@ -53,11 +53,6 @@ const FEATURED_INTEGRATIONS: readonly Integration[] = (() => { ) })() -/** Lookup integration metadata by OAuth service display name (case-insensitive). */ -const INTEGRATION_BY_LOWER_NAME: ReadonlyMap = new Map( - INTEGRATIONS.map((i) => [i.name.toLowerCase(), i]) -) - const ALL_CATEGORY_SECTIONS: readonly { label: string; integrations: Integration[] }[] = (() => { const grouped = new Map() for (const integration of INTEGRATIONS) { @@ -105,7 +100,12 @@ interface ConnectedDisplayItem { credential: WorkspaceCredential name: string description: string - serviceName: string + /** + * Extra haystack for the search box: the service name plus every integration + * the credential authenticates, so searching "jira" surfaces an Atlassian + * service account even when the user has replaced its description. + */ + searchText: string integrationType: string | null blockType: string slug: string @@ -165,20 +165,24 @@ export function Integrations() { const connectedItems = useMemo(() => { return oauthCredentials.flatMap((credential) => { - if (!credential.providerId) return [] - const service = getServiceConfigByProviderId(credential.providerId) - if (!service) return [] - const integration = INTEGRATION_BY_LOWER_NAME.get(service.name.toLowerCase()) + const display = resolveCredentialDisplay(credential) + if (!display.service || !display.icon) return [] return [ { credential, name: credential.displayName, - description: credential.description || `${service.name} integration`, - serviceName: service.name, - integrationType: integration?.integrationType ?? null, - blockType: integration?.type ?? '', - slug: integration?.slug ?? '', - icon: service.icon as ComponentType<{ className?: string }>, + description: credential.description || display.subtitle, + searchText: [ + display.familyName, + display.service.name, + ...display.coveredIntegrations.map((i) => i.name), + ] + .filter(Boolean) + .join(' '), + integrationType: display.integration?.integrationType ?? null, + blockType: display.blockType, + slug: display.integration?.slug ?? '', + icon: display.icon, }, ] }) @@ -264,7 +268,7 @@ export function Integrations() { return ( item.name.toLowerCase().includes(normalizedSearch) || item.description.toLowerCase().includes(normalizedSearch) || - item.serviceName.toLowerCase().includes(normalizedSearch) + item.searchText.toLowerCase().includes(normalizedSearch) ) }) }, [ diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/credential-selector.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/credential-selector.tsx index 6674f8f6d6f..466224ba63f 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/credential-selector.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/credential-selector.tsx @@ -511,16 +511,14 @@ export function CredentialSelector({ /> )} - {showSetupModal && serviceAccountService?.serviceAccountProviderId && ( + {showSetupModal && serviceAccountTarget && ( { setStoreValue(newCredentialId) refetchCredentials() diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts index 968d1bbfad2..638eb4d2497 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts @@ -1,6 +1,5 @@ import type { ComponentType } from 'react' -import { blockTypeToIconMap, INTEGRATIONS } from '@/lib/integrations' -import { getServiceConfigByProviderId } from '@/lib/oauth' +import { blockTypeToIconMap, INTEGRATIONS, resolveCredentialDisplay } from '@/lib/integrations' import { CONNECT_MODE, CONNECT_QUERY_PARAM, @@ -11,12 +10,6 @@ import type { WorkspaceCredential } from '@/hooks/queries/credentials' /** Fallback brand color for credentials whose integration metadata cannot be resolved. */ const FALLBACK_BG_COLOR = '#6B7280' -/** - * Module-level lookup of integration metadata by OAuth service display name - * (case-insensitive). Mirrors the same map in `integrations.tsx`. - */ -const INTEGRATION_BY_LOWER_NAME = new Map(INTEGRATIONS.map((i) => [i.name.toLowerCase(), i])) - /** * Module-level base array of resolvable integrations (entries without a * registered icon are dropped, matching the catalog's `if (!Icon) return null` @@ -76,19 +69,16 @@ export function buildConnectedAccountSearchItems( ): IntegrationSearchItem[] { return credentials.flatMap((credential) => { if (credential.type !== 'oauth' && credential.type !== 'service_account') return [] - if (!credential.providerId) return [] - - const service = getServiceConfigByProviderId(credential.providerId) - if (!service) return [] - const integration = INTEGRATION_BY_LOWER_NAME.get(service.name.toLowerCase()) + const display = resolveCredentialDisplay(credential) + if (!display.service || !display.icon) return [] return [ { id: credential.id, name: credential.displayName, - icon: service.icon as ComponentType<{ className?: string }>, - bgColor: integration?.bgColor ?? FALLBACK_BG_COLOR, + icon: display.icon, + bgColor: display.integration?.bgColor ?? FALLBACK_BG_COLOR, href: `/workspace/${workspaceId}/integrations/connected/${credential.id}`, }, ] diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index 5a2e8424b26..deb3548b350 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -2221,21 +2221,108 @@ export function EyeIcon(props: SVGProps) { ) } +/** + * Corporate Atlassian mark, used for family-wide Atlassian credentials — one + * API token authenticates Jira, Jira Service Management, and Confluence, so no + * single product mark represents it. Individual products keep their own icons. + */ +export function AtlassianIcon(props: SVGProps) { + const id = useId() + const gradientId = `atlassian_gradient_${id}` + + return ( + + ) +} + export function ConfluenceIcon(props: SVGProps) { + const id = useId() + const topGradientId = `confluence_top_${id}` + const bottomGradientId = `confluence_bottom_${id}` + return ( ) @@ -2768,19 +2855,58 @@ export function LinkupIcon(props: SVGProps) { } export function JiraIcon(props: SVGProps) { + const id = useId() + const middleGradientId = `jira_middle_${id}` + const bottomGradientId = `jira_bottom_${id}` + return ( ) diff --git a/apps/sim/lib/integrations/credential-display.test.ts b/apps/sim/lib/integrations/credential-display.test.ts new file mode 100644 index 00000000000..5305a30066f --- /dev/null +++ b/apps/sim/lib/integrations/credential-display.test.ts @@ -0,0 +1,237 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + getIntegrationsForCredentialProvider, + getServiceAccountCoverageSentence, + getServiceAccountFamilyName, + isFamilyServiceAccount, + resolveCredentialDisplay, +} from '@/lib/integrations/credential-display' +import integrationsJson from '@/lib/integrations/integrations.json' +import { resolveOAuthServiceForIntegration } from '@/lib/integrations/oauth-service' +import type { Integration } from '@/lib/integrations/types' +import { OAUTH_PROVIDERS } from '@/lib/oauth/oauth' +import { credentialProviderMatchesService } from '@/lib/oauth/utils' + +const INTEGRATIONS = integrationsJson.integrations as readonly Integration[] + +/** + * Every catalog integration each service-account provider id authenticates. + * + * This table is the regression guard for the family-credential fix. Credential + * display resolves through `OAUTH_PROVIDERS`, which is walked in declaration + * order — reordering it, or adding a provider that claims an existing + * service-account id, silently changes which product pages a credential appears + * on. Two entries here are the fix itself: + * + * - `atlassian-service-account` previously matched **nothing** (it resolves to + * the `Atlassian Service Account` pseudo-service, whose providerId equals no + * product's), so a service account added from the Jira page vanished from + * Jira, Jira Service Management, and Confluence alike. + * - `google-service-account` previously matched Gmail only, because Gmail is the + * first Google service declared. + * + * Every other row must stay exactly as it was before the fix. + */ +const EXPECTED_COVERAGE: Record = { + 'airtable-service-account': ['airtable'], + 'asana-service-account': ['asana'], + 'atlassian-service-account': ['confluence', 'jira', 'jira-service-management'], + 'attio-service-account': ['attio'], + 'box-service-account': ['box'], + 'calcom-service-account': ['cal-com'], + 'claude-platform-service-account': [], + 'clickup-service-account': ['clickup'], + 'google-service-account': [ + 'gmail', + 'google-bigquery', + 'google-calendar', + 'google-contacts', + 'google-docs', + 'google-drive', + 'google-forms', + 'google-groups', + 'google-meet', + 'google-sheets', + 'google-slides', + 'google-tasks', + 'google-vault', + ], + 'hubspot-service-account': ['hubspot'], + 'linear-service-account': ['linear'], + 'monday-service-account': ['monday'], + 'notion-service-account': ['notion'], + 'pipedrive-service-account': ['pipedrive'], + 'salesforce-service-account': ['salesforce'], + 'shopify-service-account': ['shopify'], + 'slack-custom-bot': ['slack'], + 'trello-service-account': ['trello'], + 'wealthbox-service-account': ['wealthbox'], + 'webflow-service-account': ['webflow'], + 'zoom-service-account': ['zoom'], +} + +/** Every provider id some service designates as its service-account id. */ +const REGISTERED_SERVICE_ACCOUNT_IDS = [ + ...new Set( + Object.values(OAUTH_PROVIDERS).flatMap((provider) => + Object.values(provider.services).flatMap((service) => + service.serviceAccountProviderId ? [service.serviceAccountProviderId] : [] + ) + ) + ), +].sort() + +const serviceAccount = (providerId: string) => ({ + type: 'service_account', + displayName: 'Automation Bot', + providerId, +}) + +describe('service-account coverage', () => { + it('pins the table to exactly the registered service-account provider ids', () => { + expect(REGISTERED_SERVICE_ACCOUNT_IDS).toEqual(Object.keys(EXPECTED_COVERAGE).sort()) + }) + + it.each(Object.entries(EXPECTED_COVERAGE))( + '%s authenticates the expected integrations', + (providerId, expectedSlugs) => { + const slugs = getIntegrationsForCredentialProvider(providerId) + .map((i) => i.slug) + .sort() + expect(slugs).toEqual([...expectedSlugs].sort()) + } + ) + + /** + * The integration detail page filters its "Connected" list with the + * predicate, not with this index, so the two must not drift. Without this + * the index could be right while the page still hid the credential — the + * original bug. + */ + it('agrees with the predicate the Connected list actually filters on', () => { + for (const providerId of REGISTERED_SERVICE_ACCOUNT_IDS) { + const covered = new Set(getIntegrationsForCredentialProvider(providerId).map((i) => i.slug)) + + for (const integration of INTEGRATIONS) { + const service = resolveOAuthServiceForIntegration(integration) + if (!service) continue + expect( + credentialProviderMatchesService(providerId, service), + `${providerId} vs ${integration.slug}` + ).toBe(covered.has(integration.slug)) + } + } + }) + + it('treats only multi-integration service accounts as families', () => { + const families = REGISTERED_SERVICE_ACCOUNT_IDS.filter(isFamilyServiceAccount) + expect(families).toEqual(['atlassian-service-account', 'google-service-account']) + }) + + it('names families after the vendor, not one of its products', () => { + expect(getServiceAccountFamilyName('atlassian-service-account')).toBe('Atlassian') + expect(getServiceAccountFamilyName('google-service-account')).toBe('Google') + expect(getServiceAccountFamilyName('notion-service-account')).toBeNull() + }) +}) + +describe('resolveCredentialDisplay', () => { + it('gives an Atlassian service account a vendor identity and a Jira brand tile', () => { + const display = resolveCredentialDisplay(serviceAccount('atlassian-service-account')) + + expect(display.familyName).toBe('Atlassian') + expect(display.detailTitle).toBe('Automation Bot') + expect(display.subtitle).toBe( + 'Atlassian service account · Confluence, Jira, and Jira Service Management' + ) + // Without a catalog fallback the name lookup misses and the row loses both + // its brand tile and its category filter membership. + expect(display.blockType).toBe('jira') + expect(display.integration?.integrationType).toBeTruthy() + }) + + it('states a count rather than enumerating 13 Google integrations', () => { + const display = resolveCredentialDisplay(serviceAccount('google-service-account')) + + expect(display.familyName).toBe('Google') + expect(display.subtitle).toBe('Google service account · all 13 Google integrations') + }) + + it('uses each vendor own noun for non-family service accounts', () => { + expect(resolveCredentialDisplay(serviceAccount('slack-custom-bot')).subtitle).toBe( + 'Slack custom bot' + ) + expect(resolveCredentialDisplay(serviceAccount('notion-service-account')).subtitle).toBe( + 'Notion integration secret' + ) + }) + + it('leaves OAuth credentials titled by their service', () => { + const display = resolveCredentialDisplay({ + type: 'oauth', + displayName: 'someone@example.com', + providerId: 'jira', + }) + + expect(display.familyName).toBeNull() + expect(display.detailTitle).toBe('Jira') + expect(display.subtitle).toBe('Jira integration') + expect(display.blockType).toBe('jira') + }) + + /** + * The detail page has always subtitled with the service's own description. + * Reusing the list subtitle there would restate the title ("Jira" over "Jira + * integration") and throw away the richer copy, so only family service + * accounts — which genuinely need their reach spelled out — diverge. + */ + it('keeps the service description as the detail subtitle for non-family credentials', () => { + const oauth = resolveCredentialDisplay({ + type: 'oauth', + displayName: 'someone@example.com', + providerId: 'jira', + }) + expect(oauth.detailSubtitle).toBe('Access Jira projects, issues, and Service Management.') + + const singleProductServiceAccount = resolveCredentialDisplay( + serviceAccount('notion-service-account') + ) + expect(singleProductServiceAccount.detailSubtitle).toBe( + singleProductServiceAccount.service?.description + ) + }) + + it('spells out reach on the detail page only for family service accounts', () => { + const display = resolveCredentialDisplay(serviceAccount('atlassian-service-account')) + expect(display.detailSubtitle).toBe(display.subtitle) + expect(display.detailSubtitle).toContain('Jira Service Management') + }) + + it('degrades safely for a credential with no provider', () => { + const display = resolveCredentialDisplay({ + type: 'service_account', + displayName: 'Orphan', + providerId: null, + }) + + expect(display.service).toBeNull() + expect(display.icon).toBeNull() + expect(display.blockType).toBe('') + expect(display.detailTitle).toBe('Orphan') + }) +}) + +describe('getServiceAccountCoverageSentence', () => { + it('tells the user up front that one Atlassian token spans all three products', () => { + expect(getServiceAccountCoverageSentence('atlassian-service-account')).toBe( + 'One token works across Confluence, Jira, and Jira Service Management.' + ) + }) + + it('returns null for providers that map to a single integration', () => { + expect(getServiceAccountCoverageSentence('notion-service-account')).toBeNull() + }) +}) diff --git a/apps/sim/lib/integrations/credential-display.ts b/apps/sim/lib/integrations/credential-display.ts new file mode 100644 index 00000000000..9b9b20d927c --- /dev/null +++ b/apps/sim/lib/integrations/credential-display.ts @@ -0,0 +1,310 @@ +/** + * Single source of truth for how a stored credential is presented: which + * catalog integrations it powers, which mark and brand tile represent it, and + * the sentence describing it. + * + * Before this module, three surfaces (the integrations list, the Cmd-K search + * items, and the credential detail page) each re-derived this by looking the + * catalog up by the OAuth service's *display name*. That silently failed for + * family service accounts: `atlassian-service-account` resolves to a + * pseudo-service named "Atlassian Service Account", which matches no catalog + * integration, so the credential lost its brand tile and its category. + */ + +import type { ComponentType } from 'react' +import { getServiceAccountConnectNoun } from '@/lib/credentials/service-account-provider-ids' +import integrationsJson from '@/lib/integrations/integrations.json' +import { CANONICAL_SERVICE_ACCOUNT_SLUGS } from '@/lib/integrations/oauth-service' +import type { Integration } from '@/lib/integrations/types' +import { OAUTH_PROVIDERS } from '@/lib/oauth/oauth' +import type { OAuthProvider, OAuthServiceConfig } from '@/lib/oauth/types' +import { + credentialProviderMatchesService, + getServiceConfigByProviderId, + getServiceConfigByServiceId, + parseProvider, +} from '@/lib/oauth/utils' + +const INTEGRATIONS_DATA: readonly Integration[] = + integrationsJson.integrations as readonly Integration[] + +/** + * Above this many covered integrations the subtitle states a count instead of + * enumerating names — Google issues one service account for 13 integrations, + * which does not fit on a list row. + */ +const MAX_ENUMERATED_INTEGRATIONS = 3 + +/** + * Catalog indexes built once at module load. `resolveCredentialDisplay` runs + * inline per credential per render on the integrations surfaces, so its lookups + * must be O(1) rather than scanning the full catalog each time. + */ +const INTEGRATION_BY_SLUG: ReadonlyMap = new Map( + INTEGRATIONS_DATA.map((i) => [i.slug, i]) +) + +/** Keyed by lowercased display name, matching how OAuth services are named. */ +const INTEGRATION_BY_LOWER_NAME: ReadonlyMap = new Map( + INTEGRATIONS_DATA.map((i) => [i.name.toLowerCase(), i]) +) + +/** Every provider id that some service designates as its service-account id. */ +const SERVICE_ACCOUNT_PROVIDER_IDS: ReadonlySet = new Set( + Object.values(OAUTH_PROVIDERS).flatMap((provider) => + Object.values(provider.services).flatMap((service) => + service.serviceAccountProviderId ? [service.serviceAccountProviderId] : [] + ) + ) +) + +/** + * `credentialProviderId` → catalog integrations it authenticates, in catalog + * order. Built once at module load: resolving a service per integration walks + * `OAUTH_PROVIDERS`, which is wasted work to repeat per lookup (same reasoning + * as `SERVICE_ACCOUNT_INTEGRATIONS` in `oauth-service.ts`). + * + * Indexing under both ids a service answers to is the predicate + * {@link credentialProviderMatchesService} expressed as a map, so the two can + * never disagree. + */ +const INTEGRATIONS_BY_CREDENTIAL_PROVIDER: ReadonlyMap = (() => { + const index = new Map() + const add = (providerId: string | undefined, integration: Integration) => { + if (!providerId) return + const existing = index.get(providerId) + if (existing) existing.push(integration) + else index.set(providerId, [integration]) + } + + for (const integration of INTEGRATIONS_DATA) { + if (integration.authType !== 'oauth' || !integration.oauthServiceId) continue + const service = getServiceConfigByServiceId(integration.oauthServiceId) + if (!service) continue + add(service.providerId, integration) + add(service.serviceAccountProviderId, integration) + } + + return index +})() + +/** Catalog integrations a credential of this provider id can authenticate. */ +export function getIntegrationsForCredentialProvider(providerId: string): readonly Integration[] { + return INTEGRATIONS_BY_CREDENTIAL_PROVIDER.get(providerId) ?? [] +} + +/** + * Whether this provider id is a service-account id shared across more than one + * catalog integration — one Atlassian API token covers Jira, Jira Service + * Management, and Confluence; one Google JSON key covers every Google + * integration. Derived from the catalog, so a new integration joining a family + * is picked up with no edit here. + */ +export function isFamilyServiceAccount(providerId: string): boolean { + return ( + SERVICE_ACCOUNT_PROVIDER_IDS.has(providerId) && + getIntegrationsForCredentialProvider(providerId).length > 1 + ) +} + +/** + * Base provider entry owning a family service-account id. + * + * `parseProvider` is the right lookup because it resolves an id to the service + * that registers it as its own `providerId` before falling back — which is how + * `atlassian-service-account` reaches the `atlassian` provider rather than + * Confluence, the first service that merely *references* it. + */ +function getFamilyProvider(providerId: string) { + if (!isFamilyServiceAccount(providerId)) return null + const { baseProvider } = parseProvider(providerId as OAuthProvider) + return OAUTH_PROVIDERS[baseProvider] ?? null +} + +/** Vendor name for a family service account ("Atlassian", "Google"), else null. */ +export function getServiceAccountFamilyName(providerId: string): string | null { + return getFamilyProvider(providerId)?.name ?? null +} + +/** Corporate mark for a family service account, else null. */ +export function getServiceAccountFamilyIcon( + providerId: string +): ComponentType<{ className?: string }> | null { + const icon = getFamilyProvider(providerId)?.icon + return (icon as ComponentType<{ className?: string }> | undefined) ?? null +} + +/** + * Sentence naming what a family service account reaches, for connect-time copy + * and credential subtitles. Returns null for non-family providers. + */ +export function getServiceAccountCoverageSentence(providerId: string): string | null { + const familyName = getServiceAccountFamilyName(providerId) + if (!familyName) return null + const covered = getIntegrationsForCredentialProvider(providerId) + if (covered.length > MAX_ENUMERATED_INTEGRATIONS) { + return `One token works across all ${covered.length} ${familyName} integrations.` + } + return `One token works across ${formatList(covered.map((i) => i.name))}.` +} + +/** Oxford-comma list: "Jira", "Jira and Confluence", "A, B, and C". */ +function formatList(names: readonly string[]): string { + if (names.length <= 1) return names[0] ?? '' + if (names.length === 2) return `${names[0]} and ${names[1]}` + return `${names.slice(0, -1).join(', ')}, and ${names[names.length - 1]}` +} + +/** Minimal credential shape this module needs — structurally satisfied by `WorkspaceCredential`. */ +interface DisplayableCredential { + type: string + displayName: string + providerId: string | null +} + +export interface CredentialDisplay { + /** Resolved OAuth service config, or null when the provider is unknown. */ + service: OAuthServiceConfig | null + /** + * Catalog integration lending the brand tile and category. For a family + * service account this is the family's canonical integration, since no single + * product owns the credential. + */ + integration: Integration | null + /** `integration.type`, or '' — drives the brand tile background. */ + blockType: string + /** Mark to render: the family's corporate icon, else the service's own. */ + icon: ComponentType<{ className?: string }> | null + /** Vendor name when this is a family service account, else null. */ + familyName: string | null + /** Catalog integrations this credential authenticates, in catalog order. */ + coveredIntegrations: readonly Integration[] + /** + * Generated sentence describing the credential's service and reach. Never the + * user's own description — list surfaces prefer `credential.description` and + * fall back to this. + */ + subtitle: string + /** Header title for the credential detail page. */ + detailTitle: string + /** + * Header subtitle for the credential detail page. Only a family service + * account needs its reach spelled out there; every other credential keeps the + * service's own richer description, which the page has always shown and which + * beats restating the title ("Jira" over "Jira integration"). + */ + detailSubtitle: string +} + +/** + * Resolves everything a surface needs to render a credential. Pure and + * server-safe apart from the icon components it passes through, so it may be + * imported by server components as well as client ones. + */ +export function resolveCredentialDisplay(credential: DisplayableCredential): CredentialDisplay { + const providerId = credential.providerId + const isServiceAccount = credential.type === 'service_account' + + if (!providerId) { + return { + service: null, + integration: null, + blockType: '', + icon: null, + familyName: null, + coveredIntegrations: [], + subtitle: 'Connected service', + detailTitle: credential.displayName, + detailSubtitle: 'Connected service', + } + } + + const service = getServiceConfigByProviderId(providerId) + const familyName = getServiceAccountFamilyName(providerId) + const coveredIntegrations = getIntegrationsForCredentialProvider(providerId) + const integration = resolveCatalogIntegration(providerId, service, Boolean(familyName)) + + const subtitle = buildSubtitle({ + providerId, + service, + familyName, + coveredIntegrations, + isServiceAccount, + }) + + return { + service, + integration, + blockType: integration?.type ?? '', + icon: + getServiceAccountFamilyIcon(providerId) ?? + (service?.icon as ComponentType<{ className?: string }> | undefined) ?? + null, + familyName, + coveredIntegrations, + subtitle, + detailTitle: isServiceAccount + ? credential.displayName + : (service?.name ?? credential.displayName), + detailSubtitle: familyName ? subtitle : (service?.description ?? subtitle), + } +} + +/** + * Catalog entry lending brand tile and category. + * + * A family service account goes straight to its canonical slug: resolving by + * service name would land on whichever product the resolver happened to pick + * (`google-service-account` → Gmail), which is arbitrary. Everything else keeps + * the long-standing name lookup. + */ +function resolveCatalogIntegration( + providerId: string, + service: OAuthServiceConfig | null, + isFamily: boolean +): Integration | null { + if (isFamily) { + const slug = CANONICAL_SERVICE_ACCOUNT_SLUGS[providerId] + const canonical = slug ? INTEGRATION_BY_SLUG.get(slug) : undefined + if (canonical) return canonical + } + if (!service) return null + const byName = INTEGRATION_BY_LOWER_NAME.get(service.name.toLowerCase()) + if (byName) return byName + // Last resort: any integration this credential authenticates, so a credential + // never loses its category filter membership. + return getIntegrationsForCredentialProvider(providerId)[0] ?? null +} + +interface SubtitleArgs { + providerId: string + service: OAuthServiceConfig | null + familyName: string | null + coveredIntegrations: readonly Integration[] + isServiceAccount: boolean +} + +/** + * Vendor-accurate nouns come from {@link getServiceAccountConnectNoun}, the + * same source the connect controls use, so a Slack credential reads "custom + * bot" here exactly as its setup button does. + */ +function buildSubtitle({ + providerId, + service, + familyName, + coveredIntegrations, + isServiceAccount, +}: SubtitleArgs): string { + if (familyName) { + const scope = + coveredIntegrations.length > MAX_ENUMERATED_INTEGRATIONS + ? `all ${coveredIntegrations.length} ${familyName} integrations` + : formatList(coveredIntegrations.map((i) => i.name)) + return `${familyName} ${getServiceAccountConnectNoun(providerId)} · ${scope}` + } + if (!service) return 'Connected service' + return isServiceAccount + ? `${service.name} ${getServiceAccountConnectNoun(providerId)}` + : `${service.name} integration` +} diff --git a/apps/sim/lib/integrations/index.ts b/apps/sim/lib/integrations/index.ts index 023135c6286..bfd42fb29c6 100644 --- a/apps/sim/lib/integrations/index.ts +++ b/apps/sim/lib/integrations/index.ts @@ -98,6 +98,15 @@ export function toIntegrationSummary(integration: Integration): IntegrationSumma } } +export { + type CredentialDisplay, + getIntegrationsForCredentialProvider, + getServiceAccountCoverageSentence, + getServiceAccountFamilyIcon, + getServiceAccountFamilyName, + isFamilyServiceAccount, + resolveCredentialDisplay, +} from '@/lib/integrations/credential-display' export { blockTypeToIconMap } from '@/lib/integrations/icon-mapping' export { type OAuthServiceMatch, diff --git a/apps/sim/lib/integrations/oauth-service.ts b/apps/sim/lib/integrations/oauth-service.ts index 6ef45d07179..d952c61f678 100644 --- a/apps/sim/lib/integrations/oauth-service.ts +++ b/apps/sim/lib/integrations/oauth-service.ts @@ -84,7 +84,7 @@ export interface ServiceAccountIntegrationMatch { * both arbitrary and a poor landing page. A caller that names a specific * integration still gets that integration. */ -const CANONICAL_SERVICE_ACCOUNT_SLUGS: Readonly> = { +export const CANONICAL_SERVICE_ACCOUNT_SLUGS: Readonly> = { 'google-service-account': 'google-drive', google: 'google-drive', 'atlassian-service-account': 'jira', diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index fc3c6bc7064..5fc37bbe756 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -4,6 +4,7 @@ import { truncate } from '@sim/utils/string' import { AirtableIcon, AsanaIcon, + AtlassianIcon, AttioIcon, AzureIcon, BoxCompanyIcon, @@ -509,15 +510,15 @@ export const OAUTH_PROVIDERS: Record = { }, atlassian: { name: 'Atlassian', - icon: JiraIcon, + icon: AtlassianIcon, services: { 'atlassian-service-account': { name: 'Atlassian Service Account', description: 'Authenticate as an Atlassian service account using a scoped API token from admin.atlassian.com.', providerId: 'atlassian-service-account', - icon: JiraIcon, - baseProviderIcon: JiraIcon, + icon: AtlassianIcon, + baseProviderIcon: AtlassianIcon, scopes: [], authType: 'service_account', }, diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index ce4e754d727..141796ba4fa 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -543,6 +543,42 @@ export function getServiceAccountProviderForProviderId(providerId: string): stri return serviceConfig?.serviceAccountProviderId } +/** + * The two provider ids a service answers to. Structurally satisfied by both + * `OAuthServiceConfig` and the lighter `OAuthServiceMatch` that catalog + * resolution returns, so callers pass whichever they already hold. + */ +export interface ServiceProviderIdentity { + providerId: string + serviceAccountProviderId?: string +} + +/** + * Whether a stored credential's `providerId` authenticates the given service. + * + * A service is reachable by two ids: its own OAuth `providerId` (`jira`) and + * the service-account provider its family issues (`atlassian-service-account`). + * One Atlassian API token authenticates Jira, Jira Service Management, and + * Confluence alike, so matching on the OAuth `providerId` alone hides a + * service-account credential from every product page it actually powers. + * + * Prefer this over comparing `getServiceConfigByProviderId(id)?.providerId` + * against a service: that resolver walks `OAUTH_PROVIDERS` in declaration + * order and answers "which service owns this id", which for a family-wide + * service-account id is an arbitrary single winner — `atlassian-service-account` + * resolves to the `Atlassian Service Account` pseudo-service and + * `google-service-account` to whichever Google service is declared first. + */ +export function credentialProviderMatchesService( + credentialProviderId: string, + service: ServiceProviderIdentity +): boolean { + return ( + service.providerId === credentialProviderId || + service.serviceAccountProviderId === credentialProviderId + ) +} + export function getCanonicalScopesForProvider(providerId: string): string[] { const service = getServiceConfigByProviderId(providerId) return service?.scopes ? [...service.scopes] : [] From c9547dcfe5514c498b62366c399bcbf19d4a6192 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 30 Jul 2026 14:25:07 -0700 Subject: [PATCH 06/21] fix(desktop): keep page headers clear of the traffic lights (#6098) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(desktop): keep page headers clear of the traffic lights In the desktop app, collapsing the sidebar hands the top-left of the screen to the content pane — the macOS traffic lights and the sidebar expander then sit over page content rather than over the sidebar. Every top-of-page header bar drew straight underneath them, so back buttons and the integrations/skills switcher were unreadable and unclickable. The reason it hit all of them at once is that the bar's geometry was copied verbatim into seven files, so nothing could reserve that lane in one place. Extract it as PAGE_HEADER_BAR and fold the lane into its top padding via `--workspace-content-title-bar-inset`, the variable the content pane already publishes for exactly this. That variable is 0px everywhere except the macOS desktop app with the sidebar collapsed, so this is inert on the web and on an expanded sidebar, where the lights sit over sidebar chrome that already reserves its own lane. Covers the settings shell, the credential/skill detail layout, the integrations and skills switcher, integration block detail and its fallback, and both upgrade headers. A guard test fails with the file named if any page re-derives the bar instead of composing it. * fix(desktop): cover the Resource header bar too The first pass found its surfaces by grepping one exact class string, which turned up the minority header. `ResourceHeader` is the same bar written as `px-4 py-[8.5px]`, and it is the one logs, files, tables, knowledge and scheduled tasks use — plus every loading fallback. Those all still drew under the traffic lights, and on the breadcrumb surfaces the occluded element is an interactive popover button, not just a title. Share the lane math as TITLE_BAR_LANE_PT and compose it from both bar geometries, which genuinely differ (the Resource bar is bordered and has a min height). Nothing nests one bar inside the other, so no reset is needed: the only out-of-pane render is the landing tables preview. Move the variable's `0px` default to `:root` alongside the other desktop-title-bar vars, so it is defined for bars outside `.workspace-content-shell` — the standalone settings shell and that landing preview — where an undefined var() inside calc() would be invalid at computed-value time and drop padding-top entirely. That replaces the per-call fallback, leaving one default instead of two that can drift. Fold the assertions into the existing desktop title-bar surface audit rather than a second audit file with its own conventions. The standalone guard is gone: it keyed on one spelling of the geometry, so it was blind to `px-4 py-[8.5px]` — the very re-derivation that made this pass necessary. * fix(desktop): cover fullscreen routes, spare embedded panels Review round 2 found the two arrangements the collapsed-sidebar selector alone gets wrong. A fullscreen route (/upgrade) slides the sidebar to zero width without collapsing it, so `data-sidebar-collapsed` is absent and the lane stayed zero while the pane was in fact sitting under the traffic lights. The pane owns the lane whenever the sidebar is not there to own it, so the selector now matches a new `data-content-fullscreen` as well. The mirror error: the variable is inherited, and the mothership panel is the right half of the pane — never under the lights — yet it embeds whole pages (KnowledgeBase and friends) whose header bars reserve the lane. Those bars were gaining ~38px inside the panel. The panel now zeroes the variable for its subtree. I had checked for exactly this nesting and concluded it did not exist, having looked only at the settings pages that import `Resource` types without rendering `Resource.Header`. `resource-content.tsx` renders the knowledge page itself, which does. An assertion each, both verified to fail when the fix is reverted. * fix(desktop): size the peek card to its content, soften the overlay shadow The floating sidebar pinned both its top and bottom edges, so it always drew at full window height. On a short surface — the settings list — that left a tall empty slab of card hanging below the last entry. The card now hugs its content and caps at the pane height less the traffic-light lane and the bottom gutter. Dropping the bottom pin is most of it: the four `h-full` rules down the chain resolve against an auto-height parent and collapse to content on their own. But nothing would then bound the sidebar's own `flex-1 overflow-y-auto` region, so a long workflow list would be clipped by the card's `overflow-hidden` instead of scrolling. The card is therefore a capped flex column, and the shell is allowed to shrink inside it, which restores a definite height for the chain to resolve against. That rule is scoped to `[data-peek]` and is inert while docked, where the shell is not a flex item. Also eases `--shadow-overlay` in both themes (alpha ~27% lighter, bloom pulled in from 48px to 30px), keeping it clearly above `--shadow-medium` so the scale still reads in order. The peek card drops the shadow entirely and separates on the same `--border` hairline the content pane beside it uses. * fix(desktop): stop the login page scrolling, drop the pane border at the window edge Two reports against the desktop window chrome. The login page scrolled by exactly the traffic-light lane. `.desktop-title-bar-page` reserved the lane with `margin-top` plus a `calc(100vh - lane)` height, which sums to the viewport on its own — but `body` carries `min-height: 100vh`, and body is a plain block box with no padding, border, or BFC, so that top margin had nothing to collapse against and collapsed through, displacing body itself. The document came out one full lane taller than the viewport while the shell's `calc` saving was re-inflated underneath it. Reserving the lane with padding *inside* the box removes both the collapse and the `calc`: global `box-sizing: border-box` keeps the padding within the `100vh`. Measured in the Electron renderer over CDP — 40px of overflow before, 0 after, with the logo still clear of the lane. Collapsing the sidebar also left a hairline outline traced just inside the window. The shell drops to `p-0` there, but the content pane kept its border and 8px radius, so both drew flush against the square window frame. The pane now drops them exactly when it is flush. Keyed off the ancestor attributes rather than React state, because the title-bar attribute is written pre-paint and a state-driven rule would flash the border before hydration settles. * docs(desktop): correct the peek card chrome comment The card's TSDoc still listed `shadow-overlay` as part of its chrome after the shadow was deliberately dropped, so the comment contradicted the code. It now records that the card is unshadowed on purpose, and documents the content-hugging height and the flex-column cap that make the sidebar's scroll region bound itself. --------- Co-authored-by: Waleed Latif --- .../_shell/desktop-title-bar-surfaces.test.ts | 73 +++++++++++++++++++ apps/sim/app/_styles/globals.css | 32 +++++--- .../components/credential-detail-layout.tsx | 4 +- .../resource-header/resource-header.tsx | 6 +- .../workspace-chrome/workspace-chrome.tsx | 41 +++++++++-- .../mothership-view/mothership-view.tsx | 4 + .../integration-block-detail-fallback.tsx | 3 +- .../[block]/integration-block-detail.tsx | 3 +- .../integration-tabs-header.tsx | 3 +- .../[workspaceId]/upgrade/upgrade.tsx | 5 +- apps/sim/components/page-header-bar.ts | 25 +++++++ .../components/settings/settings-header.tsx | 5 +- 12 files changed, 181 insertions(+), 23 deletions(-) create mode 100644 apps/sim/components/page-header-bar.ts diff --git a/apps/sim/app/_shell/desktop-title-bar-surfaces.test.ts b/apps/sim/app/_shell/desktop-title-bar-surfaces.test.ts index 50287ceaa35..4b0c822220d 100644 --- a/apps/sim/app/_shell/desktop-title-bar-surfaces.test.ts +++ b/apps/sim/app/_shell/desktop-title-bar-surfaces.test.ts @@ -14,6 +14,13 @@ const workspaceChrome = read( ) const sidebar = read('../workspace/[workspaceId]/w/components/sidebar/sidebar.tsx') const globalStyles = read('../_styles/globals.css') +const pageHeaderBar = read('../../components/page-header-bar.ts') +const resourceHeader = read( + '../workspace/[workspaceId]/components/resource/components/resource-header/resource-header.tsx' +) +const mothershipView = read( + '../workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx' +) describe('desktop title-bar surface audit', () => { it('applies the safe-area shell only when the auth route is login', () => { @@ -65,4 +72,70 @@ describe('desktop title-bar surface audit', () => { expect(sidebar).toContain('[[data-sim-desktop-title-bar=inset]_&]:pt-[var(') expect(sidebar).not.toMatch(/\[\[data-sim-desktop-title-bar=inset\]_&\]:pt-\d/) }) + + it('defines the content-pane lane once, defaulting to zero', () => { + // A `:root` default keeps the variable defined for bars that render outside + // `.workspace-content-shell` (the standalone settings shell at /account, + // /organization/[id], /selfhost; the landing tables preview). An undefined var() + // inside calc() is invalid at computed-value time and drops padding-top entirely. + expect(globalStyles).toMatch(/:root\s*\{[^}]*--workspace-content-title-bar-inset:\s*0px/s) + // The pane owns the lane in both arrangements where the sidebar is not there to + // own it: collapsed to zero width, and slid away for a fullscreen route (which + // leaves the sidebar expanded in the store, so the collapsed selector alone misses + // it and /upgrade's back chip lands under the traffic lights). + expect(globalStyles).toContain('.workspace-content-shell[data-sidebar-collapsed],') + expect(globalStyles).toContain('.workspace-content-shell[data-content-fullscreen] {') + expect(workspaceChrome).toContain('data-content-fullscreen={isFullscreen || undefined}') + }) + + it('sizes the peek card to its content, capped against the lane', () => { + // Pinning both edges made the card full height, so a short list (settings) left a + // tall empty slab over the content. It now hugs its content and caps at the pane + // height less the lane and the bottom gutter, so a long list still scrolls. + expect(workspaceChrome).toContain('max-h-[calc(100%-var(--desktop-title-bar-height)-8px)]') + expect(workspaceChrome).not.toMatch(/PEEK_CARD_CHROME[\s\S]{0,240}?bottom-2/) + }) + + it('reserves the login lane inside the box, never as a collapsing margin', () => { + // `body` carries `min-height: 100vh`, and a `margin-top` here collapses through it + // (body is a plain block box, so it opens no BFC) and displaces body itself. The + // document then measured one full lane taller than the viewport, which is what made + // the desktop login page scroll. Verified live: 40px of overflow, now 0. + // Comments are stripped first: the rule documents why `margin-top` is wrong, and a + // raw `not.toContain` would match that prose instead of a declaration. + const rule = (globalStyles.match(/\.desktop-title-bar-page \{[^}]*\}/)?.[0] ?? '').replace( + /\/\*[\s\S]*?\*\//g, + '' + ) + expect(rule).toContain('padding-top: var(--desktop-title-bar-height)') + expect(rule).toContain('min-height: 100vh') + expect(rule).not.toContain('margin-top') + }) + + it('drops the content pane border where the pane meets the window edge', () => { + // Collapsing the sidebar in the desktop shell takes the pane's padding to 0, so a + // retained border and radius drew a hairline outline inset from the square window. + const flush = '[[data-sim-desktop-title-bar=inset]_[data-sidebar-collapsed]_&]:' + expect(workspaceChrome).toContain(`${flush}rounded-none`) + expect(workspaceChrome).toContain(`${flush}border-0`) + }) + + it('clears the lane for panels that embed pages away from the lights', () => { + // The mothership panel is the right half of the pane and embeds whole pages + // (KnowledgeBase et al) whose header bars reserve the lane. It inherits the + // variable, so without this reset those bars gain the inset while sitting nowhere + // near the traffic lights. + expect(mothershipView).toContain('[--workspace-content-title-bar-inset:0px]') + }) + + it('reserves that lane in every top-of-pane header bar', () => { + // Both top-bar geometries must compose the shared lane padding. A bare + // `pt-`/`py-[8.5px]` in either is the bug: the bar then draws under the traffic + // lights and the sidebar expander whenever the sidebar is collapsed on desktop. + expect(pageHeaderBar).toContain('pt-[calc(8.5px+var(--workspace-content-title-bar-inset))]') + expect(pageHeaderBar).toContain('TITLE_BAR_LANE_PT') + + expect(resourceHeader).toContain('TITLE_BAR_LANE_PT') + expect(resourceHeader).not.toMatch(/py-\[8\.5px\]/) + }) }) diff --git a/apps/sim/app/_styles/globals.css b/apps/sim/app/_styles/globals.css index 87c1a884809..1ba253d00b5 100644 --- a/apps/sim/app/_styles/globals.css +++ b/apps/sim/app/_styles/globals.css @@ -14,6 +14,7 @@ --sidebar-collapsed-width: 51px; /* icon rail on web; desktop overrides to 0 before first paint */ --sidebar-expanded-width: 248px; /* SIDEBAR_WIDTH.DEFAULT; the width to restore to, held even while collapsed */ --desktop-title-bar-height: 0px; /* macOS traffic-light lane; desktop overrides before first paint */ + --workspace-content-title-bar-inset: 0px; /* lane the content pane must leave clear; only non-zero when the pane, not the sidebar, sits under it */ --desktop-title-bar-inset-x: 0px; /* clearance past the traffic lights; desktop overrides */ --desktop-title-bar-control-offset: 0px; /* centres a lane control; desktop overrides */ --desktop-title-bar-control-size: 0px; /* control seated in the lane; desktop overrides */ @@ -43,7 +44,7 @@ /* Shadow scale */ --shadow-subtle: 0 2px 4px 0 rgba(0, 0, 0, 0.08); --shadow-medium: 0 4px 12px rgba(0, 0, 0, 0.1); - --shadow-overlay: 0 16px 48px rgba(0, 0, 0, 0.15); + --shadow-overlay: 0 10px 30px rgba(0, 0, 0, 0.11); --shadow-kbd: 0 4px 0 0 rgba(48, 48, 48, 1); --shadow-kbd-sm: 0 2px 0 0 rgba(48, 48, 48, 1); --shadow-card: 0 1px 3px rgba(0, 0, 0, 0.04); @@ -75,8 +76,14 @@ html[data-sim-desktop-title-bar="inset"] { /** The macOS desktop login shell reserves the native traffic-light lane. */ .desktop-title-bar-page { - margin-top: var(--desktop-title-bar-height); - min-height: calc(100vh - var(--desktop-title-bar-height)); + /* Reserve the lane with padding INSIDE the box, never `margin-top` + a + `calc(100vh - lane)` height. `body` carries `min-height: 100vh` of its own, and + a top margin here has nothing to collapse against (body is a block box with no + padding, border, or BFC), so it collapses through and displaces body itself — + leaving a document one full lane taller than the viewport. Global + `box-sizing: border-box` keeps this padding inside the 100vh. */ + padding-top: var(--desktop-title-bar-height); + min-height: 100vh; } .desktop-window-drag-region { @@ -122,11 +129,10 @@ html[data-sim-desktop-title-bar="inset"] letter-spacing: 0.02em; } -.workspace-content-shell { - --workspace-content-title-bar-inset: 0px; -} - -.workspace-content-shell[data-sidebar-collapsed] { +/* The pane owns the lane whenever the sidebar is not there to own it: collapsed to + zero width, or slid away for a fullscreen route. */ +.workspace-content-shell[data-sidebar-collapsed], +.workspace-content-shell[data-content-fullscreen] { --workspace-content-title-bar-inset: var(--desktop-title-bar-height); } @@ -173,6 +179,14 @@ html[data-sim-desktop-title-bar="inset"] transition: none; } +/* The card is a flex column sized to its content, so the shell must be allowed to + shrink for the sidebar's own scroll region to bound itself once the card hits its + max height. Docked, this element is not a flex item and the rule is inert. */ +.sidebar-shell-outer[data-peek] .sidebar-shell-inner { + min-height: 0; + flex-shrink: 1; +} + .sidebar-container span, .sidebar-container .text-small { transition: opacity 120ms ease; @@ -583,7 +597,7 @@ html.sidebar-booting .sidebar-shell-inner { --scrollbar-thumb-hover-color: #6a6a6a; /* Shadow scale - dark */ - --shadow-overlay: 0 16px 48px rgba(0, 0, 0, 0.4); + --shadow-overlay: 0 10px 30px rgba(0, 0, 0, 0.3); } } diff --git a/apps/sim/app/workspace/[workspaceId]/components/credential-detail/components/credential-detail-layout.tsx b/apps/sim/app/workspace/[workspaceId]/components/credential-detail/components/credential-detail-layout.tsx index 5753feabdcb..d4099a6a6f7 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/credential-detail/components/credential-detail-layout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/credential-detail/components/credential-detail-layout.tsx @@ -1,4 +1,6 @@ import type { ReactNode } from 'react' +import { cn } from '@sim/emcn' +import { PAGE_HEADER_BAR } from '@/components/page-header-bar' interface CredentialDetailLayoutProps { /** Back link rendered at the start of the fixed action bar. */ @@ -17,7 +19,7 @@ interface CredentialDetailLayoutProps { export function CredentialDetailLayout({ back, actions, children }: CredentialDetailLayoutProps) { return (
-
+
{back} {actions ?
{actions}
: null}
diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/components/resource-header/resource-header.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/components/resource-header/resource-header.tsx index f6b464404cd..a28d735a1f8 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/resource/components/resource-header/resource-header.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/components/resource-header/resource-header.tsx @@ -31,6 +31,7 @@ import { } from '@sim/emcn' import { ArrowUpLeft } from 'lucide-react' import { createPortal } from 'react-dom' +import { TITLE_BAR_LANE_PT } from '@/components/page-header-bar' import { InlineRenameInput } from '@/app/workspace/[workspaceId]/components/inline-rename-input' import { FloatingOverflowText } from '@/app/workspace/[workspaceId]/components/resource/components/floating-overflow-text' @@ -131,7 +132,10 @@ export const ResourceHeader = memo(function ResourceHeader({ return (
diff --git a/apps/sim/app/workspace/[workspaceId]/components/workspace-chrome/workspace-chrome.tsx b/apps/sim/app/workspace/[workspaceId]/components/workspace-chrome/workspace-chrome.tsx index 7f206c26da2..6609facab82 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/workspace-chrome/workspace-chrome.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/workspace-chrome/workspace-chrome.tsx @@ -22,9 +22,18 @@ const SLIDE_TRANSITION = * The peek card's floating chrome. * * Every value is an existing token: `rounded-lg` is `--radius`, matching the content - * pane it floats beside; `--border` is that pane's border; `shadow-overlay` and - * `--z-modal` are what the app's other edge-anchored panels use. The card's fill is - * the sidebar's own `--surface-1`, so docked and floating are the same surface. + * pane it floats beside; `--border` is that pane's border; `--z-modal` is what the + * app's other edge-anchored panels use. The card's fill is the sidebar's own + * `--surface-1`, so docked and floating are the same surface. + * + * Deliberately unshadowed. It separates on the same `--border` hairline the content + * pane beside it uses; `--shadow-overlay` reads as too heavy at this size, where the + * card abuts the window edge rather than floating over the middle of the page. + * + * The card hugs its content and caps at the pane height less the lane and the bottom + * gutter — pinning both edges left a tall empty slab below short lists. It is a flex + * column so the shell can shrink inside that cap and the sidebar's own scroll region + * still bounds itself; see the `[data-peek]` rule in `globals.css`. * * `w-auto` shrink-wraps the inner shell, which `[data-peek]` has already put at the * expanded width. It must not be a length: `width` cannot interpolate to or from @@ -32,7 +41,7 @@ const SLIDE_TRANSITION = * card widens as it appears and leaves a shrinking ghost on retract. */ const PEEK_CARD_CHROME = - 'absolute top-[var(--desktop-title-bar-height)] bottom-2 left-2 z-[var(--z-modal)] w-auto origin-top-left rounded-lg border border-[var(--border)] shadow-overlay' + 'absolute top-[var(--desktop-title-bar-height)] left-2 z-[var(--z-modal)] flex max-h-[calc(100%-var(--desktop-title-bar-height)-8px)] w-auto flex-col origin-top-left rounded-lg border border-[var(--border)]' /** * Peek card enter/exit — the popper idiom rather than a slide, since the card is @@ -57,6 +66,20 @@ const PEEK_CARD_EXIT = cn( /** The docked rail: in flow, width-animated by the collapse toggle. */ const SIDEBAR_SHELL_IN_FLOW = cn('transition-[width]', SLIDE_TRANSITION) +/** + * The content pane's own chrome, dropped when the pane sits flush to the window. + * + * Collapsing the sidebar in the desktop shell takes the surrounding padding to `0`, + * which puts the pane hard against the window edge — and its border and radius then + * draw a hairline outline with rounded corners inset from the square window frame. + * + * Keyed off the ancestor attributes rather than React state on purpose: the title-bar + * attribute is written pre-paint, so a state-driven rule would flash the border on + * first paint before hydration settles. + */ +const CONTENT_PANE_FLUSH = + '[[data-sim-desktop-title-bar=inset]_[data-sidebar-collapsed]_&]:rounded-none [[data-sim-desktop-title-bar=inset]_[data-sidebar-collapsed]_&]:border-0' + interface WorkspaceChromeProps { children: React.ReactNode /** Cookie-derived collapse state from the server layout; seeds the sidebar's first render. */ @@ -295,8 +318,16 @@ export function WorkspaceChrome({ isCollapsed && '[[data-sim-desktop-title-bar=inset]_&]:p-0' )} data-sidebar-collapsed={isCollapsed || undefined} + /* A fullscreen route slides the sidebar away without collapsing it, so the pane + inherits the traffic-light lane the same way a collapsed sidebar does. */ + data-content-fullscreen={isFullscreen || undefined} > -
+
{children}
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx index 01a1ddd3d69..48957e8fcd4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx @@ -137,6 +137,10 @@ export const MothershipView = memo( className={cn( 'relative z-10 flex h-full flex-col overflow-hidden border-[var(--border)] bg-[var(--bg)] transition-[width,min-width,border-width] duration-200 ease-[cubic-bezier(0.25,0.1,0.25,1)]', isCollapsed ? 'w-0 min-w-0 border-l-0' : 'w-1/2 border-l', + /* This panel is the right half of the pane, never under the traffic lights, + yet it embeds whole pages whose header bars reserve that lane. Zeroing the + inherited variable here keeps their top bars flush inside the panel. */ + '[--workspace-content-title-bar-inset:0px]', className )} > diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail-fallback.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail-fallback.tsx index e634e7806e7..df7af0e6c8a 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail-fallback.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail-fallback.tsx @@ -2,6 +2,7 @@ import { ChipLink } from '@sim/emcn' import { ArrowLeft } from 'lucide-react' +import { PAGE_HEADER_BAR } from '@/components/page-header-bar' interface IntegrationBlockDetailFallbackProps { workspaceId: string @@ -23,7 +24,7 @@ export function IntegrationBlockDetailFallback({ }: IntegrationBlockDetailFallbackProps) { return (
-
+
Integrations diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx index 0bbf99b3630..6ad76f5996e 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx @@ -6,6 +6,7 @@ import { ArrowLeft, ArrowRight, Plus } from 'lucide-react' import Link from 'next/link' import { useRouter } from 'next/navigation' import { useQueryState } from 'nuqs' +import { PAGE_HEADER_BAR } from '@/components/page-header-bar' import { blockTypeToIconMap, type Integration, @@ -138,7 +139,7 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration return (
-
+
Integrations diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/components/integration-tabs-header/integration-tabs-header.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/components/integration-tabs-header/integration-tabs-header.tsx index 66fc7fb2f22..b4d71e20bab 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/components/integration-tabs-header/integration-tabs-header.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/components/integration-tabs-header/integration-tabs-header.tsx @@ -1,5 +1,6 @@ import type { ReactNode } from 'react' import { ChipLink } from '@sim/emcn' +import { PAGE_HEADER_BAR } from '@/components/page-header-bar' interface IntegrationTabsHeaderProps { active: 'integrations' | 'skills' @@ -18,7 +19,7 @@ export function IntegrationTabsHeader({ rightSlot, }: IntegrationTabsHeaderProps) { return ( -
+
Integrations diff --git a/apps/sim/app/workspace/[workspaceId]/upgrade/upgrade.tsx b/apps/sim/app/workspace/[workspaceId]/upgrade/upgrade.tsx index 446469a2dc6..20e698d4a2c 100644 --- a/apps/sim/app/workspace/[workspaceId]/upgrade/upgrade.tsx +++ b/apps/sim/app/workspace/[workspaceId]/upgrade/upgrade.tsx @@ -5,6 +5,7 @@ import { ArrowLeft, Chip, toast } from '@sim/emcn' import { getErrorMessage } from '@sim/utils/errors' import { useRouter } from 'next/navigation' import { useQueryState } from 'nuqs' +import { PAGE_HEADER_BAR } from '@/components/page-header-bar' import { useSession } from '@/lib/auth/auth-client' import { getUpgradeCardCta, @@ -99,7 +100,7 @@ export function Upgrade({ workspaceId }: UpgradeProps) { return (
-
+
Back @@ -192,7 +193,7 @@ export function Upgrade({ workspaceId }: UpgradeProps) { return (
-
+
Back diff --git a/apps/sim/components/page-header-bar.ts b/apps/sim/components/page-header-bar.ts new file mode 100644 index 00000000000..1c1fb8e5ce9 --- /dev/null +++ b/apps/sim/components/page-header-bar.ts @@ -0,0 +1,25 @@ +/** + * Top padding for a bar sitting at the very top of the workspace content pane. + * + * Folds in `--workspace-content-title-bar-inset`, the height that pane must leave + * clear for the desktop shell's inset title bar. That variable is `0px` everywhere + * except the macOS desktop app with the sidebar collapsed — the one arrangement where + * the pane, rather than the sidebar, sits beneath the traffic lights and the sidebar + * expander. Without it a top bar draws underneath both, which hides its controls and + * can leave them unclickable. + * + * Compose this rather than writing `pt-[8.5px]`: the app has two top-bar geometries + * ({@link PAGE_HEADER_BAR} and the `Resource` header's bordered variant), and the lane + * math has to stay identical across them. + */ +export const TITLE_BAR_LANE_PT = 'pt-[calc(8.5px+var(--workspace-content-title-bar-inset))]' + +/** + * The top-of-page header bar worn by the surfaces that put a back chip, tab switcher, + * or page actions above their content. `Resource`-based pages (tables, files, logs, + * knowledge, scheduled tasks) use their own bordered bar and compose + * {@link TITLE_BAR_LANE_PT} directly. + * + * Single source of truth for this geometry — never re-derive it per page. + */ +export const PAGE_HEADER_BAR = `flex flex-shrink-0 items-center bg-[var(--bg)] px-4 ${TITLE_BAR_LANE_PT} pb-[8.5px]` diff --git a/apps/sim/components/settings/settings-header.tsx b/apps/sim/components/settings/settings-header.tsx index 8e5f92bc49f..98d507b3d01 100644 --- a/apps/sim/components/settings/settings-header.tsx +++ b/apps/sim/components/settings/settings-header.tsx @@ -13,7 +13,8 @@ import { useRef, useState, } from 'react' -import { Chip, ChipInput, ChipLink, Search, Tooltip } from '@sim/emcn' +import { Chip, ChipInput, ChipLink, cn, Search, Tooltip } from '@sim/emcn' +import { PAGE_HEADER_BAR } from '@/components/page-header-bar' const useIsomorphicLayoutEffect = typeof window === 'undefined' ? useEffect : useLayoutEffect @@ -123,7 +124,7 @@ export function SettingsHeaderShell({ children }: { children: ReactNode }) { return (
-
+
{back ? ( configRef?.current.back?.onSelect()}> {back.text} From fadff0e35f0da0ff7eeec81a9846e4dd8ca697f8 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 30 Jul 2026 14:38:49 -0700 Subject: [PATCH 07/21] fix(integrations): keep the Atlassian coverage hint visible, and trim dead surface (#6105) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(integrations): keep the Atlassian coverage hint visible, and trim comments Cursor Bugbot: `ChipModalField` hides a `hint` whenever that field shows an `error`, so the multi-product coverage sentence vanished the moment the domain format check fired — exactly when someone mid-form most needs it. The sentence also describes the token, not the domain, so it read as domain guidance. Moves it to the API token field, which surfaces its errors through `ChipModalError` at the bottom rather than its own `error` prop, so the hint cannot be displaced. Also drops rationale comments that restated the code they sat above. * chore(integrations): trim the credential-display barrel to what consumers use * fix(emcn): let a custom modal field associate its hint with the control it wraps `ChipModalField` computes `aria-required`/`aria-invalid`/`aria-describedby` from its own state, but `type='custom'` returned its children untouched — so the `hint` and `error` text it renders was visible and never announced. The field cannot apply the ARIA itself here: a custom child may be a bare input or a wrapper several levels above one, which is the same reason `associatesLabel` already excludes custom from the label's `htmlFor`. Adds a function form for custom children that receives the ARIA, so the consumer — which knows where focus lands — attaches it. Existing `ReactNode` children are unaffected; all five current custom-field call sites keep working untouched. Uses it for the Atlassian API token field, whose coverage hint this branch had just relocated onto a custom field. --- .../[block]/integration-block-detail.tsx | 9 ++-- .../connect-service-account-modal.tsx | 43 ++++++++++--------- .../use-service-account-connect.ts | 7 +-- .../connected-credential-detail.tsx | 6 --- .../lib/integrations/credential-display.ts | 11 +++-- apps/sim/lib/integrations/index.ts | 5 --- .../src/components/chip-modal/chip-modal.tsx | 23 +++++++++- 7 files changed, 54 insertions(+), 50 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx index 6ad76f5996e..1558d9d1ca4 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx @@ -68,12 +68,9 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration useScrollRestoration(scrollContainerRef, { ready: !credentialsLoading }) /** - * Credentials that authenticate this integration. Matching goes through - * `credentialProviderMatchesService` so a family service account lists on - * every product it powers — one Atlassian token covers Jira, Jira Service - * Management, and Confluence. Comparing resolved `providerId`s instead would - * hide it from all three, since `atlassian-service-account` resolves to its - * own pseudo-service rather than to any product. + * Matches on the service's own id *or* its service-account id, so a family + * credential lists on every product it powers. Comparing resolved + * `providerId`s instead hides it from all of them. */ const connectedCredentials = useMemo(() => { if (!oauthService) return [] diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx index 22c36c6a71b..e9ceb2b019b 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx @@ -62,10 +62,11 @@ function openDocs(url: string): void { const ATLASSIAN_DOMAIN_HINT_REGEX = /^[a-z0-9-]+\.atlassian\.net$/i /** - * States the site-wide reach of the token up front. Users reaching this modal - * from the Jira page were left unsure whether they had connected Jira or Jira - * Service Management; the credential covers both, plus Confluence. Derived from - * the catalog so it cannot drift as Atlassian integrations are added. + * States the token's reach up front — the ambiguity this modal exists to remove. + * Sits on the API token field, not Site domain: it describes the token, and + * `ChipModalField` hides a `hint` whenever that field shows an `error`, which + * would drop it exactly while the user is correcting a domain typo. Derived + * from the catalog so it cannot drift as Atlassian integrations are added. */ const ATLASSIAN_COVERAGE_HINT = getServiceAccountCoverageSentence( ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID @@ -502,21 +503,24 @@ function AtlassianServiceAccountModal({ Add {serviceName} service account - - { - setApiToken(value) - if (error) setError(null) - }} - placeholder='Paste API token' - name='atlassian_service_account_api_token' - autoComplete='new-password' - autoCorrect='off' - autoCapitalize='off' - data-lpignore='true' - data-form-type='other' - /> + + {(aria) => ( + { + setApiToken(value) + if (error) setError(null) + }} + placeholder='Paste API token' + name='atlassian_service_account_api_token' + autoComplete='new-password' + autoCorrect='off' + autoCapitalize='off' + data-lpignore='true' + data-form-type='other' + /> + )} diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx index 4ddfb4a5024..13a7378f9b5 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/connected/[credentialId]/connected-credential-detail.tsx @@ -96,12 +96,6 @@ export function ConnectedCredentialDetail({ [oauthServiceNameByProviderId] ) - /** - * Service, brand tile, and copy all come from the shared resolver so this - * page, the integrations list, and the Cmd-K search agree on how a credential - * is named and branded — a family service account reads as its family - * ("Atlassian"), not as whichever product the provider walk happened to hit. - */ const display = useMemo( () => (credential ? resolveCredentialDisplay(credential) : null), [credential] diff --git a/apps/sim/lib/integrations/credential-display.ts b/apps/sim/lib/integrations/credential-display.ts index 9b9b20d927c..0dc2bcf5bfb 100644 --- a/apps/sim/lib/integrations/credential-display.ts +++ b/apps/sim/lib/integrations/credential-display.ts @@ -3,12 +3,11 @@ * catalog integrations it powers, which mark and brand tile represent it, and * the sentence describing it. * - * Before this module, three surfaces (the integrations list, the Cmd-K search - * items, and the credential detail page) each re-derived this by looking the - * catalog up by the OAuth service's *display name*. That silently failed for - * family service accounts: `atlassian-service-account` resolves to a - * pseudo-service named "Atlassian Service Account", which matches no catalog - * integration, so the credential lost its brand tile and its category. + * Replaces a display-name-keyed catalog lookup that three surfaces each + * re-derived. That keying silently failed for family service accounts: + * `atlassian-service-account` resolves to a pseudo-service named "Atlassian + * Service Account", which matches no catalog entry, so the credential lost its + * brand tile and its category filter. */ import type { ComponentType } from 'react' diff --git a/apps/sim/lib/integrations/index.ts b/apps/sim/lib/integrations/index.ts index bfd42fb29c6..12e58e2ff35 100644 --- a/apps/sim/lib/integrations/index.ts +++ b/apps/sim/lib/integrations/index.ts @@ -100,11 +100,6 @@ export function toIntegrationSummary(integration: Integration): IntegrationSumma export { type CredentialDisplay, - getIntegrationsForCredentialProvider, - getServiceAccountCoverageSentence, - getServiceAccountFamilyIcon, - getServiceAccountFamilyName, - isFamilyServiceAccount, resolveCredentialDisplay, } from '@/lib/integrations/credential-display' export { blockTypeToIconMap } from '@/lib/integrations/icon-mapping' diff --git a/packages/emcn/src/components/chip-modal/chip-modal.tsx b/packages/emcn/src/components/chip-modal/chip-modal.tsx index b722de672a2..586c7e1fbfb 100644 --- a/packages/emcn/src/components/chip-modal/chip-modal.tsx +++ b/packages/emcn/src/components/chip-modal/chip-modal.tsx @@ -549,9 +549,28 @@ export interface ChipModalEmailsFieldProps extends ChipModalFieldBaseProps { placeholder?: string } +/** + * ARIA the field derives from its own state and renders elsewhere in the row — + * the `hint`/`error` paragraph ids, plus `required`/`invalid` flags. + */ +export interface ChipModalFieldAria { + 'aria-required'?: boolean + 'aria-invalid'?: boolean + 'aria-describedby'?: string +} + interface ChipModalCustomFieldProps extends ChipModalFieldBaseProps { type: 'custom' - children: React.ReactNode + /** + * Arbitrary JSX, or a function receiving the field's {@link ChipModalFieldAria}. + * + * The owned control types wire this ARIA themselves, but a custom field can + * hold anything — a bare input, or a wrapper several levels above one — so + * the field cannot know which element should carry it. Use the function form + * whenever the child renders a focusable control, or its `hint`/`error` text + * is rendered but never announced. + */ + children: React.ReactNode | ((aria: ChipModalFieldAria) => React.ReactNode) } export type ChipModalFieldProps = @@ -718,7 +737,7 @@ function renderChipModalControl( case 'emails': return case 'custom': - return props.children + return typeof props.children === 'function' ? props.children(aria) : props.children } } From 0e4f0e733f7d88dd9894603531c39f7786ff920a Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 30 Jul 2026 14:40:19 -0700 Subject: [PATCH 08/21] fix(invitations): report every reason a batch of invites failed (#6093) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(invitations): report every reason a batch of invites failed The invite modal collapsed a multi-failure batch to `N invitations failed.` plus the first reason and discarded the rest. The batch endpoint rejects per email and each reason names its own address, so inviting several people who are all ineligible for the chosen membership — the common multi-failure case, since External requires every invitee to already be on a paid plan — named one of them and hid the others. The failed addresses are reseeded into the field as ordinary chips carrying no error state, so nothing else on screen identified them either. `buildInviteFailureMessage` now lists the distinct reasons, collapses identical ones, and reports the denominator ("2 of 5 invitations could not be sent") the way add-people-modal already did for the same partial -failure shape. Past three reasons the tail is counted rather than listed, so a large batch cannot bury the modal in near-identical sentences. Co-Authored-By: Claude * fix(invitations): name the membership field for the org it acts on The dropdown grants organization standing — Member and Admin join the org and take a seat, External does not — but the label said only "Membership", which reads as membership of the workspace being invited to, the thing the field directly above it already controls. The EE access-control modal has its own "Membership" field for permission-group membership, so the bare word was ambiguous across surfaces too. Sentence case matches the sibling "Workspace access" in the same form. Co-Authored-By: Claude --------- Co-authored-by: Claude --- .../invite-modal/invite-modal.test.tsx | 72 ++++++++++++++++++- .../components/invite-modal/invite-modal.tsx | 45 ++++++++++-- 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.test.tsx index 30dfb22fb17..5bf9becae94 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.test.tsx @@ -57,7 +57,10 @@ vi.mock('@/hooks/queries/workspace', () => ({ }, })) -import { InviteModal } from '@/app/workspace/[workspaceId]/components/invite-modal/invite-modal' +import { + buildInviteFailureMessage, + InviteModal, +} from '@/app/workspace/[workspaceId]/components/invite-modal/invite-modal' let container: HTMLDivElement let root: Root @@ -159,3 +162,70 @@ describe('InviteModal organization billing isolation', () => { expect(mockUseAdminWorkspaces).toHaveBeenCalledWith('user-1', undefined, { enabled: false }) }) }) + +describe('buildInviteFailureMessage', () => { + const notPaid = (email: string) => + `${email} is not on a paid Sim plan, so they cannot be added as an external collaborator. Invite them as a Member or Admin instead — that adds a seat.` + + it('returns the reason alone for a single failure, which already names the address', () => { + expect(buildInviteFailureMessage([{ email: 'a@x.com', error: notPaid('a@x.com') }], 3)).toBe( + notPaid('a@x.com') + ) + }) + + it('names every address when a whole batch is rejected for the same cause', () => { + const message = buildInviteFailureMessage( + [ + { email: 'a@x.com', error: notPaid('a@x.com') }, + { email: 'b@x.com', error: notPaid('b@x.com') }, + { email: 'c@x.com', error: notPaid('c@x.com') }, + ], + 3 + ) + + expect(message).toContain('None of the 3 invitations could be sent.') + expect(message).toContain('a@x.com') + expect(message).toContain('b@x.com') + expect(message).toContain('c@x.com') + }) + + it('reports the denominator when only some of the batch failed', () => { + const message = buildInviteFailureMessage( + [ + { email: 'a@x.com', error: notPaid('a@x.com') }, + { email: 'b@x.com', error: 'b@x.com has already been invited to this workspace' }, + ], + 5 + ) + + expect(message).toContain('2 of 5 invitations could not be sent.') + expect(message).toContain('b@x.com has already been invited to this workspace') + }) + + it('collapses identical reasons instead of repeating them', () => { + const message = buildInviteFailureMessage( + [ + { email: 'a@x.com', error: 'Something went wrong' }, + { email: 'b@x.com', error: 'Something went wrong' }, + ], + 2 + ) + + expect(message).toBe('None of the 2 invitations could be sent. Something went wrong') + }) + + it('counts the reasons it cannot list rather than dropping them silently', () => { + const message = buildInviteFailureMessage( + ['a', 'b', 'c', 'd', 'e'].map((name) => ({ + email: `${name}@x.com`, + error: notPaid(`${name}@x.com`), + })), + 5 + ) + + expect(message).toContain('None of the 5 invitations could be sent.') + expect(message).toContain('c@x.com') + expect(message).not.toContain('d@x.com') + expect(message).toContain('And 2 more.') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.tsx b/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.tsx index b3aa448ede2..e80ebb4033a 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/invite-modal/invite-modal.tsx @@ -13,6 +13,7 @@ import { toast, } from '@sim/emcn' import { createLogger } from '@sim/logger' +import type { BatchInvitationResult } from '@/lib/api/contracts/invitations' import { useSession } from '@/lib/auth/auth-client' import { isEnterprise } from '@/lib/billing/plan-helpers' import { isBillingEnabled } from '@/lib/core/config/env-flags' @@ -48,6 +49,40 @@ const MEMBERSHIP_HINTS: Record = { const EMPTY_WORKSPACE_IDS: string[] = [] +/** Distinct failure reasons listed in full before the remainder is counted instead. */ +const MAX_LISTED_FAILURE_REASONS = 3 + +/** + * Builds the submit error for a batch where some or all invitations failed. + * + * The server rejects per email and every reason names its own address, so the + * reasons are listed rather than collapsed to the first one: inviting several + * people who are all ineligible for the chosen membership is the common + * multi-failure case, and reporting one of them hides who else needs attention. + * Identical reasons are deduplicated, and beyond + * {@link MAX_LISTED_FAILURE_REASONS} the tail is counted rather than listed so a + * large batch cannot bury the modal in near-identical sentences. + */ +export function buildInviteFailureMessage( + failures: BatchInvitationResult['failed'], + attemptedCount: number +): string { + if (failures.length === 1) return failures[0].error + + const headline = + failures.length >= attemptedCount + ? `None of the ${failures.length} invitations could be sent.` + : `${failures.length} of ${attemptedCount} invitations could not be sent.` + + const reasons = [...new Set(failures.map((failure) => failure.error))] + const listed = reasons.slice(0, MAX_LISTED_FAILURE_REASONS) + const remaining = reasons.length - listed.length + + return [headline, ...listed, remaining > 0 ? `And ${remaining} more.` : null] + .filter(Boolean) + .join(' ') +} + interface InviteModalProps { open: boolean onOpenChange: (open: boolean) => void @@ -228,13 +263,9 @@ export function InviteModal({ } if (result.failed.length > 0) { - // Keep the failed addresses in the field, with the reason, for retry. + // Keep the failed addresses in the field, with the reasons, for retry. setEmails(result.failed.map((failure) => failure.email)) - setErrorMessage( - result.failed.length === 1 - ? result.failed[0].error - : `${result.failed.length} invitations failed. ${result.failed[0].error}` - ) + setErrorMessage(buildInviteFailureMessage(result.failed, emails.length)) return } @@ -314,7 +345,7 @@ export function InviteModal({ {isOrganizationInvite && ( Date: Thu, 30 Jul 2026 16:33:35 -0700 Subject: [PATCH 09/21] feat(function): custom sandboxes (#6071) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sandboxes): workspace dependency sets for Function blocks Named package sets a Function block can import from. The server canonicalizes and hashes the list; E2B prebuilds a content-addressed template per set, Daytona installs per execution. Create/edit is gated to Max or Enterprise via the shared workspace entitlement check; execution is deliberately ungated, so a downgraded workspace keeps running what it already built. Also on this branch: - Extract the duplicated dropdown/combobox option-fetch lifecycle into use-fetched-options. Only combobox had the dependency-change reset, so every dropdown with dependsOn + fetchOptions cleared its list and never repopulated until reopened. - Collapse the repeated Max-tier entitlement check onto one hasMaxTierWorkspaceAccess, shared by inbox, live sync, and sandboxes. - Resolve a personal payer's block state through getEffectiveBillingStatus in getBillingEntityBlockStatus, so the client-side Max gates agree with the server-side ones when blockOrgMembers' fan-out is stale. - Carve the Daytona dependency install out of the caller's execution budget instead of stacking on top of it. Co-Authored-By: Claude * chore(db): regenerate the sandboxes migration as 0273 Staging claimed 0271 and 0272 while this branch was out, so the hand-authored 0271_workspace_sandboxes was dropped before the merge and regenerated on top of the merged schema. Same DDL; drizzle emits plain CREATE TABLE/INDEX rather than the hand-added IF NOT EXISTS, which matches the repo default — that idempotent form is only needed for files with CONCURRENTLY ops below an embedded COMMIT. Regenerating also restores the meta snapshot the hand-authored migration never had. Co-Authored-By: Claude * chore(db): drop the sandboxes migration ahead of the staging merge Staging independently claims idx 0273, so remove ours before merging to avoid an add/add conflict on the drizzle migration index. Regenerated at the next free index once the merge lands. Co-Authored-By: Claude * chore(db): regenerate the sandboxes migration as 0275 Staging took 0273 and 0274, so the sandboxes DDL lands at the next free index. The emitted SQL is byte-identical to the dropped 0273. Co-Authored-By: Claude * fix(billing): consolidate the Max-tier entitlement onto one predicate The Max tier was spelled five ways. The odd one out — `isMax`, defined as `isPro(plan) && credits >= 25000` — excluded both `team_25000` and `enterprise`, and it was the sole input to the personal-workspace cap. A delinquent Max-for-Teams org admin got 1 personal workspace while a delinquent Max individual got 10. Only free/pro_6000/pro_25000 were tested, so the two broken tiers were unpinned. Separately, the server gate and the client `hasUsableMaxAccess` were independent copies of the same rule. The settings sidebar renders Sandboxes and Sim Mailer from the client one while the API answers 403 from the server one, so any drift renders a feature unlocked that the API refuses. - `MAX_TIER_CREDITS` is derived from the `CREDIT_TIERS` table; `isMaxTier` in plan-helpers is now the single definition, shared by the server gates, the client derivation, `getPlanTypeForLimits`, `plan-view`, and the cap - `hasWorkspaceTierAccess(id, predicate, { intent, onMissingWorkspace })` becomes the one org-vs-personal payer fork. `intent: 'active-use'` means active and not billing-blocked; `'retention'` means active/past_due with block state ignored, so the inbox teardown guard keeps its fail-open semantics instead of implying them through a duplicated fork - `isWorkspaceOnEnterprisePlan`'s personal branch now applies the status and block checks its own org branch always had, and its TSDoc names its real consumer (copilot BYOK, not Access Control) - the client live-sync gate gained the server's `isHosted` branch, so a self-hosted deploy with billing on no longer locks an interval the API accepts. It reads both flags directly rather than taking one as a parameter the callers sourced from the same module - `sqlIsPro`/`sqlIsTeam` escape the `_` LIKE wildcard, matching the already correct hand-rolled filter in seat-drift - deletes the `TERMINAL_SUBSCRIPTION_STATUSES` and `ENTITLED_STATUSES` shadow constants, and corrects three test mocks that asserted `trialing` was entitled or usable `max-tier-parity.test.ts` asserts the client and server answers match for every plan name. Both new guards were checked against the old code: the parity test fails 3 assertions with the previous predicate, and the self-hosted test fails without the `isHosted` branch. Co-Authored-By: Claude * chore(db): drop the sandboxes migration ahead of the staging merge Staging has claimed 0275 (table_views) and 0276 (drop_legacy_folder_tables) since the last merge, so our 0275_workspace_sandboxes collides on the index. Dropping ours first — the .sql, meta/0275_snapshot.json, and the journal entry — leaves packages/db/migrations byte-identical to the merge-base, so the merge sees no add/add conflict at all. Regenerated on the far side. Ours is the droppable side: plain additive DDL with no hand edits, which drizzle reproduces exactly. Staging's migrations are hand-written and must survive. Co-Authored-By: Claude * chore(db): regenerate the sandboxes migration as 0277 Staging claimed 0275 (table_views) and 0276 (drop_legacy_folder_tables), so the sandboxes migration dropped before the merge comes back on top as 0277. The emitted SQL is byte-identical to what was dropped — the original had no hand edits, so there is nothing to reapply. It is purely additive: two enums, sandbox_image and workspace_sandbox, their two FKs and six indexes. That it regenerated unchanged also confirms the schema.ts auto-merge was correct — had it lost staging's legacy-folder-table drops, drizzle would have emitted CREATE TABLE for them here. Snapshot chain is continuous (0273 -> 0277, each prevId matching the previous id) and the table counts track the DDL: 100 -> 101 (table_views) -> 99 (legacy folder tables dropped) -> 101 (the two sandbox tables). Co-Authored-By: Claude * feat(sandboxes): gate on the enterprise feature flags, drop the rollout switch Sandboxes shipped behind `custom-sandboxes`, an AppConfig rollout flag falling back to a `CUSTOM_SANDBOXES` secret. That made it the only Max-gated surface with no self-hosted path: `INBOX_ENABLED` can force Sim Mailer on for an operator running their own billing, and `ENTERPRISE_ENABLED` turns on the other nine features at once, but neither reached sandboxes. A self-hoster had to find a separately-named variable that was not part of that family, and one running with billing enabled could not enable it at all. Sandboxes now joins the enterprise feature set and the rollout flag is gone: - `sandboxes` is an `EnterpriseFeature` with `SANDBOXES_ENABLED` and its `NEXT_PUBLIC_` twin, so the master switch and the per-feature override both reach it like every sibling - `hasWorkspaceSandboxAccess` takes the inbox's shape exactly — the override wins, then a deployment without billing is unrestricted, then the workspace payer needs usable Max or Enterprise - the settings nav gains `selfHostedOverride`, so the section resolves through the same path as Sim Mailer instead of a second entitlement AND-ed in - `custom-sandboxes`, the `CUSTOM_SANDBOXES` secret, the now-unreachable `SANDBOXES_UNAVAILABLE` 403 copy, and the route's kill-switch branch are deleted Its legacy default is `true`, matching `inbox`: the gate already returns true whenever billing is off, so `false` would leave the nav override disagreeing with the gate that answers the request. Self-hosted builds run on the operator's own E2B/Daytona credentials, so there is no Sim-side cost to withhold — the docs now say so, since enabling the feature without a provider configured is the obvious trap. The new gate tests run with billing enabled on purpose; the `!isBillingEnabled` bail would otherwise answer every case and hide whether the override is wired. Verified by deleting the override line — exactly the one assertion fails. Co-Authored-By: Claude * fix(sandboxes): let the language menu match its trigger width `matchTriggerWidth={false}` exists for the opposite case — a narrow trigger whose option labels would truncate, letting the menu grow past it. The language field is a full-width form control with two short labels, so the override shrank the menu to "JavaScript" and pinned it to the right edge instead. The default (`true`) is correct here. Every other consumer passing `false` is a genuinely narrow trigger — a role picker in a member row, a table filter chip. Co-Authored-By: Claude * fix(sandboxes): re-queue a build when resolution finds the image unusable `ensureSandboxImage` only ran when a sandbox was saved, so resolution treated an unusable image as terminal and told the user to go fix a definition that was never wrong. Three states stuck permanently until someone re-saved in Settings: - a build that failed - a build whose worker died mid-flight, stranding the row in `building` - every sandbox created while the deployment ran a `runtime` provider, after a switch to a `prebuilt` one — `runtime` writes no image rows at all, so the whole fleet resolved to "no completed build" with nothing to repair it Resolution now re-queues through the registry's existing idempotent entry point before failing, and says a build is on its way instead of pointing at Settings. The conflict guard already claims only a `failed` row or a stale `pending`/ `building` one, so executions arriving during a healthy build enqueue nothing — no thundering herd from a hot workflow. The registry is imported dynamically for the same reason `sandboxDb` is: it pulls `@sim/db` into the static graph, which this module keeps out of the executor bundle. That also avoids a cycle, since the registry imports `invalidateSandboxResolution` from here. A repair that itself fails is logged and swallowed — it must never replace the build error naming the sandbox. Verified by deleting the repair call: exactly the three new assertions fail. Co-Authored-By: Claude * improvement(sandboxes): let the picker show just the sandbox name The label read "Test · Python · 1 package". The block's own list is already scoped to the language its sibling `language` subblock selects, so the language repeated on every row said nothing, and the package count is decoration next to the name that identifies the sandbox. The language stays for the one caller that cannot filter — agent tool-input renders this field under a synthetic id where the sibling `language` value is unreachable, so its list spans both languages and the name alone is ambiguous. That is the same missing value which disables filtering, so `showLanguage` is derived from it directly rather than passed independently and left to drift. A failed build is still marked: that suffix is the difference between a selection that runs and one that does not. Passing the flag also means dropping `.map(toSandboxOption)` for an explicit arrow — `Array.map` hands the index to the second parameter. Co-Authored-By: Claude * fix(sandboxes): show the sandbox name on the block card, not its uuid The card printed "443f4934-26ab-44ab-8...". `resolveDropdownLabel` only reads a subblock's static `options` array, and the sandbox picker is a `combobox` whose options load asynchronously, so its array is empty and the raw stored id fell through to the label. Resolved the same way skills and tools already are: a `resolveSandboxLabel` in the display layer, fed from the shared sandbox list query — the same cache entry the picker reads, so this adds no request. Two deliberate scopings: - the query is subscribed only for the sandbox row. `SubBlockRow` is memoized per subblock, and the list query polls while a build is in flight, so an unconditional hook would re-render every row on the canvas on each poll tick - the resolver matches the field id, not just the type. There is no dedicated subblock type for it, and matching `combobox` alone would relabel unrelated pickers An id with no matching sandbox resolves to null rather than a guess, so a deleted sandbox falls through to the caller's placeholder. The template preview surface is left alone: it is explicitly hook-free and passes empty lists for tools and skills too. Co-Authored-By: Claude * fix(sandboxes): hide the Sandboxes section with no provider configured Entitlement decides whether a workspace may author sandboxes; nothing decided whether anything could run one. A self-hosted deployment with SANDBOXES_ENABLED but no E2B or Daytona credentials got a fully functional tab whose output no Function block could select — the picker is gated on the provider vars, the tab was not. Both navigation planes now drop the section when neither NEXT_PUBLIC_SANDBOX_ENABLED nor the pre-Daytona NEXT_PUBLIC_E2B_ENABLED is set — the same pair the picker's `showWhenEnvSet` reads, so the two cannot disagree. Dropped rather than locked: an upgrade does not conjure a provider. The unified plane drops it in `buildUnifiedSettingsNavigation` rather than in the sidebar's filter, because the sidebar's `selfHostedOverride` short-circuit runs before its `requiresMax` check and would have revealed the tab anyway. It reads the browser twins, not the server's `isRemoteSandboxEnabled`, since this module renders on both sides. The predicate is a function, not a module constant, because the constant form was untestable and ambient: the env mock falls through to `process.env`, and `apps/sim/.env` (gitignored, so absent on CI) sets NEXT_PUBLIC_E2B_ENABLED=true. The nav tests passed locally and failed 6 assertions with the flag cleared. They now pin both flags, so the suite is identical with and without a local env file — verified by running it both ways. Co-Authored-By: Claude * docs(sandboxes): correct three claims the code no longer makes The Sandboxes section described behavior two commits on this branch changed, and led with an internal detail no reader needs. - entitlement is no longer Max/Enterprise only: self-hosted deployments unlock sandboxes with SANDBOXES_ENABLED, and the section is hidden outright when a deployment has no sandbox provider, which is the state a self-hoster is most likely to hit and least likely to diagnose - a build that is not Ready is no longer terminal. It is queued again on the next run, so the advice is to wait and re-run, not to go edit a package list that was never wrong - deleting a sandbox frees its build once nothing else references it. Builds are shared by content, so this is the one place a reader could reasonably assume deletion is immediate Dropped the `ModuleNotFoundError` aside: what the old code did instead is not something a reader needs to know to use the feature. The page is hand-written — `function` has category 'blocks' and is absent from `NATIVE_RESOURCE_BLOCK_TYPES`, so generate-docs skips it and these edits will not be overwritten. Co-Authored-By: Claude * feat(sandboxes): release the provider image when nothing references it Deleting a sandbox only removed its row, leaving the built template in E2B until the 30-day retention sweep — up to a month of paying to store an image nothing could select. Editing a package list had the same effect on the old content address, which is the more common case since every edit re-points the sandbox. `releaseSandboxImage(specHash)` now deletes the provider image and its row from both paths. It reuses the sweep's provider call and its ordering: image first, row second, so a refused delete leaves the row for the sweep to retry rather than orphaning a remote template nothing points at. Two guards make eager deletion safe: - builds are keyed by content, not by workspace, so two workspaces declaring the same package list share one image. The release no-ops while any sandbox still references the hash — otherwise one workspace's delete would break the other's - an in-flight build is left alone rather than raced; the sweep collects it once it settles Called detached from both routes. The row is already committed by then, so the user's action has succeeded whatever the provider says, and awaiting would hold a UI delete open on a remote call the sweep would retry anyway. Every failure inside is logged and swallowed for the same reason. E2B's delete verified against their API reference: DELETE /templates/{templateID} with X-API-Key, 204 on success. The existing implementation already matched, so this commit only adds the call sites and the guards. Co-Authored-By: Claude * fix(sandboxes): rate-limit the automatic rebuild, drop the one-off status dot Two follow-ups to the resolution repair. The repair had no rate limit. `ensureSandboxImage` re-claims a `failed` row on sight, and a bad package name fails in seconds, so the in-flight guard never closed the window: a workflow on a one-minute schedule would enqueue a build a minute against a package list that will never resolve, each one real provider build compute. Before the repair existed resolution simply threw, so this was introduced with it. The two callers want different things, so the cooldown is opt-in. A save is a person explicitly asking for another attempt and still retries immediately; resolution passes `FAILED_BUILD_RETRY_COOLDOWN_MS` and gets at most one attempt per window no matter how often the workflow runs. Ten minutes: long enough that per-minute runs cannot drive per-minute builds, short enough that a transient registry outage clears within the hour. The status line loses its colour dot. `size-[6px] rounded-full` appeared in exactly one file in the repo, so it was a new primitive rather than a pattern, and it duplicated state the text colour already carries — the label now turns `--text-error` on a failed build, which is what every other status row in settings does. `ChipTag` was the wrong home for this: its variants are `mono`/`invite`, with no semantic tone, so a status version would have meant overriding its chrome from the consumer. Also corrects the docs line this changes: a failed build is retried periodically, and saving is the way to retry now, so "wait a moment and run again" no longer describes it. Co-Authored-By: Claude * fix(sandboxes): claim the image row and its reference check in one statement Greptile P1. Reading references in one statement and deleting in another left a window — a wide one, since a provider delete is a network call — where a second workspace could declare the same package list, inherit the `ready` row, and have its next run fail against a template already on its way out. Content addressing is what makes that reachable: the image is shared, so one workspace's delete can strand another's sandbox. The reference check now lives in the conditional DELETE itself, so winning the delete is the proof that nothing referenced the hash. A workspace that adopts the hash first makes the delete match nothing and the release becomes a no-op. Claiming the row before the provider call would otherwise strand a template nothing points at if the provider then refused, so that path puts the row back and the retention sweep inherits the retry — the same property the previous ordering had. The sweep is deliberately left as it is: its equivalent window needs a hash unreferenced AND unused for 30 days, and its provider-first ordering encodes the documented retry-on-refusal behaviour this path now reproduces explicitly. No transaction is opened. The provider call sits between discrete statements rather than inside one, so no pooled connection is held across it — which is why this uses a conditional delete instead of the repo's `pg_advisory_xact_lock` pattern, whose lock only releases at commit. Co-Authored-By: Claude * fix(sandboxes): route the retention sweep through the same image claim Cursor and Greptile both flagged the sweep as still carrying the interleaving just fixed in releaseSandboxImage, and they are right — the reason given for leaving it alone last round does not survive scrutiny. That reason was that provider-first ordering encodes retry-on-refusal, so making the claim atomic would trade a race for an orphaned template. The release path already answers that: claim the row, and put it back if the provider refuses. The sweep can have both properties too. The rarity argument was also weaker than stated. The sweep nominates up to 200 candidates and then works through them eight network deletes at a time, so its check-to-delete gap is seconds to minutes — wider than the window that was just closed, not narrower. Both callers now share `claimAndDeleteImage`, which owns the whole contract: the unreferenced check lives inside the DELETE, the provider call runs only after the claim succeeds, and a refusal restores the row. Having written that ordering twice is what let the two paths drift, so it exists once now. The sweep's query becomes a nomination step only. Its retention cutoff is passed into the claim rather than trusted from the earlier read, so a candidate that stops qualifying mid-sweep fails its claim and is skipped instead of losing its image. Co-Authored-By: Claude * fix(sandboxes): rebuild a hash adopted while its image was being deleted Greptile's third pass on this path, and a case the previous two did not cover: the adopter starting a *fresh build* rather than inheriting a ready row. Claiming removes the registry row, so between that and the provider delete finishing, a workspace can declare the same package list, get a new row, and start a build under the same content-derived imageRef — which the in-flight delete then removes. The window itself is inherent. The registry row and the provider template are two systems with no shared transaction, so it can be narrowed but not closed. A Redis lock would not close it either: acquireLock returns true when Redis is absent, so it cannot be a correctness guarantee for self-hosted. Holding a Postgres advisory lock would, but only by pinning a pooled connection for the length of a provider call, which is a worse trade. What was avoidable is the adopter finding out the slow way. Its row is new and healthy-looking, so nothing noticed: resolution only repairs a row that is missing or failed, and a failed one waits out the retry cooldown first. The release path now re-checks after the delete and re-enqueues, so the rebuild starts immediately instead of one failed run plus a cooldown later. A build already in flight is left to the conflict guard, since it may still outlive the delete. Co-Authored-By: Claude * fix(sandboxes): reclaim a ready row whose image was deleted underneath it Greptile found the hole the previous commit left, and it is the case that made the claim in that commit's message wrong: this one is permanent, not transient. If a re-adopted hash reaches `ready` before the in-flight provider delete lands — plausible, since E2B layer caching can rebuild an identical spec in seconds — the row looks healthy while its imageRef points at nothing. Resolution repairs a row that is missing or failed, never one claiming to be ready, so nothing recovers it. The sandbox stays broken until someone re-saves it by hand. `rebuildIfReadopted` called `ensureSandboxImage` with no options, whose conflict guard reclaims only a failed or stale in-flight row, so it silently did nothing in exactly that case. The release path now passes `imageKnownGone`, which widens the re-claim to any settled row rather than only a failed one. It is the one caller that knows the image is gone regardless of what the row says. An in-flight build is still left alone: it either recreates the template it was building or fails into the normal repair path, and resetting it would only add a duplicate build. The three ways a settled row may be re-claimed now sit in one `settledRebuildBranch` helper — any settled row when the image is known gone, a failed one after the cooldown for an automatic caller, a failed one immediately for a person — because inlining the third case is what hid the gap. Co-Authored-By: Claude * fix(sandboxes): let a same-spec save retry a failed build Cursor Bugbot. `scheduleSandboxBuild` sat inside the changed-hash branch, so a save that did not alter the package list never reached the registry. The comment above it described the opposite — that an unchanged spec finds a ready row and enqueues nothing — which is what `ensureSandboxImage` does, but only if it is called. That made the docs wrong too. They tell a reader to save the sandbox again to retry a failed build immediately, and this branch is exactly why that did nothing: the only way to retry was to edit the package list into a different hash, which is not what someone recovering from a transient registry failure wants to do. The call is now unconditional and the registry decides what a save costs, which is what its conflict guard is for: a ready or in-flight row is left alone, a failed one is re-claimed at once. Releasing the previous image stays behind the hash check, since only a changed hash orphans one. Cache invalidation is unchanged — `scheduleSandboxBuild` already does it, which is why the else branch existed. Co-Authored-By: Claude * docs(sandboxes): correct the image cache's staleness invariant Cursor Bugbot found that a released image can still be served from another replica's cache. The finding is real, and the reason it went unnoticed is that the cache documented an invariant which eager release quietly broke. It claimed a `ready` row is terminal for its spec hash, so a cached hit could not go stale in a way that matters. That held while the only ways a row changed were an edit (new hash) or a delete (caught by the `workspace_sandbox` read). Releasing an image eagerly made a `ready` row disappear with the hash unchanged, so the premise no longer holds and the comment was actively misleading to the next reader. No behaviour change here — the exposure is bounded at IMAGE_TTL_MS on replicas other than the one that ran the release, and it self-heals once the entry expires and the row read finds nothing. Closing it properly needs cross-replica invalidation or a provider-error path that invalidates on "template not found", both of which are larger than a review fix; the comment now says so instead of implying the problem cannot exist. Co-Authored-By: Claude * docs(sandboxes): note that a JavaScript sandbox needs an import to apply Cursor Bugbot pointed out that `useRemoteSandbox` keys on detected static import/require and never on the selected sandbox, so JavaScript without one runs locally and the selection has no effect. Keeping the behaviour: honouring the selection would force those blocks remote, and the large-value-ref guard immediately below would then reject code that runs fine today. Documenting it instead, next to the picker, since a selection that silently does nothing is only surprising if nothing says so. Python is unaffected — it always runs remotely, so its sandbox always applies. Co-Authored-By: Claude * fix(sandboxes): stop create mode surviving a return to an open sandbox Cursor Bugbot. Create mode and having a sandbox open are mutually exclusive, but nothing enforced it, so both could be set at once — and the screen then lied about which sandbox its Delete pointed at. With `isCreating` true and `selectedId` restored, `baseline` is null, so the editor renders an empty "New sandbox" form, while the Delete action is built from `selected` and still targets the restored sandbox. An admin looking at a blank create form could delete a sandbox it never named. Two ways in, both closed: - Browser Forward after starting a new sandbox restores `selectedId` without going through `closeEditor`. The render-time sync that already drops a stale draft now also leaves create mode, which is the same class of correction and the reason that block exists. - "New sandbox" set `isCreating` without clearing `selectedId`, so the same contradiction was reachable without touching history at all. It now clears the selection, with `history: 'replace'` because switching mode is not a destination. Co-Authored-By: Claude * fix(ci): pin the sandbox flag in the second nav catalog test, bump the chart Two CI failures, both mine. `app/workspace/[workspaceId]/settings/navigation.test.ts` asserts the unified catalog and was left on ambient env. Dropping the Sandboxes section without a sandbox provider made it 26 items instead of 27 on CI, which has no `apps/sim/.env` — the same trap already fixed in the sibling `components/settings/navigation.test.ts`, in the one file that was missed. Fixing it needs `vi.hoisted` rather than the sibling's `beforeEach`, because this file reads `allNavigationItems`, built once at module load; a hook would run after the value it is trying to influence already exists. The chart gate is separate: this branch adds sandbox settings to `helm/sim/values.yaml`, and the workflow requires a Chart.yaml bump whenever `helm/sim/**` changes. Additive config, so 1.3.0 -> 1.4.0 by SemVer. Verified by running the whole suite with the flags forced off, not just the two navigation files — no other test depends on a local env file. Co-Authored-By: Claude * fix(sandboxes): keep the row restore to a refused delete only Cursor and Greptile, independently, on the same code. `deleteImage` and `rebuildIfReadopted` shared one try/catch, so a rebuild failure after a *successful* provider delete was handled as if the provider had refused: the catch put the claimed row back, `ready` status and all, pointing at a template that no longer exists. That is the one state resolution cannot repair — it fixes a row that is missing or failed, never one claiming to be ready — so it reintroduced the permanent breakage an earlier commit had just closed, through the error path rather than the happy one. Restoring now belongs strictly to a refused delete. Once the template is gone the row stays gone, and the rebuild runs past that catch. The rebuild also swallows its own failures: it follows a delete that already succeeded, so it must not be reported as a failed release, and inside the sweep it must not reject the rest of its chunk. The adopter's next run still reaches the normal repair path. The regression test drives a rebuild failure and asserts no row is restored. It fails against the original shape — rebuild inside the shared try, no inner catch — which is what the two reviewers were describing. Co-Authored-By: Claude * fix(sandboxes): drop the dead row when a re-adopt rebuild cannot be scheduled Greptile, one layer under the previous fix. Making the post-delete rebuild swallow its own failures kept it from being reported as a failed release, but left the adopter's row claiming a `ready` image whose template is already deleted — the one state resolution cannot repair, since it rebuilds a row that is missing or failed and never one that says ready. So the row is now dropped when the rebuild does not take. That turns the adopter into the missing-row case, which the next execution repairs on its own, instead of a sandbox that stays broken until someone re-saves it by hand. A failure to drop it is logged at error, because at that point two writes in a row have failed and there is nothing further this path can do. Also gives the release tests a default "nothing re-adopted" select. Without it the rebuild threw on an unstubbed mock and the cleanup delete overwrote the predicate the claim assertions read, so two of them were passing on the wrong statement. Co-Authored-By: Claude * feat(sandboxes): repair a missing image at create, where the truth is observable Six review rounds narrowed the window between deleting a shared template and another workspace adopting its content hash, and each fix exposed the next facet. They all share a cause: the registry row and the provider template are two systems with no shared transaction, so any scheme that keeps them in step is guessing. Create is the one step that does not have to guess. It either gets a sandbox or it does not, so a `ready` row pointing at a deleted template now corrects itself the first time it is used, rather than needing someone to re-save the sandbox. - `SandboxImageBuilder.isMissingImage` asks the provider to classify its own failure. Prebuilt-only, because a runtime provider has no image to miss - E2B answers it off `NotFoundError`, which the SDK maps from a 404. The only resource a create names is the template, and the two subclasses that describe other calls — a missing file, an exited sandbox — are excluded. The classifier stays deliberately narrow: treating auth or rate-limit failures as a missing image would turn a provider outage into a build storm - `repairMissingSandboxImage` invalidates the cache, rebuilds with `imageKnownGone` (no cooldown, since this observed the image is gone rather than inferring it), and returns copy telling the author to run again - `ResolvedSandbox` carries `specHash` so the failing execution can name what to rebuild This subsumes the open facets rather than adding another guard beside them: the stale per-replica cache, an adopter left `ready` against a deleted ref, and a rebuild that never took all end at the same place — the next run repairs itself. Co-Authored-By: Claude * fix(sandboxes): key the build trigger by attempt, not by spec Cursor Bugbot. The Trigger.dev idempotency key was the content address alone, so a second attempt at the same spec was deduped against the first: the SDK returns the finished run instead of starting one, and the row that `ensureSandboxImage` just flipped to `pending` sits there with no worker. Nothing can re-claim a `pending` row until it goes stale, so a retry inside the 5-minute TTL did nothing for the next half hour. That silently disabled every repair path — save-to-retry, which the docs name explicitly, and both the resolution and create-time rebuilds. The key's own comment already said it exists "to collapse concurrent saves of the same spec into one build, not to suppress a retry after one failed". The conditional update above it is what actually collapses concurrent saves: only one caller gets a row back, so only one ever reaches the trigger. Keying by the claim's `updatedAt` keeps that property and makes each genuine attempt distinct, while a duplicate delivery of one attempt still collapses. Co-Authored-By: Claude * feat(sandboxes): create a sandbox from the picker, and fix three UI papercuts The Function block's sandbox field now pins a "Create Sandbox" row above its options, matching the "Create Skill" / "Create Tool" rows it sits beside, so authoring a package list no longer means leaving the workflow for Settings. The row is declared by the field (`createAction`) rather than hardcoded by id; block configs are read by the serializer and executor, so the name maps to a modal in the picker rather than carrying a component. Two things the modal has to get right. It seeds the new sandbox's language from the sibling the list is scoped by, or a sandbox created off a JavaScript block would land in the Python list and vanish. And the created option is held locally until a real fetch carries it, or the field would sit on a raw uuid until hydration answered. Also: - The Sandboxes icon was the Logs block's icon (`blocks/blocks/logs.ts`), in both the settings nav and the list rows. It is the Function block's now. - "Default image (no extra packages)" claimed something untrue: E2B and Daytona base images both ship with packages installed. - A new sandbox opened in Python while the Function block defaults to JavaScript. The test pins the two together rather than the literal. Draft shape and helpers moved out of the editor component into `utils.ts` — three consumers now, and it makes the defaults testable without a DOM. * feat(settings): one Max-plan wall, and give the create modal the same one The create-sandbox modal answered a non-Max workspace with a red line under a form it could never submit, and no way to act on it. It now renders the same wall the Settings > Sandboxes tab does — heading, one sentence on what the plan unlocks, and an Upgrade to Max chip — instead of the fields. That wall existed twice already (sandboxes and Sim Mailer), so this extracts it rather than adding a third copy. `SettingsUpgradeNotice` owns the copy rhythm and the route, and `compact` trades the page's full-height centering for a modal's. Both settings consumers now compose it; neither keeps its own markup. The action lands on billing, which `resolveSettingsHref` already redirects to the plan-comparison page for a member who cannot manage billing — so it is a route to explore plans, never a dead end. The chip stays hidden for non-admins, exactly as the settings pages had it. A non-admin on an entitled workspace gets the muted reason rather than the upgrade wall: buying a plan is not what is in their way. --------- Co-authored-by: Claude --- .../en/platform/enterprise/self-hosted.mdx | 5 +- .../docs/en/workflows/blocks/function.mdx | 78 +- .../api/cron/cleanup-sandbox-images/route.ts | 31 + apps/sim/app/api/function/execute/route.ts | 51 +- apps/sim/app/api/organizations/route.test.ts | 5 +- apps/sim/app/api/wand/route.ts | 4 + .../[id]/sandboxes/[sandboxId]/route.ts | 139 + .../workspaces/[id]/sandboxes/authorize.ts | 124 + .../api/workspaces/[id]/sandboxes/route.ts | 101 + .../add-connector-modal.tsx | 3 +- .../components/connector-entitlements.test.ts | 76 +- .../[id]/components/connector-entitlements.ts | 16 +- .../edit-connector-modal.tsx | 3 +- .../[workspaceId]/settings/[section]/page.tsx | 7 +- .../settings/[section]/settings.tsx | 6 + .../settings/components/inbox/inbox.tsx | 30 +- .../components/sandbox-create-modal.tsx | 201 + .../sandboxes/components/sandbox-editor.tsx | 177 + .../settings/components/sandboxes/index.ts | 1 + .../components/sandboxes/sandboxes.tsx | 303 + .../components/sandboxes/search-params.ts | 17 + .../components/sandboxes/utils.test.ts | 57 + .../settings/components/sandboxes/utils.ts | 58 + .../settings-upgrade-notice/index.ts | 1 + .../settings-upgrade-notice.tsx | 58 + .../[workspaceId]/settings/navigation.test.ts | 15 +- .../sub-block/components/code/code.tsx | 2 + .../components/combobox/combobox.tsx | 244 +- .../components/dropdown/dropdown.tsx | 179 +- .../components/tools/sub-block-renderer.tsx | 12 +- .../sub-block/hooks/use-fetched-options.ts | 197 + .../hooks/use-editor-subblock-layout.ts | 4 + .../workflow-block/workflow-block.tsx | 20 + .../w/[workflowId]/hooks/use-wand.ts | 27 +- .../preview-editor/preview-editor.tsx | 4 + .../components/block/block.tsx | 4 + apps/sim/background/cleanup-sandbox-images.ts | 20 + apps/sim/background/sandbox-image-build.ts | 28 + apps/sim/blocks/blocks/function.ts | 70 +- apps/sim/blocks/types.ts | 25 +- .../components/settings/navigation.test.ts | 53 +- apps/sim/components/settings/navigation.ts | 81 +- .../handlers/function/function-handler.ts | 1 + apps/sim/hooks/queries/sandboxes.ts | 136 + apps/sim/lib/api/contracts/hotspots.ts | 6 + apps/sim/lib/api/contracts/index.ts | 1 + apps/sim/lib/api/contracts/sandboxes.ts | 151 + apps/sim/lib/billing/client/plan-view.ts | 4 +- apps/sim/lib/billing/client/utils.ts | 7 +- apps/sim/lib/billing/constants.ts | 19 +- apps/sim/lib/billing/core/access.test.ts | 160 + apps/sim/lib/billing/core/access.ts | 30 +- .../sim/lib/billing/core/subscription.test.ts | 141 +- apps/sim/lib/billing/core/subscription.ts | 205 +- apps/sim/lib/billing/core/workspace-access.ts | 4 + .../lib/billing/enterprise-provisioning.ts | 5 +- apps/sim/lib/billing/max-tier-parity.test.ts | 108 + apps/sim/lib/billing/plan-helpers.ts | 26 +- .../sim/lib/billing/webhooks/invoices.test.ts | 2 +- .../core/config/enterprise-entitlements.ts | 8 + apps/sim/lib/core/config/env-flags.ts | 21 + apps/sim/lib/core/config/env.ts | 6 + .../lib/core/rate-limiter/route-helpers.ts | 17 + .../execution/remote-sandbox/build-errors.ts | 154 + .../remote-sandbox/conformance.test.ts | 157 + .../lib/execution/remote-sandbox/daytona.ts | 32 +- apps/sim/lib/execution/remote-sandbox/e2b.ts | 196 +- .../remote-sandbox/image-registry.test.ts | 438 + .../remote-sandbox/image-registry.ts | 592 + .../sim/lib/execution/remote-sandbox/index.ts | 168 +- .../lib/execution/remote-sandbox/provider.ts | 41 + .../execution/remote-sandbox/resolve.test.ts | 433 + .../lib/execution/remote-sandbox/resolve.ts | 464 + .../remote-sandbox/sandbox-spec.test.ts | 190 + .../execution/remote-sandbox/sandbox-spec.ts | 294 + .../sim/lib/execution/remote-sandbox/types.ts | 76 +- .../execution/remote-sandbox/wand-enricher.ts | 170 + .../remote-sandbox/workspace-sandboxes.ts | 206 + apps/sim/lib/workflows/autolayout/utils.ts | 5 + .../lib/workflows/subblocks/display.test.ts | 24 + apps/sim/lib/workflows/subblocks/display.ts | 23 + apps/sim/lib/workflows/subblocks/options.ts | 118 + .../sim/lib/workflows/subblocks/visibility.ts | 31 +- apps/sim/lib/workspaces/admin-move.ts | 10 +- apps/sim/lib/workspaces/policy.test.ts | 70 + apps/sim/lib/workspaces/policy.ts | 14 +- apps/sim/providers/types.ts | 7 + apps/sim/providers/utils.ts | 71 +- apps/sim/scripts/verify-sandbox-parity.ts | 55 + apps/sim/serializer/index.ts | 6 + apps/sim/tools/function/execute.ts | 21 + apps/sim/tools/function/types.ts | 9 + apps/sim/tools/params.ts | 24 +- helm/sim/Chart.yaml | 2 +- helm/sim/values.yaml | 12 + .../migrations/0277_workspace_sandboxes.sql | 39 + .../db/migrations/meta/0277_snapshot.json | 18285 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/schema.ts | 83 + packages/testing/src/mocks/env-flags.mock.ts | 2 + packages/testing/src/mocks/schema.mock.ts | 27 + scripts/check-api-validation-contracts.ts | 5 +- scripts/setup/checks.ts | 20 + 103 files changed, 25593 insertions(+), 583 deletions(-) create mode 100644 apps/sim/app/api/cron/cleanup-sandbox-images/route.ts create mode 100644 apps/sim/app/api/workspaces/[id]/sandboxes/[sandboxId]/route.ts create mode 100644 apps/sim/app/api/workspaces/[id]/sandboxes/authorize.ts create mode 100644 apps/sim/app/api/workspaces/[id]/sandboxes/route.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-create-modal.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-editor.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/sandboxes.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/search-params.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/index.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/settings-upgrade-notice.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-fetched-options.ts create mode 100644 apps/sim/background/cleanup-sandbox-images.ts create mode 100644 apps/sim/background/sandbox-image-build.ts create mode 100644 apps/sim/hooks/queries/sandboxes.ts create mode 100644 apps/sim/lib/api/contracts/sandboxes.ts create mode 100644 apps/sim/lib/billing/core/access.test.ts create mode 100644 apps/sim/lib/billing/max-tier-parity.test.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/build-errors.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/image-registry.test.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/image-registry.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/provider.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/resolve.test.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/resolve.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/sandbox-spec.test.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/sandbox-spec.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/wand-enricher.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/workspace-sandboxes.ts create mode 100644 packages/db/migrations/0277_workspace_sandboxes.sql create mode 100644 packages/db/migrations/meta/0277_snapshot.json diff --git a/apps/docs/content/docs/en/platform/enterprise/self-hosted.mdx b/apps/docs/content/docs/en/platform/enterprise/self-hosted.mdx index db0889249d0..b54020b1c31 100644 --- a/apps/docs/content/docs/en/platform/enterprise/self-hosted.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/self-hosted.mdx @@ -23,7 +23,7 @@ ENTERPRISE_ENABLED=true NEXT_PUBLIC_ENTERPRISE_ENABLED=true ``` -That turns on organizations, permission groups, SSO, whitelabeling, audit logs, session policies, data retention, data drains, workspace forks, and the inbox. +That turns on organizations, permission groups, SSO, whitelabeling, audit logs, session policies, data retention, data drains, workspace forks, sandboxes, and the inbox. ### Turning one feature off @@ -51,6 +51,9 @@ The individual flags also work on their own if you would rather opt in one at a | Data drains | `DATA_DRAINS_ENABLED` | `NEXT_PUBLIC_DATA_DRAINS_ENABLED` | | Workspace forks | `FORKING_ENABLED` | — | | Sim Mailer inbox | `INBOX_ENABLED` | `NEXT_PUBLIC_INBOX_ENABLED` | +| Sandboxes | `SANDBOXES_ENABLED` | `NEXT_PUBLIC_SANDBOXES_ENABLED` | + +Sandboxes also need a remote execution provider, since the deployment builds the images itself: set `E2B_API_KEY`, or `SANDBOX_PROVIDER=daytona` with a `DAYTONA_API_KEY` that has `write:snapshots` and `write:sandboxes`. Without one the settings section appears but builds fail. Data retention is the one feature that deletes data. Its flag controls the cleanup pass, not the settings screen — retention windows are always configurable. Nothing is ever deleted until you enable it, and even then only against windows you configured explicitly. Sim never applies the hosted plan defaults to a self-hosted deployment. diff --git a/apps/docs/content/docs/en/workflows/blocks/function.mdx b/apps/docs/content/docs/en/workflows/blocks/function.mdx index e89939df2ee..2e555979bd6 100644 --- a/apps/docs/content/docs/en/workflows/blocks/function.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/function.mdx @@ -74,6 +74,82 @@ Beyond the Python standard library, the sandbox ships E2B's data-science stack p - **Math and testing:** `sympy`, `pytest` - **Sim additions:** `awscli`, `yq`, `csvkit` +## Sandboxes + +A sandbox is a named dependency set your workspace maintains — a language plus a +list of pip or npm packages. Select one on a Function block and its code can +import everything on that list. Leave it empty and the block runs on the default +image, exactly as before. + +Create and edit sandboxes in **Settings → Sandboxes**. Only workspace admins can +create or edit them. On sim.ai they need an active Max or Enterprise plan; +self-hosted deployments turn them on with `SANDBOXES_ENABLED` (see +[self-hosted enterprise](/platform/enterprise/self-hosted)). The section is +hidden when a deployment has no sandbox provider configured. + +1. **Name** the sandbox — `bigquery-etl`, `scraping`, whatever the job is. +2. Pick the **language**. A sandbox is language-scoped, so a Python block only + ever lists Python sandboxes. +3. Paste your **dependencies**, one per line. Version pins are optional. + +``` +google-cloud-bigquery==3.25.0 +pyairtable>=3.0 +pandas +``` + +Then open the block's advanced options and choose the sandbox under **Sandbox**. + +In JavaScript the sandbox applies to code that uses `import` or `require` — that is +what sends the block to a remote sandbox in the first place, so a block without them +keeps running locally and ignores the selection. Python always runs remotely, so a +selected sandbox always applies. + + +Two sandboxes with the same language and the same package list share one build, +so duplicating a set costs nothing. Editing a package list starts a new build; +runs already in flight keep using the old one. Deleting a sandbox frees its build +once nothing else uses it. + + +### Build status + +On sim.ai, each dependency set is prebuilt into a reusable image, so runs pay no +install cost. The status row in Settings shows **Queued**, **Building**, +**Ready**, or **Failed**. A failed build reports what went wrong — a package that +does not exist, a version that has no match, a resolver conflict — with the +installer log behind a disclosure. + +Running a block before its sandbox is **Ready** stops the run and shows you the +status. A failed build is retried periodically on its own; to retry immediately, +save the sandbox again in Settings. + +On a self-hosted deployment using Daytona, dependencies install inside the +sandbox at the start of every run instead, adding roughly 10–30 seconds per +execution. Prebuilt images require E2B. + +### What is allowed + +Package names and version specifiers only. URLs, `git+` references, `-e`, local +paths, `--index-url`, and npm aliases are rejected, with the offending line +number reported. A sandbox may declare up to 50 packages. + +## Scoping secrets for agent tools + +When a Function block is used as an Agent tool, its code can read every workspace +secret by default — both `{{MY_SECRET}}` and `environmentVariables['MY_SECRET']`. + +To narrow that, set **Secret access** to *Selected secrets* in the block's +tool configuration and pick the names the code may read. Two things change: + +- Only those secrets are injected. `{{OTHER_SECRET}}` no longer resolves either. +- The selected **names** are added to the tool's description, so the model knows + what it can reference. Values are never sent to the model — they are injected + server-side at execution. + +Leaving the default (*All secrets*) resolves the list at run time, so a secret +added next month is included automatically. + ## Examples ### Reshape an API response @@ -170,6 +246,6 @@ The lazy `sim.files` and `sim.values` helpers are available only in JavaScript f { question: "When does code run locally vs. in a sandbox?", answer: "JavaScript without external imports runs in a local isolated sandbox for speed. JavaScript that uses import or require runs in E2B. Python always runs in the E2B sandbox, with or without imports." }, { question: "How do I reference outputs from other blocks inside my code?", answer: "Use angle-bracket syntax directly, like or , with no quotes around the tag — Sim replaces it with the real value before execution. For environment variables, use double curly braces: {{API_KEY}}." }, { question: "What does the Function block return?", answer: "Two outputs: result (the return value of your code, read as ) and stdout (anything logged with console.log or print, read as ). Include a return statement in JavaScript, or print JSON in Python, to pass data downstream." }, - { question: "Can I make HTTP requests from a Function block?", answer: "Yes. fetch() is available in JavaScript with async/await; libraries like axios are not, only the built-in fetch. In Python, use requests or httpx in the E2B sandbox." }, + { question: "Can I make HTTP requests from a Function block?", answer: "Yes. fetch() is available in JavaScript with async/await; libraries like axios are only available when the block has a sandbox selected. In Python, use requests or httpx in the E2B sandbox." }, { question: "Is there a timeout for Function block execution?", answer: "Yes, a configurable execution timeout. If your code exceeds it, the run is terminated and the block reports an error. Keep this in mind for external calls or heavy processing." }, ]} /> diff --git a/apps/sim/app/api/cron/cleanup-sandbox-images/route.ts b/apps/sim/app/api/cron/cleanup-sandbox-images/route.ts new file mode 100644 index 00000000000..0990d8ccdbc --- /dev/null +++ b/apps/sim/app/api/cron/cleanup-sandbox-images/route.ts @@ -0,0 +1,31 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { verifyCronAuth } from '@/lib/auth/internal' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { runCleanupSandboxImages } from '@/background/cleanup-sandbox-images' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('CleanupSandboxImagesAPI') + +/** + * Retention sweep for prebuilt sandbox images. A runtime-strategy deployment has + * nothing to collect, so the sweep is a no-op there rather than a special case + * here. + */ +export const GET = withRouteHandler(async (request: NextRequest) => { + const authError = verifyCronAuth(request, 'sandbox image cleanup') + if (authError) return authError + + try { + const result = await runCleanupSandboxImages() + return NextResponse.json({ success: true, ...result }) + } catch (error) { + logger.error('Failed to sweep sandbox images', { error }) + return NextResponse.json( + { error: getErrorMessage(error, 'Failed to sweep sandbox images') }, + { status: 500 } + ) + } +}) diff --git a/apps/sim/app/api/function/execute/route.ts b/apps/sim/app/api/function/execute/route.ts index e7c87cb6457..59a295bc924 100644 --- a/apps/sim/app/api/function/execute/route.ts +++ b/apps/sim/app/api/function/execute/route.ts @@ -546,6 +546,42 @@ function resolveWorkflowVariables( return resolvedCode } +/** + * Narrows the secrets an execution can see, per the block's stored scope. + * + * | Stored value | Behavior | + * |-----------------------------|-------------------------------------------------| + * | unset (every block today) | all secrets — the regression-safe default | + * | `'all'` | all secrets, resolved now so later additions land | + * | `'selected'` + names | only those | + * | `'selected'` + empty list | none — an explicit deny | + * + * Unset and `'all'` must both inject everything: agent-authored code already + * reads `{{MY_SECRET}}` and `environmentVariables['MY_SECRET']` today, so a + * default-deny would silently break prompts that work right now. + */ +function scopeEnvironmentVariables( + envVars: Record, + scope: 'all' | 'selected' | undefined, + mountedSecrets: string[] | undefined +): Record { + if (scope !== 'selected') return envVars + + const allowed = new Set(mountedSecrets ?? []) + const scoped: Record = {} + const missing: string[] = [] + for (const name of allowed) { + if (name in envVars) scoped[name] = envVars[name] + else missing.push(name) + } + if (missing.length > 0) { + // A secret that was renamed or deleted since the block was configured. Drop + // it rather than failing: the code's own error is clearer than ours. + logger.warn('Mounted secrets no longer exist in this workspace', { missing }) + } + return scoped +} + function resolveEnvironmentVariables( code: string, params: Record, @@ -1401,7 +1437,10 @@ export const POST = withRouteHandler(async (req: NextRequest) => { outputSandboxPath, overwriteFileId, outputs, - envVars = {}, + envVars: rawEnvVars = {}, + secretScope, + mountedSecrets, + sandboxId: selectedSandboxId, blockData = {}, blockNameMapping = {}, blockOutputSchemas = {}, @@ -1417,6 +1456,10 @@ export const POST = withRouteHandler(async (req: NextRequest) => { isCustomTool = false, _sandboxFiles, } = body + // Scoped before {{VAR}} resolution so the `{{NAME}}` path and the + // `environmentVariables[...]` dict narrow together — filtering only the dict + // would leave `{{OTHER_SECRET}}` resolving, which is a hole, not a scope. + const envVars = scopeEnvironmentVariables(rawEnvVars, secretScope, mountedSecrets) sourceCodeForErrors = sourceCode const outputFiles = getOutputFileDeclarations({ outputs, @@ -1547,6 +1590,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => { sandboxFiles: _sandboxFiles, outputSandboxPath, outputSandboxPaths, + workspaceId, + sandboxId: selectedSandboxId, }) const executionTime = Date.now() - execStart @@ -1706,6 +1751,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => { sandboxFiles: _sandboxFiles, outputSandboxPath, outputSandboxPaths, + workspaceId, + sandboxId: selectedSandboxId, }) const executionTime = Date.now() - execStart stdout += e2bStdout @@ -1794,6 +1841,8 @@ export const POST = withRouteHandler(async (req: NextRequest) => { sandboxFiles: _sandboxFiles, outputSandboxPath, outputSandboxPaths, + workspaceId, + sandboxId: selectedSandboxId, }) const executionTime = Date.now() - execStart stdout += e2bStdout diff --git a/apps/sim/app/api/organizations/route.test.ts b/apps/sim/app/api/organizations/route.test.ts index a0e076b22dc..24fd78a66b3 100644 --- a/apps/sim/app/api/organizations/route.test.ts +++ b/apps/sim/app/api/organizations/route.test.ts @@ -41,12 +41,13 @@ vi.mock('@/lib/billing/organizations/create-organization', () => ({ OrganizationSlugTakenError: class OrganizationSlugTakenError extends Error {}, })) +/** Mirrors the real predicate, which also admits the `team_*` credit tiers. */ vi.mock('@/lib/billing/plan-helpers', () => ({ - isOrgPlan: (plan: string) => plan === 'team' || plan === 'enterprise', + isOrgPlan: (plan: string) => plan === 'team' || plan.startsWith('team_') || plan === 'enterprise', })) vi.mock('@/lib/billing/subscriptions/utils', () => ({ - ENTITLED_SUBSCRIPTION_STATUSES: ['active', 'trialing'], + ENTITLED_SUBSCRIPTION_STATUSES: ['active', 'past_due'], })) vi.mock('@/lib/workspaces/organization-workspaces', () => ({ diff --git a/apps/sim/app/api/wand/route.ts b/apps/sim/app/api/wand/route.ts index 6222c75ad45..617629d4a06 100644 --- a/apps/sim/app/api/wand/route.ts +++ b/apps/sim/app/api/wand/route.ts @@ -20,6 +20,7 @@ import { env } from '@/lib/core/config/env' import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { enrichSandboxPackages } from '@/lib/execution/remote-sandbox/wand-enricher' import { enrichTableSchema } from '@/lib/table/llm/wand' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' import { extractResponseText, parseResponsesUsage } from '@/providers/openai/utils' @@ -90,6 +91,9 @@ Use this context to calculate relative dates like "yesterday", "last week", "beg }, 'table-schema': enrichTableSchema, + // Both the JavaScript and Python code prompts generate under this type — the + // Python swap in `code.tsx` replaces the prompt text but keeps the type. + 'javascript-function-body': enrichSandboxPackages, } async function updateUserStatsForWand( diff --git a/apps/sim/app/api/workspaces/[id]/sandboxes/[sandboxId]/route.ts b/apps/sim/app/api/workspaces/[id]/sandboxes/[sandboxId]/route.ts new file mode 100644 index 00000000000..98bdce0b87c --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/sandboxes/[sandboxId]/route.ts @@ -0,0 +1,139 @@ +import { db } from '@sim/db' +import { workspaceSandbox } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { and, eq } from 'drizzle-orm' +import { type NextRequest, NextResponse } from 'next/server' +import { deleteSandboxContract, updateSandboxContract } from '@/lib/api/contracts/sandboxes' +import { parseRequest } from '@/lib/api/server' +import { runDetached } from '@/lib/core/utils/background' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { releaseSandboxImage } from '@/lib/execution/remote-sandbox/image-registry' +import { invalidateSandboxResolution } from '@/lib/execution/remote-sandbox/resolve' +import { + isSandboxNameTaken, + readWorkspaceSandbox, + scheduleSandboxBuild, +} from '@/lib/execution/remote-sandbox/workspace-sandboxes' +import { + authorizeSandboxMutation, + buildSpecOrResponse, + isNameConflictError, + nameConflictResponse, +} from '@/app/api/workspaces/[id]/sandboxes/authorize' + +const logger = createLogger('WorkspaceSandboxAPI') + +type SandboxContext = { params: Promise<{ id: string; sandboxId: string }> } + +export const PATCH = withRouteHandler(async (request: NextRequest, context: SandboxContext) => { + const { id: workspaceId, sandboxId } = await context.params + + const authorized = await authorizeSandboxMutation(workspaceId) + if (!authorized.ok) return authorized.response + + const parsed = await parseRequest(updateSandboxContract, request, context) + if (!parsed.success) return parsed.response + const { name, language, dependencies } = parsed.data.body + + const [existing] = await db + .select({ + id: workspaceSandbox.id, + name: workspaceSandbox.name, + language: workspaceSandbox.language, + dependencies: workspaceSandbox.dependencies, + specHash: workspaceSandbox.specHash, + }) + .from(workspaceSandbox) + .where(and(eq(workspaceSandbox.id, sandboxId), eq(workspaceSandbox.workspaceId, workspaceId))) + .limit(1) + + if (!existing) { + return NextResponse.json({ error: 'Sandbox not found' }, { status: 404 }) + } + + const nextName = name ?? existing.name + if (name && name !== existing.name && (await isSandboxNameTaken(workspaceId, name, sandboxId))) { + return nameConflictResponse(name) + } + + // Both halves are revalidated together even when only one changed: switching + // language has to re-check the existing list against the new language's rules, + // and editing dependencies has to check them against the stored language. + const nextLanguage = language ?? (existing.language as 'javascript' | 'python') + const nextDependencies = dependencies ?? existing.dependencies ?? [] + + const built = buildSpecOrResponse(nextLanguage, nextDependencies) + if (!built.ok) return built.response + const { spec } = built + + try { + await db + .update(workspaceSandbox) + .set({ + name: nextName, + language: spec.language, + dependencies: spec.dependencies, + specHash: spec.specHash, + updatedAt: new Date(), + }) + // Scoped by workspace as well as id: every other query here is, and relying on + // the SELECT above to have 404'd first makes authz an ordering invariant. + .where(and(eq(workspaceSandbox.id, sandboxId), eq(workspaceSandbox.workspaceId, workspaceId))) + } catch (error) { + // The pre-check above can lose a race with a concurrent rename; the unique + // index is the real arbiter, and losing it is a conflict, not a server fault. + if (isNameConflictError(error)) return nameConflictResponse(nextName) + throw error + } + + // Unconditional, because the registry decides what a save costs: a `ready` or + // in-flight row is left alone, so renaming or re-saving an unchanged spec + // enqueues nothing, while a failed one gets the immediate retry a person saving + // is asking for. Gating this on a changed hash meant a same-spec save silently + // did nothing, and the only way to retry a failed build was to edit the package + // list into a different hash. + await scheduleSandboxBuild(spec) + + if (spec.specHash !== existing.specHash) { + // The previous content address is unreferenced by this sandbox now. Release + // no-ops when another sandbox still declares the same package list. + runDetached('release-sandbox-image', () => releaseSandboxImage(existing.specHash)) + logger.info('Sandbox spec changed, scheduled a build', { workspaceId, sandboxId }) + } + + const sandbox = await readWorkspaceSandbox(workspaceId, sandboxId) + if (!sandbox) { + return NextResponse.json({ error: 'Failed to read back the updated sandbox' }, { status: 500 }) + } + return NextResponse.json({ sandbox }) +}) + +export const DELETE = withRouteHandler(async (request: NextRequest, context: SandboxContext) => { + const { id: workspaceId, sandboxId } = await context.params + + const authorized = await authorizeSandboxMutation(workspaceId) + if (!authorized.ok) return authorized.response + + const parsed = await parseRequest(deleteSandboxContract, request, context) + if (!parsed.success) return parsed.response + + // A block may still reference this sandbox. Deleting is allowed anyway; that + // execution then fails closed with a message naming the missing sandbox, + // rather than silently falling back to an image without its dependencies. + const deleted = await db + .delete(workspaceSandbox) + .where(and(eq(workspaceSandbox.id, sandboxId), eq(workspaceSandbox.workspaceId, workspaceId))) + .returning({ id: workspaceSandbox.id, specHash: workspaceSandbox.specHash }) + + if (deleted.length === 0) { + return NextResponse.json({ error: 'Sandbox not found' }, { status: 404 }) + } + + invalidateSandboxResolution() + // Detached: the row is already gone, so the caller's delete succeeded whatever + // the provider says. Awaiting would hold a UI delete open on a remote call the + // retention sweep would retry anyway. + runDetached('release-sandbox-image', () => releaseSandboxImage(deleted[0].specHash)) + logger.info('Deleted workspace sandbox', { workspaceId, sandboxId }) + return NextResponse.json({ success: true }) +}) diff --git a/apps/sim/app/api/workspaces/[id]/sandboxes/authorize.ts b/apps/sim/app/api/workspaces/[id]/sandboxes/authorize.ts new file mode 100644 index 00000000000..7cfef4bea23 --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/sandboxes/authorize.ts @@ -0,0 +1,124 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { getSession } from '@/lib/auth' +import { hasWorkspaceSandboxAccess } from '@/lib/billing/core/subscription' +import { enforceWorkspaceRateLimit } from '@/lib/core/rate-limiter/route-helpers' +import type { SandboxLanguage } from '@/lib/execution/remote-sandbox/sandbox-spec' +import { + buildSpecUpdate, + MAX_PLAN_REQUIRED, + SANDBOX_ADMIN_REQUIRED, + SANDBOX_MUTATION_LIMIT, + SandboxDependencyError, + type SandboxSpecUpdate, + WORKSPACE_SANDBOX_NAME_INDEX, +} from '@/lib/execution/remote-sandbox/workspace-sandboxes' +import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' + +export interface SandboxMutationActor { + userId: string + name?: string | null + email?: string | null +} + +/** + * The 409 both write paths return for a duplicate name. Shared so the pre-check + * and the unique-index catch cannot describe the same conflict differently. + */ +export function nameConflictResponse(name: string): NextResponse { + return NextResponse.json( + { error: `A sandbox named "${name}" already exists in this workspace` }, + { status: 409 } + ) +} + +/** + * Validates a submitted dependency list, returning the 400 the editor knows how + * to read — `issues` carries a line number per rejected row, which the generic + * validation error does not. + */ +export function buildSpecOrResponse( + language: SandboxLanguage, + dependencies: readonly string[] +): { ok: true; spec: SandboxSpecUpdate } | { ok: false; response: NextResponse } { + try { + return { ok: true, spec: buildSpecUpdate(language, dependencies) } + } catch (error) { + if (error instanceof SandboxDependencyError) { + return { + ok: false, + response: NextResponse.json( + { error: error.message, issues: error.issues }, + { status: 400 } + ), + } + } + throw error + } +} + +/** + * Whether a write failed because it collided with the workspace/name unique + * index. Both paths pre-check the name, but the index is the real arbiter and a + * concurrent write can still lose the race — which is a 409, not a 500. + */ +export function isNameConflictError(error: unknown): boolean { + const message = getErrorMessage(error) + return message.includes(WORKSPACE_SANDBOX_NAME_INDEX) || message.includes('23505') +} + +/** + * Authenticates, authorizes, entitles, and rate-limits a sandbox mutation — in + * that order, and always before any untrusted input is parsed. + * + * Shared by both route files so the create path and the edit/delete path cannot + * drift into different checks. + */ +export async function authorizeSandboxMutation( + workspaceId: string +): Promise<{ ok: true; actor: SandboxMutationActor } | { ok: false; response: NextResponse }> { + const session = await getSession() + if (!session?.user?.id) { + return { ok: false, response: NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } + } + const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) + if (permission !== 'admin') { + return { + ok: false, + response: NextResponse.json({ error: SANDBOX_ADMIN_REQUIRED }, { status: 403 }), + } + } + if (!(await hasWorkspaceSandboxAccess(workspaceId))) { + return { + ok: false, + response: NextResponse.json({ error: MAX_PLAN_REQUIRED }, { status: 403 }), + } + } + const limited = await enforceWorkspaceRateLimit( + 'sandbox-mutations', + workspaceId, + SANDBOX_MUTATION_LIMIT + ) + if (limited) return { ok: false, response: limited } + + return { + ok: true, + actor: { userId: session.user.id, name: session.user.name, email: session.user.email }, + } +} + +/** Reads a workspace sandbox list; any member may look, only admins may write. */ +export async function authorizeSandboxRead( + _request: NextRequest, + workspaceId: string +): Promise<{ ok: true; userId: string } | { ok: false; response: NextResponse }> { + const session = await getSession() + if (!session?.user?.id) { + return { ok: false, response: NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } + } + const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) + if (!permission) { + return { ok: false, response: NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } + } + return { ok: true, userId: session.user.id } +} diff --git a/apps/sim/app/api/workspaces/[id]/sandboxes/route.ts b/apps/sim/app/api/workspaces/[id]/sandboxes/route.ts new file mode 100644 index 00000000000..f39bc6b4215 --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/sandboxes/route.ts @@ -0,0 +1,101 @@ +import { db } from '@sim/db' +import { workspaceSandbox } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { type NextRequest, NextResponse } from 'next/server' +import { createSandboxContract } from '@/lib/api/contracts/sandboxes' +import { parseRequest } from '@/lib/api/server' +import { hasWorkspaceSandboxAccess } from '@/lib/billing/core/subscription' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + currentSandboxStrategy, + isSandboxNameTaken, + listWorkspaceSandboxes, + readWorkspaceSandbox, + scheduleSandboxBuild, +} from '@/lib/execution/remote-sandbox/workspace-sandboxes' +import { + authorizeSandboxMutation, + authorizeSandboxRead, + buildSpecOrResponse, + isNameConflictError, + nameConflictResponse, +} from '@/app/api/workspaces/[id]/sandboxes/authorize' + +const logger = createLogger('WorkspaceSandboxesAPI') + +export const GET = withRouteHandler( + async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { + const workspaceId = (await context.params).id + + const viewer = await authorizeSandboxRead(request, workspaceId) + if (!viewer.ok) return viewer.response + + // The list itself is not plan-gated: a workspace that downgraded must still + // see (and keep executing) what it already built. `entitled` drives whether + // the editor renders or an upgrade prompt does. + const [sandboxes, entitled] = await Promise.all([ + listWorkspaceSandboxes(workspaceId), + hasWorkspaceSandboxAccess(workspaceId), + ]) + + return NextResponse.json({ + sandboxes, + strategy: currentSandboxStrategy(), + entitled, + }) + } +) + +export const POST = withRouteHandler( + async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { + const workspaceId = (await context.params).id + + const authorized = await authorizeSandboxMutation(workspaceId) + if (!authorized.ok) return authorized.response + + const parsed = await parseRequest(createSandboxContract, request, context) + if (!parsed.success) return parsed.response + const { name, language, dependencies } = parsed.data.body + + const built = buildSpecOrResponse(language, dependencies) + if (!built.ok) return built.response + const { spec } = built + + if (await isSandboxNameTaken(workspaceId, name)) { + return nameConflictResponse(name) + } + + const id = generateId() + try { + await db.insert(workspaceSandbox).values({ + id, + workspaceId, + name, + language: spec.language, + dependencies: spec.dependencies, + specHash: spec.specHash, + createdBy: authorized.actor.userId, + }) + } catch (error) { + // The unique index is the real arbiter — the pre-check above only exists to + // return a friendlier message when there is no race. + if (isNameConflictError(error)) return nameConflictResponse(name) + logger.error('Failed to insert sandbox', { workspaceId, error: getErrorMessage(error) }) + throw error + } + + await scheduleSandboxBuild(spec) + logger.info('Created workspace sandbox', { workspaceId, sandboxId: id, language }) + + const sandbox = await readWorkspaceSandbox(workspaceId, id) + if (!sandbox) { + return NextResponse.json( + { error: 'Failed to read back the created sandbox' }, + { status: 500 } + ) + } + return NextResponse.json({ sandbox }) + } +) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index 934e4b07011..661bc325188 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -22,7 +22,6 @@ import { } from '@sim/emcn' import { ArrowLeft, Plus } from 'lucide-react' import { useParams } from 'next/navigation' -import { isBillingEnabled } from '@/lib/core/config/env-flags' import { consumeOAuthReturnContext } from '@/lib/credentials/client-state' import { getCanonicalScopesForProvider, @@ -79,7 +78,7 @@ export function AddConnectorModal({ const { ownerBilling } = useWorkspaceHostContext() const { mutate: createConnector, isPending: isCreating } = useCreateConnector() - const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling, isBillingEnabled) + const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling) const connectorConfig = selectedType ? CONNECTOR_META_REGISTRY[selectedType] : null const isApiKeyMode = connectorConfig?.auth.mode === 'apiKey' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.test.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.test.ts index be610b98877..beef4a6d514 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.test.ts @@ -1,7 +1,8 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it } from 'vitest' import type { WorkspaceOwnerBilling } from '@/lib/api/contracts/workspaces' import { hasWorkspaceMaxConnectorAccess } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements' @@ -19,52 +20,57 @@ const HOST_MAX_BILLING: WorkspaceOwnerBilling = { billingBlockedReason: null, } +const FREE_BILLING: WorkspaceOwnerBilling = { + ...HOST_MAX_BILLING, + plan: 'free', + status: null, + isPaid: false, + isTeam: false, + isOrgScoped: false, + organizationId: null, +} + +afterAll(resetEnvFlagsMock) + describe('hasWorkspaceMaxConnectorAccess', () => { + beforeEach(() => { + resetEnvFlagsMock() + setEnvFlags({ isHosted: true, isBillingEnabled: true }) + }) + it('uses the workspace host Max entitlement', () => { - expect(hasWorkspaceMaxConnectorAccess(HOST_MAX_BILLING, true)).toBe(true) + expect(hasWorkspaceMaxConnectorAccess(HOST_MAX_BILLING)).toBe(true) }) it('does not unlock live sync from a free host plan', () => { - expect( - hasWorkspaceMaxConnectorAccess( - { - ...HOST_MAX_BILLING, - plan: 'free', - status: null, - isPaid: false, - isTeam: false, - isOrgScoped: false, - organizationId: null, - }, - true - ) - ).toBe(false) + expect(hasWorkspaceMaxConnectorAccess(FREE_BILLING)).toBe(false) }) it('does not unlock live sync for a blocked Max payer', () => { expect( - hasWorkspaceMaxConnectorAccess( - { - ...HOST_MAX_BILLING, - billingBlocked: true, - billingBlockedReason: 'payment_failed', - }, - true - ) + hasWorkspaceMaxConnectorAccess({ + ...HOST_MAX_BILLING, + billingBlocked: true, + billingBlockedReason: 'payment_failed', + }) ).toBe(false) }) it('keeps connector intervals available when billing is disabled', () => { - expect( - hasWorkspaceMaxConnectorAccess( - { - ...HOST_MAX_BILLING, - plan: 'free', - status: null, - isPaid: false, - }, - false - ) - ).toBe(true) + setEnvFlags({ isBillingEnabled: false }) + + expect(hasWorkspaceMaxConnectorAccess(FREE_BILLING)).toBe(true) + }) + + /** + * The server gate is `!isHosted || !isBillingEnabled` — sub-hourly sync is + * ungated off the hosted deployment. This client helper omitted the `isHosted` + * branch, so a self-hosted operator with billing enabled saw "Live" locked while + * the API would have accepted it. + */ + it('keeps connector intervals available off the hosted deployment even with billing enabled', () => { + setEnvFlags({ isHosted: false, isBillingEnabled: true }) + + expect(hasWorkspaceMaxConnectorAccess(FREE_BILLING)).toBe(true) }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.ts index a25769cf001..dba607958d1 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements.ts @@ -1,10 +1,16 @@ import type { WorkspaceOwnerBilling } from '@/lib/api/contracts/workspaces' import { getSubscriptionAccessState } from '@/lib/billing/client' +import { isBillingEnabled, isHosted } from '@/lib/core/config/env-flags' -export function hasWorkspaceMaxConnectorAccess( - ownerBilling: WorkspaceOwnerBilling, - billingEnabled: boolean -): boolean { - if (!billingEnabled) return true +/** + * Client mirror of `hasWorkspaceLiveSyncAccess`. + * + * Reads the same two env flags in the same order as the server helper so the two + * cannot diverge: sub-hourly sync is ungated off the hosted deployment even when + * billing is enabled. Without the `isHosted` branch a self-hosted operator with + * billing on saw the "Live" interval locked while the API would have accepted it. + */ +export function hasWorkspaceMaxConnectorAccess(ownerBilling: WorkspaceOwnerBilling): boolean { + if (!isHosted || !isBillingEnabled) return true return getSubscriptionAccessState(ownerBilling).hasUsableMaxAccess } diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index 943f975cbe5..b916a31b232 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -17,7 +17,6 @@ import { } from '@sim/emcn' import { createLogger } from '@sim/logger' import { ExternalLink, RotateCcw } from 'lucide-react' -import { isBillingEnabled } from '@/lib/core/config/env-flags' import { ConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields' import { hasWorkspaceMaxConnectorAccess } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements' import { SYNC_INTERVALS } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' @@ -194,7 +193,7 @@ export function EditConnectorModal({ const { ownerBilling } = useWorkspaceHostContext() const { mutate: updateConnector, isPending: isSaving } = useUpdateConnector() - const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling, isBillingEnabled) + const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling) const persistedCanonicalModes = useMemo( () => readPersistedCanonicalModes(connector.sourceConfig), diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx index 6303933ab1c..fee9364735d 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx @@ -11,7 +11,7 @@ import { } from '@/components/settings/navigation' import { getSession } from '@/lib/auth' import { isOrganizationOnEnterprisePlan } from '@/lib/billing' -import { hasWorkspaceInboxAccess } from '@/lib/billing/core/subscription' +import { hasWorkspaceInboxAccess, hasWorkspaceSandboxAccess } from '@/lib/billing/core/subscription' import { getEnv, isTruthy } from '@/lib/core/config/env' import { isBillingEnabled, isHosted } from '@/lib/core/config/env-flags' import { canOpenOrganizationSettingsSection } from '@/lib/organizations/settings-access' @@ -49,6 +49,7 @@ const WORKSPACE_SECTION_MAP: Partial item.id === workspaceSection)) notFound() diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx index b594b4c1e04..b9a95f6564c 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx @@ -29,6 +29,11 @@ const Forks = dynamic(() => import('@/ee/workspace-forking/components/forks').th const Secrets = dynamic(() => import('@/app/workspace/[workspaceId]/settings/components/secrets/secrets').then((m) => m.Secrets) ) +const Sandboxes = dynamic(() => + import('@/app/workspace/[workspaceId]/settings/components/sandboxes/sandboxes').then( + (m) => m.Sandboxes + ) +) const CustomTools = dynamic(() => import('@/app/workspace/[workspaceId]/settings/components/custom-tools/custom-tools').then( (m) => m.CustomTools @@ -187,6 +192,7 @@ export function SettingsPage({ section }: SettingsPageProps) { )} {effectiveSection === 'byok' && } + {effectiveSection === 'sandboxes' && } {effectiveSection === 'mcp' && } {effectiveSection === 'forks' && } {effectiveSection === 'custom-tools' && } diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/inbox/inbox.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/inbox/inbox.tsx index b980b83c048..e1ff2ee7904 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/inbox/inbox.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/inbox/inbox.tsx @@ -1,7 +1,5 @@ 'use client' -import { Chip } from '@sim/emcn' -import { ArrowRight } from 'lucide-react' import { useParams } from 'next/navigation' import { canMutateWorkspaceSettingsSection } from '@/components/settings/navigation' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' @@ -12,12 +10,11 @@ import { } from '@/app/workspace/[workspaceId]/settings/components/inbox/components' import { SettingsPanel } from '@/app/workspace/[workspaceId]/settings/components/settings-panel' import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' +import { SettingsUpgradeNotice } from '@/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice' import { useInboxConfig } from '@/hooks/queries/inbox' -import { useSettingsNavigation } from '@/hooks/use-settings-navigation' export function Inbox() { const params = useParams() - const { navigateToSettings } = useSettingsNavigation() const workspaceId = params.workspaceId as string const { data: config, isLoading } = useInboxConfig(workspaceId) @@ -38,26 +35,11 @@ export function Inbox() { } return ( -
-
-

- Sim Mailer requires an active Max plan -

-

- Upgrade to Max and ensure billing is active to receive tasks via email and let Sim - work on your behalf. -

-
- {canAdmin && ( - navigateToSettings({ section: 'billing' })} - > - Upgrade to Max - - )} -
+ ) } diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-create-modal.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-create-modal.tsx new file mode 100644 index 00000000000..059bf95fd95 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-create-modal.tsx @@ -0,0 +1,201 @@ +'use client' + +import { useState } from 'react' +import { + ChipDropdown, + ChipModal, + ChipModalBody, + ChipModalError, + ChipModalField, + ChipModalFooter, + ChipModalHeader, +} from '@sim/emcn' +import { getErrorMessage } from '@sim/utils/errors' +import { useParams } from 'next/navigation' +import { canMutateWorkspaceSettingsSection } from '@/components/settings/navigation' +import type { SandboxDependencyIssue } from '@/lib/api/contracts/sandboxes' +import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +import { + DEPENDENCY_PLACEHOLDERS, + emptyDraft, + extractIssues, + LANGUAGE_OPTIONS, + SANDBOX_UPGRADE_DESCRIPTION, + SANDBOX_UPGRADE_TITLE, + type SandboxDraft, + type SandboxLanguage, + toSubmittedLines, +} from '@/app/workspace/[workspaceId]/settings/components/sandboxes/utils' +import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' +import { SettingsUpgradeNotice } from '@/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice' +import { type Sandbox, useCreateSandbox, useSandboxes } from '@/hooks/queries/sandboxes' + +interface SandboxCreateModalProps { + open: boolean + onOpenChange: (open: boolean) => void + /** + * Seeds the language of the new sandbox. Pickers scope their list to one + * language, so without this a sandbox created from a JavaScript block could + * land in the Python list and never appear. + */ + defaultLanguage?: SandboxLanguage + /** Receives the created sandbox so the caller can select it. */ + onCreated?: (sandbox: Sandbox) => void +} + +/** + * Creates a sandbox from wherever one is being picked, so authoring a package + * list never means leaving the workflow for Settings. + * + * Editing stays in Settings — this is deliberately create-only, matching the + * "Create Skill" / "Create Tool" rows it sits beside. + */ +export function SandboxCreateModal({ + open, + onOpenChange, + defaultLanguage, + onCreated, +}: SandboxCreateModalProps) { + const params = useParams() + const workspaceId = params.workspaceId as string + + // Keyed off `open` so the list is not fetched by every mounted picker, only by + // one the user actually opened. It shares the picker's cache entry either way. + const { data } = useSandboxes(open ? workspaceId : undefined) + const permissions = useUserPermissionsContext() + const canAdmin = canMutateWorkspaceSettingsSection('sandboxes', permissions) + // Assume entitled until the list answers — a flash of the upgrade copy on a + // workspace that has it would be worse than a late, accurate one. + const entitled = data?.entitled ?? true + + const createSandbox = useCreateSandbox() + + const [draft, setDraft] = useState(emptyDraft) + const [issues, setIssues] = useState([]) + const [error, setError] = useState(null) + + // The modal stays mounted for its exit animation, so each opening has to clear + // what the last one left behind. + const [wasOpen, setWasOpen] = useState(open) + if (wasOpen !== open) { + setWasOpen(open) + if (open) { + setDraft({ ...emptyDraft(), ...(defaultLanguage ? { language: defaultLanguage } : {}) }) + setIssues([]) + setError(null) + } + } + + // A gate answers the whole dialog rather than reddening a form the user can + // never submit — the same call the settings page makes. + const gate = !entitled ? 'plan' : !canAdmin ? 'permission' : null + const saving = createSandbox.isPending + + const handleCreate = async () => { + setIssues([]) + setError(null) + try { + const { sandbox } = await createSandbox.mutateAsync({ + workspaceId, + name: draft.name.trim(), + language: draft.language, + dependencies: toSubmittedLines(draft.dependencies), + }) + onCreated?.(sandbox) + onOpenChange(false) + } catch (caught) { + const lineIssues = extractIssues(caught) + if (lineIssues.length > 0) { + setIssues(lineIssues) + return + } + setError(getErrorMessage(caught, 'Failed to create sandbox')) + } + } + + return ( + + onOpenChange(false)}>Create sandbox + + + {gate === 'plan' ? ( + + ) : gate === 'permission' ? ( + + Only workspace admins can create sandboxes. + + ) : ( + <> + setDraft((prev) => ({ ...prev, name }))} + placeholder='bigquery-etl' + maxLength={64} + autoComplete='off' + required + disabled={saving} + /> + + + + setDraft((prev) => ({ ...prev, language: language as SandboxLanguage })) + } + options={LANGUAGE_OPTIONS.map((option) => ({ + label: option.label, + value: option.value, + }))} + disabled={saving} + aria-label='Language' + /> + + + setDraft((prev) => ({ ...prev, dependencies }))} + placeholder={DEPENDENCY_PLACEHOLDERS[draft.language]} + rows={8} + disabled={saving} + hint='One per line. Version pins are optional.' + error={ + issues.length > 0 ? ( + <> + {issues.map((issue) => ( + + Line {issue.line}: {issue.reason} + + ))} + + ) : undefined + } + /> + + {error} + + )} + + + {gate === null && ( + onOpenChange(false)} + cancelDisabled={saving} + primaryAction={{ + label: saving ? 'Creating...' : 'Create', + onClick: () => void handleCreate(), + disabled: saving || draft.name.trim().length === 0, + }} + /> + )} + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-editor.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-editor.tsx new file mode 100644 index 00000000000..c6887f23277 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-editor.tsx @@ -0,0 +1,177 @@ +'use client' + +import { useMemo, useState } from 'react' +import { Chip, ChipDropdown, ChipInput, ChipTextarea, cn } from '@sim/emcn' +import type { SandboxDependencyIssue } from '@/lib/api/contracts/sandboxes' +import { + DEPENDENCY_PLACEHOLDERS, + LANGUAGE_OPTIONS, + type SandboxDraft, + type SandboxLanguage, +} from '@/app/workspace/[workspaceId]/settings/components/sandboxes/utils' +import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' +import type { Sandbox } from '@/hooks/queries/sandboxes' + +interface SandboxEditorProps { + draft: SandboxDraft + onChange: (draft: SandboxDraft) => void + /** Server-reported bad lines, addressed to the row the user typed them on. */ + issues: SandboxDependencyIssue[] + disabled?: boolean + /** Build status row. Absent for an unsaved sandbox and under the runtime strategy. */ + status?: React.ReactNode +} + +export function SandboxEditor({ + draft, + onChange, + issues, + disabled = false, + status, +}: SandboxEditorProps) { + const issuesByLine = useMemo(() => { + const byLine = new Map() + for (const issue of issues) { + if (!byLine.has(issue.line)) byLine.set(issue.line, issue) + } + return byLine + }, [issues]) + + return ( +
+ +
+
+ Name + onChange({ ...draft, name: event.target.value })} + placeholder='bigquery-etl' + disabled={disabled} + maxLength={64} + autoComplete='off' + /> +
+
+ Language + onChange({ ...draft, language: language as SandboxLanguage })} + options={LANGUAGE_OPTIONS.map((option) => ({ + label: option.label, + value: option.value, + }))} + disabled={disabled} + /> +
+
+ + + +
+ onChange({ ...draft, dependencies: event.target.value })} + placeholder={DEPENDENCY_PLACEHOLDERS[draft.language]} + rows={8} + disabled={disabled} + error={issues.length > 0} + spellCheck={false} + autoComplete='off' + /> +

+ One per line. Version pins are optional. +

+ {issues.length > 0 && ( +
    + {[...issuesByLine.values()].map((issue) => ( +
  • + Line {issue.line}: {issue.reason} +
    • + ))} +
+ )} +
+ + + {status && {status}} +
+ ) +} + +const STATUS_LABEL: Record = { + ready: 'Ready', + failed: 'Failed', + building: 'Building', + pending: 'Queued', +} + +interface SandboxStatusProps { + sandbox: Sandbox + /** Runtime-strategy deployments have no build to report. */ + strategy: 'prebuilt' | 'runtime' +} + +export function SandboxStatus({ sandbox, strategy }: SandboxStatusProps) { + const [showLog, setShowLog] = useState(false) + + if (strategy === 'runtime') { + return ( +

+ Dependencies install at run time on this deployment, adding roughly 10–30s per execution. + Prebuilt sandboxes require E2B. +

+ ) + } + + const packageCount = sandbox.dependencies.length + + // A sandbox with no packages declares nothing to build, so it has no registry + // row — reporting that as "Queued" would describe a build that will never come. + if (packageCount === 0) { + return ( +

+ No packages yet. Add dependencies above to build this sandbox; until then it runs on the + default image. +

+ ) + } + + const status = sandbox.buildStatus ?? 'pending' + + return ( +
+
+ + {STATUS_LABEL[status]} + + + · {packageCount} {packageCount === 1 ? 'package' : 'packages'} + +
+ + {status === 'failed' && sandbox.errorMessage && ( +
+

{sandbox.errorMessage}

+ {sandbox.errorDetail && ( + <> + setShowLog((open) => !open)}> + {showLog ? 'Hide log' : 'Show log'} + + {showLog && ( + 
+                  {sandbox.errorDetail}
+
+ )} + + )} +
+ )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/index.ts b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/index.ts new file mode 100644 index 00000000000..51d9aee1e22 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/index.ts @@ -0,0 +1 @@ +export { Sandboxes } from '@/app/workspace/[workspaceId]/settings/components/sandboxes/sandboxes' diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/sandboxes.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/sandboxes.tsx new file mode 100644 index 00000000000..a1ad58c0872 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/sandboxes.tsx @@ -0,0 +1,303 @@ +'use client' + +import { useCallback, useMemo, useState } from 'react' +import { toast } from '@sim/emcn' +import { ArrowLeft, Plus } from '@sim/emcn/icons' +import { getErrorMessage } from '@sim/utils/errors' +import { useParams } from 'next/navigation' +import { useQueryState } from 'nuqs' +import { CodeIcon } from '@/components/icons' +import { canMutateWorkspaceSettingsSection } from '@/components/settings/navigation' +import type { SandboxDependencyIssue } from '@/lib/api/contracts/sandboxes' +import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail' +import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +import { RowActionsMenu } from '@/app/workspace/[workspaceId]/settings/components/row-actions-menu' +import { + SandboxEditor, + SandboxStatus, +} from '@/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-editor' +import { + sandboxIdParam, + sandboxIdUrlKeys, +} from '@/app/workspace/[workspaceId]/settings/components/sandboxes/search-params' +import { + draftFromSandbox, + emptyDraft, + extractIssues, + SANDBOX_UPGRADE_DESCRIPTION, + SANDBOX_UPGRADE_TITLE, + type SandboxDraft, + toSubmittedLines, +} from '@/app/workspace/[workspaceId]/settings/components/sandboxes/utils' +import { saveDiscardActions } from '@/app/workspace/[workspaceId]/settings/components/save-discard-actions/save-discard-actions' +import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' +import { SettingsPanel } from '@/app/workspace/[workspaceId]/settings/components/settings-panel' +import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' +import { SettingsUpgradeNotice } from '@/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice' +import { useSettingsSearch } from '@/app/workspace/[workspaceId]/settings/components/use-settings-search' +import { useSettingsUnsavedGuard } from '@/app/workspace/[workspaceId]/settings/hooks/use-settings-unsaved-guard' +import { + type Sandbox, + useCreateSandbox, + useDeleteSandbox, + useSandboxes, + useUpdateSandbox, +} from '@/hooks/queries/sandboxes' + +export function Sandboxes() { + const params = useParams() + const workspaceId = params.workspaceId as string + + const [searchTerm, setSearchTerm] = useSettingsSearch() + const [selectedId, setSelectedId] = useQueryState(sandboxIdParam.key, { + ...sandboxIdParam.parser, + ...sandboxIdUrlKeys, + }) + + const { data, isLoading } = useSandboxes(workspaceId) + const createSandbox = useCreateSandbox() + const updateSandbox = useUpdateSandbox() + const deleteSandbox = useDeleteSandbox() + + const permissions = useUserPermissionsContext() + const canAdmin = canMutateWorkspaceSettingsSection('sandboxes', permissions) + + const [draft, setDraft] = useState(null) + const [issues, setIssues] = useState([]) + const [isCreating, setIsCreating] = useState(false) + + // The draft belongs to whatever was open when it was typed. Browser Back + // clears `selectedId` without going through `closeEditor`, so without this the + // next sandbox opened would render — and save — the previous one's edits. + const [draftOwnerId, setDraftOwnerId] = useState(null) + if (draftOwnerId !== selectedId) { + setDraftOwnerId(selectedId) + if (draft) { + setDraft(null) + setIssues([]) + } + // Creating and having one open are mutually exclusive, and history can land on + // a sandbox while create mode is still set — Forward after starting a new one. + // Leaving both on renders an empty "New sandbox" form whose Delete still points + // at the restored sandbox. + if (selectedId) setIsCreating(false) + } + + const sandboxes = data?.sandboxes ?? [] + const strategy = data?.strategy ?? 'prebuilt' + const entitled = data?.entitled ?? false + + // Derived, never duplicated into state: a stale id from an old link resolves to + // nothing and simply falls back to the list. + const selected = selectedId ? (sandboxes.find((s) => s.id === selectedId) ?? null) : null + const baseline = isCreating ? null : selected + const original = useMemo(() => (baseline ? draftFromSandbox(baseline) : emptyDraft()), [baseline]) + const current = draft ?? original + const isEditing = isCreating || Boolean(selected) + const isDirty = + isEditing && + (isCreating + ? current.name.trim().length > 0 || current.dependencies.trim().length > 0 + : current.name !== original.name || + current.language !== original.language || + current.dependencies !== original.dependencies) + + // Called before every early return — a hook after a gate is skipped on gated renders. + const guard = useSettingsUnsavedGuard({ isDirty }) + + const closeEditor = useCallback(() => { + setDraft(null) + setIssues([]) + setIsCreating(false) + // Opening pushed a history entry; closing must not push another. + void setSelectedId(null, { history: 'replace' }) + }, [setSelectedId]) + + const handleSave = useCallback(async () => { + setIssues([]) + const body = { + name: current.name.trim(), + language: current.language, + dependencies: toSubmittedLines(current.dependencies), + } + try { + if (isCreating) { + await createSandbox.mutateAsync({ workspaceId, ...body }) + } else if (selected) { + await updateSandbox.mutateAsync({ workspaceId, sandboxId: selected.id, ...body }) + } + closeEditor() + } catch (error) { + const lineIssues = extractIssues(error) + if (lineIssues.length > 0) { + setIssues(lineIssues) + return + } + toast.error(getErrorMessage(error, 'Failed to save sandbox')) + } + }, [current, isCreating, selected, workspaceId, createSandbox, updateSandbox, closeEditor]) + + const handleDelete = useCallback( + async (sandbox: Sandbox) => { + try { + await deleteSandbox.mutateAsync({ workspaceId, sandboxId: sandbox.id }) + if (selectedId === sandbox.id) closeEditor() + } catch (error) { + toast.error(getErrorMessage(error, 'Failed to delete sandbox')) + } + }, + [deleteSandbox, workspaceId, selectedId, closeEditor] + ) + + const filtered = useMemo(() => { + const query = searchTerm.trim().toLowerCase() + if (!query) return sandboxes + return sandboxes.filter( + (sandbox) => + sandbox.name.toLowerCase().includes(query) || + sandbox.dependencies.some((dependency) => dependency.toLowerCase().includes(query)) + ) + }, [sandboxes, searchTerm]) + + if (isLoading) { + return ( + + Loading sandboxes... + + ) + } + + if (!entitled) { + return ( + + + + ) + } + + if (isEditing) { + const saving = createSandbox.isPending || updateSandbox.isPending + return ( + <> + guard.guardBack(closeEditor), + }} + actions={[ + ...saveDiscardActions({ + dirty: isDirty, + saving, + onSave: () => void handleSave(), + onDiscard: () => { + setDraft(null) + setIssues([]) + }, + saveDisabled: !canAdmin || current.name.trim().length === 0, + }), + ...(selected && canAdmin + ? [ + { + text: 'Delete', + textTone: 'error' as const, + onSelect: () => void handleDelete(selected), + disabled: deleteSandbox.isPending, + }, + ] + : []), + ]} + > + : undefined} + /> + + + + + ) + } + + return ( + { + setDraft(emptyDraft()) + setIsCreating(true) + // Starting a new one is not editing the open one. Replace rather + // than push: this is a mode switch, not a destination. + void setSelectedId(null, { history: 'replace' }) + }, + }, + ] + : [] + } + > + + {filtered.length === 0 ? ( + + {searchTerm + ? 'No sandboxes match your search.' + : 'No sandboxes yet. Create one to let Function blocks import packages.'} + + ) : ( +
+ {filtered.map((sandbox) => ( + } + title={ + + } + description={`${sandbox.language === 'python' ? 'Python' : 'JavaScript'} · ${sandbox.dependencies.length} ${sandbox.dependencies.length === 1 ? 'package' : 'packages'}`} + trailing={ + canAdmin ? ( + void setSelectedId(sandbox.id) }, + { + label: 'Delete', + destructive: true, + onSelect: () => void handleDelete(sandbox), + }, + ]} + /> + ) : undefined + } + /> + ))} +
+ )} + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/search-params.ts b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/search-params.ts new file mode 100644 index 00000000000..a2e2b8cb876 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/search-params.ts @@ -0,0 +1,17 @@ +import { parseAsString } from 'nuqs/server' + +/** + * The sandbox open in the editor. Only the id lives in the URL; the object is + * derived from the already-loaded list, so a stale id from an old link simply + * falls back to the list rather than rendering a broken detail view. + */ +export const sandboxIdParam = { + key: 'sandboxId', + parser: parseAsString, +} as const + +/** Opening a sandbox is a destination — Back should close it. */ +export const sandboxIdUrlKeys = { + history: 'push', + clearOnDefault: true, +} as const diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.test.ts b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.test.ts new file mode 100644 index 00000000000..41c3fae6cfa --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.test.ts @@ -0,0 +1,57 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + emptyDraft, + extractIssues, + LANGUAGE_OPTIONS, + toSubmittedLines, +} from '@/app/workspace/[workspaceId]/settings/components/sandboxes/utils' +import { FunctionBlock } from '@/blocks/blocks/function' + +const languageField = FunctionBlock.subBlocks.find((subBlock) => subBlock.id === 'language') +const sandboxField = FunctionBlock.subBlocks.find((subBlock) => subBlock.id === 'sandboxId') + +describe('sandbox draft defaults', () => { + it('starts a new sandbox in the language the Function block itself defaults to', () => { + const blockDefault = typeof languageField?.value === 'function' ? languageField.value() : null + expect(blockDefault).toBe('javascript') + expect(emptyDraft().language).toBe(blockDefault) + }) + + it('offers languages in the Function block dropdown order', () => { + expect(LANGUAGE_OPTIONS.map((option) => option.value)).toEqual( + languageField?.options?.map((option) => (typeof option === 'string' ? option : option.id)) + ) + }) + + it('starts empty so nothing is submitted by accident', () => { + expect(emptyDraft()).toEqual({ name: '', language: 'javascript', dependencies: '' }) + }) +}) + +describe('sandbox picker create action', () => { + it('declares the inline create row the picker renders', () => { + expect(sandboxField?.createAction).toBe('sandbox') + }) +}) + +describe('toSubmittedLines', () => { + it('keeps blank rows so a rejection can address the line the user typed on', () => { + expect(toSubmittedLines('axios\n\nzod')).toEqual(['axios', '', 'zod']) + }) +}) + +describe('extractIssues', () => { + it('reads the per-line rejections off a failed save', () => { + const error = { body: { issues: [{ line: 2, reason: 'not a package name' }] } } + expect(extractIssues(error)).toEqual([{ line: 2, reason: 'not a package name' }]) + }) + + it('returns nothing for an error that carries no issues', () => { + expect(extractIssues(new Error('network'))).toEqual([]) + expect(extractIssues({ body: { issues: 'nope' } })).toEqual([]) + expect(extractIssues(undefined)).toEqual([]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.ts b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.ts new file mode 100644 index 00000000000..ae5c909621e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/sandboxes/utils.ts @@ -0,0 +1,58 @@ +import type { Sandbox, SandboxDependencyIssue } from '@/lib/api/contracts/sandboxes' + +export type SandboxLanguage = Sandbox['language'] + +/** Shared by the settings page and the picker's create modal so the wall reads identically. */ +export const SANDBOX_UPGRADE_TITLE = 'Sandboxes require an active Max plan' +export const SANDBOX_UPGRADE_DESCRIPTION = + 'Upgrade to Max and ensure billing is active to install Python or npm packages that your Function blocks can import.' + +/** Ordered to match the Function block's own `language` dropdown. */ +export const LANGUAGE_OPTIONS = [ + { label: 'JavaScript', value: 'javascript' }, + { label: 'Python', value: 'python' }, +] as const + +/** + * Placeholders double as the format documentation, so they swap with the + * language rather than describing one syntax for both. + */ +export const DEPENDENCY_PLACEHOLDERS: Record = { + python: 'google-cloud-bigquery==3.25.0\npyairtable>=3.0\npandas', + javascript: 'axios@^1.7.0\n@aws-sdk/client-s3\nzod', +} + +export interface SandboxDraft { + name: string + language: SandboxLanguage + /** Raw textarea contents — one dependency per line, comments allowed. */ + dependencies: string +} + +export function draftFromSandbox(sandbox: Sandbox): SandboxDraft { + return { + name: sandbox.name, + language: sandbox.language, + dependencies: sandbox.dependencies.join('\n'), + } +} + +/** Defaults to the Function block's own default language so the two agree. */ +export function emptyDraft(): SandboxDraft { + return { name: '', language: 'javascript', dependencies: '' } +} + +/** Splits the textarea into one entry per row so a rejection keeps its line number. */ +export function toSubmittedLines(dependencies: string): string[] { + return dependencies.split('\n') +} + +/** + * Pulls the per-line rejections off a failed save. The server addresses each one + * to the row the user typed it on, so they are surfaced against the textarea + * rather than as a single opaque error. + */ +export function extractIssues(error: unknown): SandboxDependencyIssue[] { + const issues = (error as { body?: { issues?: SandboxDependencyIssue[] } })?.body?.issues + return Array.isArray(issues) ? issues : [] +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/index.ts b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/index.ts new file mode 100644 index 00000000000..89c77a89ac1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/index.ts @@ -0,0 +1 @@ +export { SettingsUpgradeNotice } from '@/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/settings-upgrade-notice' diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/settings-upgrade-notice.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/settings-upgrade-notice.tsx new file mode 100644 index 00000000000..591e1bd8efb --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-upgrade-notice/settings-upgrade-notice.tsx @@ -0,0 +1,58 @@ +'use client' + +import { Chip, cn } from '@sim/emcn' +import { ArrowRight } from '@sim/emcn/icons' +import { useSettingsNavigation } from '@/hooks/use-settings-navigation' + +interface SettingsUpgradeNoticeProps { + /** Names the gated surface, e.g. `Sandboxes require an active Max plan`. */ + title: string + /** One sentence on what the plan unlocks. */ + description: string + /** + * Whether to offer the upgrade action. Members who cannot act on it are shown + * the reason without a button that would only dead-end them. + */ + canUpgrade?: boolean + /** + * Tightens the vertical rhythm for a modal, where the full-height centering a + * settings page wants would leave the dialog mostly empty. + */ + compact?: boolean +} + +/** + * Canonical wall for a surface gated behind the Max plan. Owns the copy rhythm + * and the route to upgrade, so every gated section reads and behaves the same. + * + * The action lands on billing, which redirects a member who cannot manage + * billing to the plan-comparison page instead — so it is never a dead end. + */ +export function SettingsUpgradeNotice({ + title, + description, + canUpgrade = false, + compact = false, +}: SettingsUpgradeNoticeProps) { + const { navigateToSettings } = useSettingsNavigation() + + return ( +
+
+

{title}

+

{description}

+
+ {canUpgrade && ( + navigateToSettings({ section: 'billing' })} + > + Upgrade to Max + + )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts index d24163419a8..5c2a5bfe3e8 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts @@ -1,4 +1,16 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' + +/** + * The Sandboxes section is dropped when no sandbox provider is configured, and + * `allNavigationItems` is built once at module load — so this has to be set before + * the module graph is imported, where a `beforeEach` would run too late. Pinning it + * also keeps the catalog assertions off the developer's untracked `apps/sim/.env`, + * which CI does not have. + */ +vi.hoisted(() => { + process.env.NEXT_PUBLIC_SANDBOX_ENABLED = 'true' +}) + import { SETTINGS_SECTION_REGISTRY, WORKSPACE_SETTINGS_ITEMS, @@ -39,6 +51,7 @@ describe('unified settings navigation', () => { { id: 'apikeys', label: 'Sim API keys', section: 'system' }, { id: 'workflow-mcp-servers', label: 'MCP servers', section: 'system' }, { id: 'byok', label: 'BYOK', section: 'system' }, + { id: 'sandboxes', label: 'Sandboxes', section: 'system' }, { id: 'inbox', label: 'Sim mailer', section: 'system' }, { id: 'recently-deleted', label: 'Recently deleted', section: 'system' }, { id: 'sso', label: 'Single sign-on', section: 'enterprise' }, diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/code/code.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/code/code.tsx index bc3972c4b94..a0d18f4d63b 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/code/code.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/code/code.tsx @@ -321,12 +321,14 @@ export const Code = memo(function Code({ }, [wandConfig, languageValue]) const [tableIdValue] = useSubBlockValue(blockId, 'tableId') + const [sandboxIdValue] = useSubBlockValue(blockId, 'sandboxId') const wandHook = useWand({ wandConfig: dynamicWandConfig || { enabled: false, prompt: '' }, currentValue: code, contextParams: { tableId: typeof tableIdValue === 'string' ? tableIdValue : null, + sandboxId: typeof sandboxIdValue === 'string' ? sandboxIdValue : null, }, onStreamStart: () => handleStreamStartRef.current?.(), onStreamChunk: (chunk: string) => handleStreamChunkRef.current?.(chunk), diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/combobox/combobox.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/combobox/combobox.tsx index b2856d10013..a2442070369 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/combobox/combobox.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/combobox/combobox.tsx @@ -1,24 +1,21 @@ import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Combobox, type ComboboxOption, cn } from '@sim/emcn' -import { getErrorMessage } from '@sim/utils/errors' -import { isEqual } from 'es-toolkit' +import { Plus } from '@sim/emcn/icons' import { useReactFlow } from 'reactflow' -import { useStoreWithEqualityFn } from 'zustand/traditional' -import { buildCanonicalIndex, resolveDependencyValue } from '@/lib/workflows/subblocks/visibility' +import { SandboxCreateModal } from '@/app/workspace/[workspaceId]/settings/components/sandboxes/components/sandbox-create-modal' +import type { SandboxLanguage } from '@/app/workspace/[workspaceId]/settings/components/sandboxes/utils' import { formatDisplayText } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/formatted-text' import { SubBlockInputController } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sub-block-input-controller' import { getWorkflowSearchLabelHighlight } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/workflow-search-highlight' +import { useFetchedOptions } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-fetched-options' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' import { useActiveSearchTarget } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/providers/active-search-target-provider' import { useAccessibleReferencePrefixes } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-accessible-reference-prefixes' -import { getBlock } from '@/blocks/registry' import type { SubBlockConfig } from '@/blocks/types' import { getDependsOnFields } from '@/blocks/utils' import { usePermissionConfig } from '@/hooks/use-permission-config' import { getProviderFromModel } from '@/providers/utils' -import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' -import { useWorkflowStore } from '@/stores/workflows/workflow/store' /** * Constants for ComboBox component behavior @@ -29,6 +26,17 @@ const MIN_ZOOM = 0.1 const MAX_ZOOM = 1 const ZOOM_DURATION = 0 +const CREATE_ACTION_LABEL: Record, string> = { + sandbox: 'Create Sandbox', +} + +/** + * Reserved value for the pinned create row. It can never collide with a stored + * value: emcn short-circuits on the option's `onSelect`, so the row never + * reaches `onChange`. + */ +const CREATE_ACTION_VALUE = '__sub-block-create-action__' + /** * Represents a selectable option in the combobox */ @@ -94,56 +102,6 @@ export const ComboBox = memo(function ComboBox({ // Dependency tracking for fetchOptions const dependsOnFields = useMemo(() => getDependsOnFields(dependsOn), [dependsOn]) - const activeWorkflowId = useWorkflowRegistry((s) => s.activeWorkflowId) - const blockState = useWorkflowStore((state) => state.blocks[blockId]) - const blockConfig = blockState?.type ? getBlock(blockState.type) : null - const canonicalIndex = useMemo( - () => buildCanonicalIndex(blockConfig?.subBlocks || []), - [blockConfig?.subBlocks] - ) - const canonicalModeOverrides = blockState?.data?.canonicalModes - const dependencyValues = useStoreWithEqualityFn( - useSubBlockStore, - useCallback( - (state) => { - if (dependsOnFields.length === 0 || !activeWorkflowId) return [] - const workflowValues = state.workflowValues[activeWorkflowId] || {} - const blockValues = workflowValues[blockId] || {} - return dependsOnFields.map((depKey) => - resolveDependencyValue(depKey, blockValues, canonicalIndex, canonicalModeOverrides) - ) - }, - [dependsOnFields, activeWorkflowId, blockId, canonicalIndex, canonicalModeOverrides] - ), - isEqual - ) - - // State management - const [fetchedOptions, setFetchedOptions] = useState>([]) - const [isLoadingOptions, setIsLoadingOptions] = useState(false) - const [fetchError, setFetchError] = useState(null) - const [hydratedOption, setHydratedOption] = useState<{ label: string; id: string } | null>(null) - const previousDependencyValuesRef = useRef('') - - /** - * Fetches options from the async fetchOptions function if provided - */ - const fetchOptionsIfNeeded = useCallback(async () => { - if (!fetchOptions || isPreview || disabled) return - - setIsLoadingOptions(true) - setFetchError(null) - try { - const options = await fetchOptions(blockId) - setFetchedOptions(options) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to fetch options') - setFetchError(errorMessage) - setFetchedOptions([]) - } finally { - setIsLoadingOptions(false) - } - }, [fetchOptions, blockId, isPreview, disabled]) // Determine the active value based on mode (preview vs. controlled vs. store) const value = isPreview ? previewValue : propValue !== undefined ? propValue : storeValue @@ -174,6 +132,47 @@ export const ComboBox = memo(function ComboBox({ return opts }, [options, subBlockId, isProviderAllowed, isModelAllowed]) + const { + fetchedOptions, + isLoadingOptions, + fetchError, + hydratedOption, + refetch: refetchOptions, + } = useFetchedOptions({ + blockId, + dependsOnFields, + fetchOptions, + fetchOptionById, + isPreview: Boolean(isPreview), + disabled: Boolean(disabled), + valueToHydrate: value as string | null | undefined, + localOptions: staticOptions, + }) + + const [isCreateOpen, setIsCreateOpen] = useState(false) + const [createLanguage, setCreateLanguage] = useState(undefined) + const [createdOption, setCreatedOption] = useState<{ label: string; id: string } | null>(null) + + /** + * The pinned "create a new one" row, when the field declares one. Seeded from + * the sibling the list is scoped by, so a sandbox created off a JavaScript + * block does not land in the Python list and vanish. + */ + const createOption = useMemo((): ComboboxOption | null => { + const action = config.createAction + if (!action || isPreview || disabled) return null + return { + label: CREATE_ACTION_LABEL[action], + value: CREATE_ACTION_VALUE, + icon: Plus, + onSelect: () => { + const language = useSubBlockStore.getState().getValue(blockId, 'language') + setCreateLanguage(language === 'python' || language === 'javascript' ? language : undefined) + setIsCreateOpen(true) + }, + } + }, [config.createAction, isPreview, disabled, blockId]) + // Normalize fetched options to match ComboBoxOption format const normalizedFetchedOptions = useMemo((): ComboBoxOption[] => { return fetchedOptions.map((opt) => ({ label: opt.label, id: opt.id })) @@ -206,12 +205,25 @@ export const ComboBox = memo(function ComboBox({ } } + // Something just created through the pinned create row is selected before any + // list has refetched, so without this the field would sit on the raw id until + // hydration answered. Dropped again the moment a real fetch carries it. + if (createdOption) { + const alreadyPresent = opts.some((o) => + typeof o === 'string' ? o === createdOption.id : o.id === createdOption.id + ) + if (!alreadyPresent) { + opts = [createdOption, ...opts] + } + } + return opts }, [ fetchOptions, normalizedFetchedOptions, staticOptions, hydratedOption, + createdOption, subBlockId, isProviderAllowed, isModelAllowed, @@ -219,13 +231,14 @@ export const ComboBox = memo(function ComboBox({ // Convert options to Combobox format const comboboxOptions = useMemo((): ComboboxOption[] => { - return evaluatedOptions.map((option) => { + const mapped = evaluatedOptions.map((option): ComboboxOption => { if (typeof option === 'string') { return { label: option, value: option } } return { label: option.label, value: option.id, icon: option.icon } }) - }, [evaluatedOptions]) + return createOption ? [createOption, ...mapped] : mapped + }, [evaluatedOptions, createOption]) /** * Extracts the value identifier from an option @@ -260,12 +273,20 @@ export const ComboBox = memo(function ComboBox({ } } + // Auto-selecting the first option is only right for a field that must hold + // something. When empty is a real, documented choice (`sandboxId` — "no extra + // packages"), pre-filling it silently mutates and persists the block the + // moment the user opens advanced options. + if (config.emptyIsValid) { + return undefined + } + if (evaluatedOptions.length > 0) { return getOptionValue(evaluatedOptions[0]) } return undefined - }, [defaultValue, evaluatedOptions, subBlockId, getOptionValue]) + }, [defaultValue, evaluatedOptions, subBlockId, getOptionValue, config.emptyIsValid]) /** * Resolve the user-facing text for the current stored value. @@ -302,93 +323,6 @@ export const ComboBox = memo(function ComboBox({ } }, [value, defaultOptionValue, setStoreValue, isPermissionLoading]) - // Clear fetched options and hydrated option when dependencies change - useEffect(() => { - if (fetchOptions && dependsOnFields.length > 0) { - const currentDependencyValuesStr = JSON.stringify(dependencyValues) - const previousDependencyValuesStr = previousDependencyValuesRef.current - - if ( - previousDependencyValuesStr && - currentDependencyValuesStr !== previousDependencyValuesStr - ) { - setFetchedOptions([]) - setHydratedOption(null) - } - - previousDependencyValuesRef.current = currentDependencyValuesStr - } - }, [dependencyValues, fetchOptions, dependsOnFields.length]) - - // Fetch options when needed (on mount, when enabled, or when dependencies change) - useEffect(() => { - if ( - fetchOptions && - !isPreview && - !disabled && - fetchedOptions.length === 0 && - !isLoadingOptions && - !fetchError - ) { - fetchOptionsIfNeeded() - } - // eslint-disable-next-line react-hooks/exhaustive-deps -- fetchOptionsIfNeeded deps already covered above - }, [ - fetchOptions, - isPreview, - disabled, - fetchedOptions.length, - isLoadingOptions, - fetchError, - dependencyValues, - ]) - - // Hydrate the stored value's label by fetching it individually - useEffect(() => { - if (!fetchOptionById || isPreview || disabled) return - - const valueToHydrate = value as string | null | undefined - if (!valueToHydrate) return - - // Skip if value is an expression (not a real ID) - if (valueToHydrate.startsWith('<') || valueToHydrate.includes('{{')) return - - // Skip if already hydrated with the same value - if (hydratedOption?.id === valueToHydrate) return - - // Skip if value is already in fetched options or static options - const alreadyInFetchedOptions = fetchedOptions.some((opt) => opt.id === valueToHydrate) - const alreadyInStaticOptions = staticOptions.some((opt) => - typeof opt === 'string' ? opt === valueToHydrate : opt.id === valueToHydrate - ) - if (alreadyInFetchedOptions || alreadyInStaticOptions) return - - // Track if effect is still active (cleanup on unmount or value change) - let isActive = true - - // Fetch the hydrated option - fetchOptionById(blockId, valueToHydrate) - .then((option) => { - if (isActive) setHydratedOption(option) - }) - .catch(() => { - if (isActive) setHydratedOption(null) - }) - - return () => { - isActive = false - } - }, [ - fetchOptionById, - value, - blockId, - isPreview, - disabled, - fetchedOptions, - staticOptions, - hydratedOption?.id, - ]) - /** * Handles wheel event for ReactFlow zoom control * Intercepts Ctrl/Cmd+Wheel to zoom the canvas @@ -434,10 +368,10 @@ export const ComboBox = memo(function ComboBox({ const handleOpenChange = useCallback( (open: boolean) => { if (open) { - void fetchOptionsIfNeeded() + refetchOptions() } }, - [fetchOptionsIfNeeded] + [refetchOptions] ) /** @@ -590,6 +524,18 @@ export const ComboBox = memo(function ComboBox({ ) }} + + {config.createAction === 'sandbox' && ( + { + setCreatedOption({ label: sandbox.name, id: sandbox.id }) + setStoreValue(sandbox.id) + }} + /> + )}
) }) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx index d4f831d559f..cf50fe16738 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx @@ -1,13 +1,10 @@ -import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { memo, useCallback, useEffect, useMemo, useRef } from 'react' import { ChipTag, Combobox, type ComboboxOption } from '@sim/emcn' -import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' -import { isEqual } from 'es-toolkit' -import { useStoreWithEqualityFn } from 'zustand/traditional' -import { buildCanonicalIndex, resolveDependencyValue } from '@/lib/workflows/subblocks/visibility' import { formatDisplayText } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/formatted-text' import { getWorkflowSearchLabelHighlight } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/workflow-search-highlight' +import { useFetchedOptions } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-fetched-options' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' import { useActiveSearchTarget } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/providers/active-search-target-provider' import { getBlock } from '@/blocks/registry' @@ -15,8 +12,6 @@ import type { SubBlockConfig } from '@/blocks/types' import { getDependsOnFields } from '@/blocks/utils' import { ResponseBlockHandler } from '@/executor/handlers/response/response-handler' import { usePermissionConfig } from '@/hooks/use-permission-config' -import { useWorkflowRegistry } from '@/stores/workflows/registry/store' -import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { useWorkflowStore } from '@/stores/workflows/workflow/store' /** Selected-value badges shown before folding the rest into a "+N" badge. */ @@ -107,37 +102,10 @@ export const Dropdown = memo(function Dropdown({ const dependsOnFields = useMemo(() => getDependsOnFields(dependsOn), [dependsOn]) - const activeWorkflowId = useWorkflowRegistry((s) => s.activeWorkflowId) - const blockState = useWorkflowStore((state) => state.blocks[blockId]) - const blockConfig = blockState?.type ? getBlock(blockState.type) : null - const canonicalIndex = useMemo( - () => buildCanonicalIndex(blockConfig?.subBlocks || []), - [blockConfig?.subBlocks] - ) - const canonicalModeOverrides = blockState?.data?.canonicalModes - const dependencyValues = useStoreWithEqualityFn( - useSubBlockStore, - useCallback( - (state) => { - if (dependsOnFields.length === 0 || !activeWorkflowId) return [] - const workflowValues = state.workflowValues[activeWorkflowId] || {} - const blockValues = workflowValues[blockId] || {} - return dependsOnFields.map((depKey) => - resolveDependencyValue(depKey, blockValues, canonicalIndex, canonicalModeOverrides) - ) - }, - [dependsOnFields, activeWorkflowId, blockId, canonicalIndex, canonicalModeOverrides] - ), - isEqual - ) - - const [fetchedOptions, setFetchedOptions] = useState>([]) - const [isLoadingOptions, setIsLoadingOptions] = useState(false) - const [fetchError, setFetchError] = useState(null) - const [hydratedOption, setHydratedOption] = useState<{ label: string; id: string } | null>(null) + const blockType = useWorkflowStore((state) => state.blocks[blockId]?.type) + const blockConfig = blockType ? getBlock(blockType) : null const previousModeRef = useRef(null) - const previousDependencyValuesRef = useRef('') const [builderData, setBuilderData] = useSubBlockValue(blockId, 'builderData') const [data, setData] = useSubBlockValue(blockId, 'data') @@ -161,22 +129,26 @@ export const Dropdown = memo(function Dropdown({ : [] : null - const fetchOptionsIfNeeded = useCallback(async () => { - if (!fetchOptions || isPreview || disabled) return + const evaluatedOptions = useMemo(() => { + return typeof options === 'function' ? options() : options + }, [options]) - setIsLoadingOptions(true) - setFetchError(null) - try { - const options = await fetchOptions(blockId) - setFetchedOptions(options) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to fetch options') - setFetchError(errorMessage) - setFetchedOptions([]) - } finally { - setIsLoadingOptions(false) - } - }, [fetchOptions, blockId, isPreview, disabled]) + const { + fetchedOptions, + isLoadingOptions, + fetchError, + hydratedOption, + refetch: refetchOptions, + } = useFetchedOptions({ + blockId, + dependsOnFields, + fetchOptions, + fetchOptionById, + isPreview: Boolean(isPreview), + disabled: Boolean(disabled), + valueToHydrate: singleValue, + localOptions: evaluatedOptions, + }) /** * Handles combobox open state changes to trigger option fetching @@ -184,16 +156,12 @@ export const Dropdown = memo(function Dropdown({ const handleOpenChange = useCallback( (open: boolean) => { if (open) { - void fetchOptionsIfNeeded() + refetchOptions() } }, - [fetchOptionsIfNeeded] + [refetchOptions] ) - const evaluatedOptions = useMemo(() => { - return typeof options === 'function' ? options() : options - }, [options]) - const normalizedFetchedOptions = useMemo(() => { return fetchedOptions.map((opt) => ({ label: opt.label, id: opt.id })) }, [fetchedOptions]) @@ -388,103 +356,6 @@ export const Dropdown = memo(function Dropdown({ [isPreview, disabled, setStoreValue] ) - /** - * Effect to clear fetched options and hydrated option when dependencies actually change - * This ensures options are refetched with new dependency values (e.g., new credentials) - */ - useEffect(() => { - if (fetchOptions && dependsOnFields.length > 0) { - const currentDependencyValuesStr = JSON.stringify(dependencyValues) - const previousDependencyValuesStr = previousDependencyValuesRef.current - - if ( - previousDependencyValuesStr && - currentDependencyValuesStr !== previousDependencyValuesStr - ) { - setFetchedOptions([]) - setHydratedOption(null) - } - - previousDependencyValuesRef.current = currentDependencyValuesStr - } - }, [dependencyValues, fetchOptions, dependsOnFields.length]) - - /** - * Effect to fetch options when needed (on mount, when enabled, or when dependencies change) - */ - useEffect(() => { - if ( - fetchOptions && - !isPreview && - !disabled && - fetchedOptions.length === 0 && - !isLoadingOptions && - !fetchError - ) { - fetchOptionsIfNeeded() - } - // eslint-disable-next-line react-hooks/exhaustive-deps -- fetchOptionsIfNeeded deps already covered above - }, [ - fetchOptions, - isPreview, - disabled, - fetchedOptions.length, - isLoadingOptions, - fetchError, - dependencyValues, - ]) - - /** - * Effect to hydrate the stored value's label by fetching it individually - * This ensures the correct label is shown before the full options list loads - */ - useEffect(() => { - if (!fetchOptionById || isPreview || disabled) return - - // Get the value to hydrate (single value only, not multi-select) - const valueToHydrate = multiSelect ? null : (singleValue as string | null | undefined) - if (!valueToHydrate) return - - // Skip if value is an expression (not a real ID) - if (valueToHydrate.startsWith('<') || valueToHydrate.includes('{{')) return - - // Skip if already hydrated with the same value - if (hydratedOption?.id === valueToHydrate) return - - // Skip if value is already in fetched options or static options - const alreadyInFetchedOptions = fetchedOptions.some((opt) => opt.id === valueToHydrate) - const alreadyInStaticOptions = evaluatedOptions.some((opt) => - typeof opt === 'string' ? opt === valueToHydrate : opt.id === valueToHydrate - ) - if (alreadyInFetchedOptions || alreadyInStaticOptions) return - - // Track if effect is still active (cleanup on unmount or value change) - let isActive = true - - // Fetch the hydrated option - fetchOptionById(blockId, valueToHydrate) - .then((option) => { - if (isActive) setHydratedOption(option) - }) - .catch(() => { - if (isActive) setHydratedOption(null) - }) - - return () => { - isActive = false - } - }, [ - fetchOptionById, - singleValue, - multiSelect, - blockId, - isPreview, - disabled, - fetchedOptions, - evaluatedOptions, - hydratedOption?.id, - ]) - /** * Custom overlay content for multi-select mode. Shows at most two badges * and folds the rest into a "+N" badge, matching the summary notation used diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tool-input/components/tools/sub-block-renderer.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tool-input/components/tools/sub-block-renderer.tsx index 00798f4a976..20492c4127d 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tool-input/components/tools/sub-block-renderer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tool-input/components/tools/sub-block-renderer.tsx @@ -37,6 +37,16 @@ interface ToolSubBlockRendererProps { */ const OBJECT_SUBBLOCK_TYPES = new Set(['file-upload', 'table', 'grouped-checkbox-list']) +/** + * Whether this subblock's store value is a non-string. Covers the always-object + * types above plus any `multiSelect` control, whose value is an array — without + * this a multi-select's JSON string is rendered as a single literal chip and the + * next edit persists a nested-encoded value. + */ +function holdsObjectValue(subBlock: { type: string; multiSelect?: boolean }): boolean { + return OBJECT_SUBBLOCK_TYPES.has(subBlock.type) || Boolean(subBlock.multiSelect) +} + /** * Bridges the subblock store with StoredTool.params via a synthetic store key, * then delegates all rendering to SubBlock for full parity. @@ -55,7 +65,7 @@ export function ToolSubBlockRenderer({ }: ToolSubBlockRendererProps) { const syntheticId = buildToolSubBlockId(subBlockId, toolIndex, effectiveParamId) const toolParamValue = toolParams?.[effectiveParamId] ?? '' - const isObjectType = OBJECT_SUBBLOCK_TYPES.has(subBlock.type) + const isObjectType = holdsObjectValue(subBlock) const syncedRef = useRef(null) const onParamChangeRef = useRef(onParamChange) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-fetched-options.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-fetched-options.ts new file mode 100644 index 00000000000..2fd69a61caf --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-fetched-options.ts @@ -0,0 +1,197 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { getErrorMessage } from '@sim/utils/errors' +import { isEqual } from 'es-toolkit' +import { useStoreWithEqualityFn } from 'zustand/traditional' +import { buildCanonicalIndex, resolveDependencyValue } from '@/lib/workflows/subblocks/visibility' +import { getBlock } from '@/blocks/registry' +import { useWorkflowRegistry } from '@/stores/workflows/registry/store' +import { useSubBlockStore } from '@/stores/workflows/subblock/store' +import { useWorkflowStore } from '@/stores/workflows/workflow/store' + +export interface FetchedOption { + label: string + id: string +} + +/** An option the control already knows about, static or previously fetched. */ +type LocalOption = string | { id: string } + +interface UseFetchedOptionsProps { + blockId: string + /** Sibling subblock ids this list is scoped by; a change refetches. */ + dependsOnFields: string[] + fetchOptions?: (blockId: string) => Promise + fetchOptionById?: (blockId: string, optionId: string) => Promise + isPreview: boolean + disabled: boolean + /** + * The stored value whose label needs resolving before the full list loads. + * Multi-select controls pass `null` — there is no single label to hydrate. + */ + valueToHydrate: string | null | undefined + /** Options already resolvable without a fetch, so hydration can skip one. */ + localOptions: readonly LocalOption[] +} + +export interface UseFetchedOptionsResult { + fetchedOptions: FetchedOption[] + isLoadingOptions: boolean + fetchError: string | null + hydratedOption: FetchedOption | null + /** Fetches now, bypassing the once-per-dependency-set guard. For open handlers. */ + refetch: () => void +} + +function hasLocalOption(options: readonly LocalOption[], id: string): boolean { + return options.some((option) => (typeof option === 'string' ? option === id : option.id === id)) +} + +/** + * Owns the async-option lifecycle shared by the Dropdown and ComboBox subblock + * controls: fetching the list, clearing and refetching it when the fields it + * depends on change, and hydrating a stored value's label before the list loads. + * + * This exists as one hook because the two controls previously carried the same + * ~115 lines twice and drifted: a fix that added a `hasFetched` guard to both + * added the matching reset to only one, leaving every dependent Dropdown unable + * to refetch after its dependency changed. + */ +export function useFetchedOptions({ + blockId, + dependsOnFields, + fetchOptions, + fetchOptionById, + isPreview, + disabled, + valueToHydrate, + localOptions, +}: UseFetchedOptionsProps): UseFetchedOptionsResult { + const activeWorkflowId = useWorkflowRegistry((s) => s.activeWorkflowId) + const blockState = useWorkflowStore((state) => state.blocks[blockId]) + const blockConfig = blockState?.type ? getBlock(blockState.type) : null + const canonicalModeOverrides = blockState?.data?.canonicalModes + const canonicalIndex = useMemo( + () => buildCanonicalIndex(blockConfig?.subBlocks || []), + [blockConfig?.subBlocks] + ) + + const dependencyValues = useStoreWithEqualityFn( + useSubBlockStore, + useCallback( + (state) => { + if (dependsOnFields.length === 0 || !activeWorkflowId) return [] + const workflowValues = state.workflowValues[activeWorkflowId] || {} + const blockValues = workflowValues[blockId] || {} + return dependsOnFields.map((depKey) => + resolveDependencyValue(depKey, blockValues, canonicalIndex, canonicalModeOverrides) + ) + }, + [dependsOnFields, activeWorkflowId, blockId, canonicalIndex, canonicalModeOverrides] + ), + isEqual + ) + + const [fetchedOptions, setFetchedOptions] = useState([]) + const [isLoadingOptions, setIsLoadingOptions] = useState(false) + const [fetchError, setFetchError] = useState(null) + const [hydratedOption, setHydratedOption] = useState(null) + + const previousDependencyValuesRef = useRef('') + /** + * Whether a fetch has already been attempted for the current dependency values. + * "Have we fetched?" cannot be inferred from `fetchedOptions.length === 0` — a + * fetcher that legitimately returns no options (a workspace with no sandboxes, + * no credential selected) leaves the length at 0 while the loading flag flips + * back to false, re-satisfying the effect's guards and spinning it forever. + */ + const hasFetchedRef = useRef(false) + + const runFetch = useCallback(async () => { + if (!fetchOptions || isPreview || disabled) return + + setIsLoadingOptions(true) + setFetchError(null) + try { + const options = await fetchOptions(blockId) + setFetchedOptions(options) + } catch (error) { + setFetchError(getErrorMessage(error, 'Failed to fetch options')) + setFetchedOptions([]) + } finally { + setIsLoadingOptions(false) + } + }, [fetchOptions, blockId, isPreview, disabled]) + + useEffect(() => { + if (!fetchOptions || dependsOnFields.length === 0) return + + const current = JSON.stringify(dependencyValues) + const previous = previousDependencyValuesRef.current + if (previous && current !== previous) { + setFetchedOptions([]) + setHydratedOption(null) + // Both flags are what gate the fetch effect below, so both have to clear + // with the list: a stale error would block every future refetch, and a + // stale `hasFetched` would stop the new dependency values ever loading. + setFetchError(null) + hasFetchedRef.current = false + } + previousDependencyValuesRef.current = current + }, [dependencyValues, fetchOptions, dependsOnFields.length]) + + useEffect(() => { + if ( + fetchOptions && + !isPreview && + !disabled && + !hasFetchedRef.current && + !isLoadingOptions && + !fetchError + ) { + hasFetchedRef.current = true + void runFetch() + } + // eslint-disable-next-line react-hooks/exhaustive-deps -- runFetch deps already covered above + }, [fetchOptions, isPreview, disabled, isLoadingOptions, fetchError, dependencyValues]) + + useEffect(() => { + if (!fetchOptionById || isPreview || disabled) return + if (!valueToHydrate) return + + // An expression rather than a real id — there is nothing to look up. + if (valueToHydrate.startsWith('<') || valueToHydrate.includes('{{')) return + + if (hydratedOption?.id === valueToHydrate) return + if (hasLocalOption(fetchedOptions, valueToHydrate)) return + if (hasLocalOption(localOptions, valueToHydrate)) return + + let isActive = true + fetchOptionById(blockId, valueToHydrate) + .then((option) => { + if (isActive) setHydratedOption(option) + }) + .catch(() => { + if (isActive) setHydratedOption(null) + }) + + return () => { + isActive = false + } + }, [ + fetchOptionById, + valueToHydrate, + blockId, + isPreview, + disabled, + fetchedOptions, + localOptions, + hydratedOption?.id, + ]) + + const refetch = useCallback(() => { + hasFetchedRef.current = true + void runFetch() + }, [runFetch]) + + return { fetchedOptions, isLoadingOptions, fetchError, hydratedOption, refetch } +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-editor-subblock-layout.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-editor-subblock-layout.ts index 9c4a0a09213..6ca9b69470d 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-editor-subblock-layout.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-editor-subblock-layout.ts @@ -6,6 +6,7 @@ import { isSubBlockHidden, isSubBlockVisibleForMode, isSubBlockVisibleForTriggerMode, + isToolInputOnlySubBlock, shouldUseSubBlockForTriggerModeCanonicalIndex, } from '@/lib/workflows/subblocks/visibility' import type { BlockConfig, SubBlockConfig } from '@/blocks/types' @@ -117,6 +118,9 @@ export function useEditorSubblockLayout( const visibleSubBlocks = (config.subBlocks || []).filter((block) => { if (block.hidden) return false + // Configures the block as an agent tool; it has no meaning on the canvas. + if (isToolInputOnlySubBlock(block)) return false + // Filter by reactive condition (evaluated via hooks before useMemo) if (hiddenByReactiveCondition.has(block.id)) return false diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx index 4794bd82128..71668643285 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx @@ -17,6 +17,7 @@ import { getDisplayValue, resolveDropdownLabel, resolveFilterFieldLabel, + resolveSandboxLabel, resolveSkillsLabel, resolveToolsLabel, resolveVariablesLabel, @@ -30,6 +31,7 @@ import { isSubBlockFeatureEnabled, isSubBlockHidden, isSubBlockVisibleForMode, + isToolInputOnlySubBlock, isTriggerModeSubBlock, resolveDependencyValue, } from '@/lib/workflows/subblocks/visibility' @@ -60,6 +62,7 @@ import { useCustomTools } from '@/hooks/queries/custom-tools' import { useDeployWorkflow } from '@/hooks/queries/deployments' import { useMcpServers, useMcpToolsQuery } from '@/hooks/queries/mcp' import { useCredentialName } from '@/hooks/queries/oauth/oauth-credentials' +import { useSandboxes } from '@/hooks/queries/sandboxes' import { useReactivateSchedule, useScheduleInfo } from '@/hooks/queries/schedules' import { useSkills } from '@/hooks/queries/skills' import { useTablesList } from '@/hooks/queries/tables' @@ -446,6 +449,19 @@ const SubBlockRow = memo(function SubBlockRow({ [subBlock, rawValue, workspaceSkills] ) + /** + * Hydrates the Function block's sandbox id to its name. Deliberately scoped to + * the sandbox row: this row is memoized per subblock, and the shared list query + * polls while a build is in flight, so subscribing unconditionally would + * re-render every row on the canvas on each poll tick. + */ + const isSandboxField = subBlock?.id === 'sandboxId' && subBlock?.type === 'combobox' + const { data: sandboxData } = useSandboxes(isSandboxField ? workspaceId || undefined : undefined) + const sandboxDisplayValue = useMemo( + () => resolveSandboxLabel(subBlock, rawValue, sandboxData?.sandboxes ?? []), + [subBlock, rawValue, sandboxData] + ) + const isPasswordField = subBlock?.password === true const maskedValue = isPasswordField && value && value !== '-' ? '•••' : null const isMonospaceField = Boolean(filterDisplayValue) @@ -458,6 +474,7 @@ const SubBlockRow = memo(function SubBlockRow({ filterDisplayValue || toolsDisplayValue || skillsDisplayValue || + sandboxDisplayValue || knowledgeBaseDisplayName || workflowSelectionName || mcpServerDisplayName || @@ -628,6 +645,9 @@ export const WorkflowBlock = memo(function WorkflowBlock({ if (block.hideFromPreview) return false if (hiddenByReactiveCondition.has(block.id)) return false if (!isSubBlockFeatureEnabled(block)) return false + + // Configures the block as an agent tool; it has no meaning on the canvas. + if (isToolInputOnlySubBlock(block)) return false if (isSubBlockHidden(block)) return false const isPureTriggerBlock = config?.triggers?.enabled && config.category === 'triggers' diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand.ts index dc6bd65b33f..34c0b712ee2 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-wand.ts @@ -1,6 +1,7 @@ import { useCallback, useRef, useState } from 'react' import { toast } from '@sim/emcn' import { createLogger } from '@sim/logger' +import { filterUndefined } from '@sim/utils/object' import { useQueryClient } from '@tanstack/react-query' import { useParams } from 'next/navigation' import { requestRaw } from '@/lib/api/client' @@ -76,12 +77,29 @@ export interface WandConfig { maintainHistory?: boolean // Whether to keep conversation history } +/** + * Client-supplied context the server's `wandEnrichers` expand into extra system + * prompt. Sent as ids only — the enricher does the DB/registry lookup, so no + * schema or package metadata has to round-trip through the browser. + */ +interface WandContextParams { + tableId?: string | null + sandboxId?: string | null +} + +/** Drops the unset keys so an all-empty context is omitted from the request. */ +function buildWandContext(params?: WandContextParams): Record | undefined { + const context = filterUndefined({ + tableId: params?.tableId ?? undefined, + sandboxId: params?.sandboxId ?? undefined, + }) + return Object.keys(context).length > 0 ? context : undefined +} + interface UseWandProps { wandConfig?: WandConfig currentValue?: string - contextParams?: { - tableId?: string | null - } + contextParams?: WandContextParams onGeneratedContent: (content: string) => void onStreamChunk?: (chunk: string) => void onStreamStart?: () => void @@ -185,7 +203,7 @@ export function useWand({ generationType: wandConfig?.generationType, workflowId: workflowId ?? undefined, workspaceId: workspaceId ?? undefined, - wandContext: contextParams?.tableId ? { tableId: contextParams.tableId } : undefined, + wandContext: buildWandContext(contextParams), }, signal: abortControllerRef.current.signal, }, @@ -267,6 +285,7 @@ export function useWand({ onGenerationComplete, queryClient, contextParams?.tableId, + contextParams?.sandboxId, workflowId, workspaceId, navigateToSettings, diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx index 46af4b662d8..6bbeafd5977 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx @@ -38,6 +38,7 @@ import { hasAdvancedValues, isSubBlockFeatureEnabled, isSubBlockVisibleForMode, + isToolInputOnlySubBlock, } from '@/lib/workflows/subblocks/visibility' import { DELETED_WORKFLOW_LABEL } from '@/app/workspace/[workspaceId]/logs/utils' import { SubBlock } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components' @@ -1171,6 +1172,9 @@ function PreviewEditorContent({ if (effectiveTrigger && subBlock.mode !== 'trigger' && subBlock.mode !== 'trigger-advanced') return false if (!isSubBlockFeatureEnabled(subBlock)) return false + + // Configures the block as an agent tool; it has no meaning on the canvas. + if (isToolInputOnlySubBlock(subBlock)) return false if ( !isSubBlockVisibleForMode( subBlock, diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx index 4dde0ebfbf3..9c9b9f8a933 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-workflow/components/block/block.tsx @@ -17,6 +17,7 @@ import { evaluateSubBlockCondition, isSubBlockFeatureEnabled, isSubBlockVisibleForMode, + isToolInputOnlySubBlock, } from '@/lib/workflows/subblocks/visibility' import { getBlock } from '@/blocks' import { getTileIconColorClass } from '@/blocks/icon-color' @@ -212,6 +213,9 @@ function WorkflowPreviewBlockInner({ data }: NodeProps if (subBlock.hideFromPreview) return false if (!isSubBlockFeatureEnabled(subBlock)) return false + // Configures the block as an agent tool; it has no meaning on the canvas. + if (isToolInputOnlySubBlock(subBlock)) return false + if (effectiveTrigger) { const isValidTriggerSubblock = isPureTriggerBlock ? subBlock.mode === 'trigger' || subBlock.mode === 'trigger-advanced' || !subBlock.mode diff --git a/apps/sim/background/cleanup-sandbox-images.ts b/apps/sim/background/cleanup-sandbox-images.ts new file mode 100644 index 00000000000..7342479ac2b --- /dev/null +++ b/apps/sim/background/cleanup-sandbox-images.ts @@ -0,0 +1,20 @@ +import { createLogger } from '@sim/logger' +import { cleanupSandboxImages } from '@/lib/execution/remote-sandbox/image-registry' + +const logger = createLogger('CleanupSandboxImages') + +/** + * Days a build may go unused before the retention sweep removes it. A rebuild is + * cheap next to keeping every abandoned dependency set alive indefinitely, and + * E2B's docs flag possible future template-storage pricing. + */ +export const SANDBOX_IMAGE_RETENTION_DAYS = 30 + +export async function runCleanupSandboxImages(): Promise<{ deleted: number; failed: number }> { + const result = await cleanupSandboxImages(SANDBOX_IMAGE_RETENTION_DAYS) + logger.info('Swept unreferenced sandbox images', { + ...result, + retentionDays: SANDBOX_IMAGE_RETENTION_DAYS, + }) + return result +} diff --git a/apps/sim/background/sandbox-image-build.ts b/apps/sim/background/sandbox-image-build.ts new file mode 100644 index 00000000000..a7768e3bce7 --- /dev/null +++ b/apps/sim/background/sandbox-image-build.ts @@ -0,0 +1,28 @@ +import { task } from '@trigger.dev/sdk' +import { + runSandboxImageBuild, + type SandboxImageBuildPayload, +} from '@/lib/execution/remote-sandbox/image-registry' + +/** + * Trigger.dev wrapper around `runSandboxImageBuild`. The build's lifecycle lives + * in the `sandbox_image` row rather than the task, so this is a thin shell: the + * runner claims the row conditionally, which makes a re-delivery a no-op. + * + * `maxAttempts: 1` — the runner already polls the provider to a terminal state + * and records a classified failure the user can act on. A blind retry would just + * race the row it already marked `failed`; the user retries by saving again. + */ +export const sandboxImageBuildTask = task({ + id: 'sandbox-image-build', + machine: 'small-1x', + maxDuration: 1200, + retry: { maxAttempts: 1 }, + queue: { + name: 'sandbox-image-build', + concurrencyLimit: 5, + }, + run: async (payload: SandboxImageBuildPayload) => { + await runSandboxImageBuild(payload) + }, +}) diff --git a/apps/sim/blocks/blocks/function.ts b/apps/sim/blocks/blocks/function.ts index f4a8e72b77d..6229b718a75 100644 --- a/apps/sim/blocks/blocks/function.ts +++ b/apps/sim/blocks/blocks/function.ts @@ -1,5 +1,10 @@ import { CodeIcon } from '@/components/icons' import { CodeLanguage, getLanguageDisplayName } from '@/lib/execution/languages' +import { + fetchWorkspaceSandboxOption, + fetchWorkspaceSandboxOptions, + fetchWorkspaceSecretNameOptions, +} from '@/lib/workflows/subblocks/options' import type { BlockConfig } from '@/blocks/types' import type { CodeExecutionOutput } from '@/tools/function/types' @@ -8,11 +13,12 @@ export const FunctionBlock: BlockConfig = { name: 'Function', description: 'Run custom logic', longDescription: - 'This is a core workflow block. Execute custom JavaScript or Python code within your workflow. JavaScript without imports runs locally for fast execution, while code with imports or Python uses E2B sandbox.', + 'This is a core workflow block. Execute custom JavaScript or Python code within your workflow. JavaScript without imports runs locally for fast execution, while code with imports or Python runs in a remote sandbox.', bestPractices: ` - JavaScript code without external imports runs in a local VM for fastest execution. - - JavaScript code with import/require statements requires E2B and runs in a secure sandbox. - - Python code always requires E2B and runs in a secure sandbox. + - JavaScript code with import/require statements runs in a remote sandbox. + - Python code always runs in a remote sandbox. + - To import third-party packages, create a sandbox in Settings > Sandboxes and select it under the block's advanced options. Without one, only the standard library and built-in modules are available. - Can reference workflow variables using syntax as usual within code. Avoid XML/HTML tags. `, docsLink: 'https://docs.sim.ai/workflows/blocks/function', @@ -29,7 +35,7 @@ export const FunctionBlock: BlockConfig = { ], placeholder: 'Select language', value: () => CodeLanguage.JavaScript, - showWhenEnvSet: 'NEXT_PUBLIC_E2B_ENABLED', + showWhenEnvSet: 'NEXT_PUBLIC_SANDBOX_ENABLED,NEXT_PUBLIC_E2B_ENABLED', }, { id: 'code', @@ -50,7 +56,7 @@ IMPORTANT FORMATTING RULES: 1. Reference Environment Variables: Use the exact syntax {{VARIABLE_NAME}}. Do NOT wrap it in quotes (e.g., use 'apiKey = {{SERVICE_API_KEY}}' not 'apiKey = "{{SERVICE_API_KEY}}"'). Our system replaces these placeholders before execution. 2. Reference Input Parameters/Workflow Variables: Use the exact syntax . Do NOT wrap it in quotes (e.g., use 'userId = ;' not 'userId = "";'). This includes parameters defined in the block's schema and outputs from previous blocks. 3. Function Body ONLY: Do NOT include the function signature (e.g., 'async function myFunction() {' or the surrounding '}'). -4. Imports: Do NOT include import/require statements unless they are standard Node.js built-in modules (e.g., 'crypto', 'fs'). External libraries are not supported in this context. +4. Imports: Standard Node.js built-in modules (e.g., 'crypto', 'fs') are always available. Third-party packages are available ONLY when the block has a sandbox selected — the sandbox's package list is appended below when one is. Never import a package that is not on that list. 5. Output: Ensure the code returns a value if the function is expected to produce output. Use 'return'. 6. Clarity: Write clean, readable code. 7. No Explanations: Do NOT include markdown formatting, comments explaining the rules, or any text other than the raw JavaScript code for the function body. @@ -89,6 +95,54 @@ try { generationType: 'javascript-function-body', }, }, + { + id: 'sandboxId', + title: 'Sandbox', + type: 'combobox', + mode: 'advanced', + searchable: true, + // Empty means the default image — the picker must never auto-select for us. + emptyIsValid: true, + createAction: 'sandbox', + // Refetched whenever `language` changes, so the list is always scoped to + // sandboxes this block can actually run in. + dependsOn: ['language'], + showWhenEnvSet: 'NEXT_PUBLIC_SANDBOX_ENABLED,NEXT_PUBLIC_E2B_ENABLED', + placeholder: 'Default image', + description: + 'Packages this block can import. Manage sandboxes in Settings > Sandboxes. Leaving this empty runs on the default image.', + options: [], + fetchOptions: (blockId) => fetchWorkspaceSandboxOptions(blockId), + fetchOptionById: (blockId, optionId) => fetchWorkspaceSandboxOption(blockId, optionId), + }, + { + id: 'secretScope', + title: 'Secret access', + type: 'dropdown', + // Only meaningful when an agent calls this block as a tool. + context: 'tool-input', + paramVisibility: 'user-only', + options: [ + { label: 'All secrets', id: 'all' }, + { label: 'Selected secrets', id: 'selected' }, + ], + value: () => 'all', + description: + 'Code can read any workspace secret, including ones added later. Narrow this to advertise a specific set to the model.', + }, + { + id: 'mountedSecrets', + title: 'Secrets', + type: 'dropdown', + context: 'tool-input', + paramVisibility: 'user-only', + multiSelect: true, + searchable: true, + options: [], + condition: { field: 'secretScope', value: 'selected' }, + placeholder: 'Select secrets this tool can read', + fetchOptions: () => fetchWorkspaceSecretNameOptions(), + }, ], tools: { access: ['function_execute'], @@ -97,6 +151,12 @@ try { code: { type: 'string', description: 'JavaScript or Python code to execute' }, language: { type: 'string', description: 'Language (javascript or python)' }, timeout: { type: 'number', description: 'Execution timeout' }, + sandboxId: { type: 'string', description: 'Workspace sandbox providing importable packages' }, + secretScope: { type: 'string', description: 'Secret access mode: all or selected' }, + mountedSecrets: { + type: 'json', + description: 'Workspace secret names this block may read when secretScope is selected', + }, }, outputs: { result: { type: 'json', description: 'Return value from the executed JavaScript function' }, diff --git a/apps/sim/blocks/types.ts b/apps/sim/blocks/types.ts index 110e062e97d..2b29a9a1ab4 100644 --- a/apps/sim/blocks/types.ts +++ b/apps/sim/blocks/types.ts @@ -262,6 +262,29 @@ export interface SubBlockConfig { canonicalParamId?: string /** Controls parameter visibility in agent/tool-input context */ paramVisibility?: 'user-or-llm' | 'user-only' | 'llm-only' | 'hidden' + /** + * Marks "nothing selected" as a real choice, so a dynamic-option control does + * not pre-fill itself with the first option it fetches. Without it a combobox + * silently writes and persists a value the user never picked. + */ + emptyIsValid?: boolean + /** + * Pins a "create a new one" row above the options of a picker, so authoring a + * resource never means leaving the workflow for Settings. + * + * Names the resource rather than carrying a component: block configs are read + * by the serializer and the executor, which must not pull in React. The picker + * owns the modal each name maps to. + */ + createAction?: 'sandbox' + /** + * Restricts where a subblock renders. `tool-input` means it configures how the + * block behaves *as an agent tool* and has no meaning on the canvas, so the + * canvas editor skips it while the agent's tool-input config still shows it. + * + * Generic on purpose: shared code branches on this flag, never on a block type. + */ + context?: 'tool-input' required?: | boolean | { @@ -315,7 +338,7 @@ export interface SubBlockConfig { hidden?: boolean hideFromPreview?: boolean // Hide this subblock from the workflow block preview hideDividerBefore?: boolean // Visually group this field with the preceding visible subblock - showWhenEnvSet?: string // Show this subblock only when the named NEXT_PUBLIC_ env var is truthy + showWhenEnvSet?: string // Show this subblock only when a named NEXT_PUBLIC_ env var is truthy; comma-separated means any of them hideWhenHosted?: boolean // Hide this subblock when running on hosted sim hideWhenEnvSet?: string // Hide this subblock when the named NEXT_PUBLIC_ env var is truthy description?: string diff --git a/apps/sim/components/settings/navigation.test.ts b/apps/sim/components/settings/navigation.test.ts index 0dc9a894ff9..0bf912e7df4 100644 --- a/apps/sim/components/settings/navigation.test.ts +++ b/apps/sim/components/settings/navigation.test.ts @@ -1,7 +1,8 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { resetEnvMock, setEnv } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it } from 'vitest' import { ACCOUNT_SETTINGS_ITEMS, ACCOUNT_SETTINGS_PATH_ALIASES, @@ -23,6 +24,18 @@ import { WORKSPACE_SETTINGS_PATH_ALIASES, } from '@/components/settings/navigation' +/** + * The Sandboxes section is dropped on a deployment with no sandbox provider, and + * the env mock falls through to `process.env` — so without pinning this, every + * assertion below would depend on whether the developer's untracked + * `apps/sim/.env` sets the flag, passing locally and failing on CI. + */ +beforeEach(() => { + setEnv({ NEXT_PUBLIC_SANDBOX_ENABLED: 'true', NEXT_PUBLIC_E2B_ENABLED: undefined }) +}) + +afterAll(resetEnvMock) + describe('settings navigation boundaries', () => { it('preserves the order of all four settings catalogs', () => { expect(buildUnifiedSettingsNavigation().map(({ id }) => id)).toEqual([ @@ -42,6 +55,7 @@ describe('settings navigation boundaries', () => { 'apikeys', 'workflow-mcp-servers', 'byok', + 'sandboxes', 'inbox', 'recently-deleted', 'sso', @@ -76,6 +90,7 @@ describe('settings navigation boundaries', () => { 'teammates', 'secrets', 'byok', + 'sandboxes', 'custom-tools', 'mcp', 'workflow-mcp-servers', @@ -87,6 +102,37 @@ describe('settings navigation boundaries', () => { ]) }) + /** + * Entitlement decides whether a workspace may author sandboxes; this decides + * whether anything could run one. With no provider the tab is a dead end, so it + * is dropped rather than locked — an upgrade would not fix it. Both planes are + * asserted because each has its own filter. + */ + it('drops the Sandboxes section when no sandbox provider is configured', () => { + setEnv({ NEXT_PUBLIC_SANDBOX_ENABLED: undefined, NEXT_PUBLIC_E2B_ENABLED: undefined }) + + expect(buildUnifiedSettingsNavigation().map(({ id }) => id)).not.toContain('sandboxes') + expect( + resolveWorkspaceNavigation({ + permission: 'admin', + permissionConfig: {}, + entitlements: { + byok: true, + inbox: true, + customBlocks: true, + forks: true, + sandboxes: true, + }, + }).map(({ id }) => id) + ).not.toContain('sandboxes') + }) + + it('keeps the Sandboxes section on the pre-Daytona E2B flag alone', () => { + setEnv({ NEXT_PUBLIC_SANDBOX_ENABLED: undefined, NEXT_PUBLIC_E2B_ENABLED: 'true' }) + + expect(buildUnifiedSettingsNavigation().map(({ id }) => id)).toContain('sandboxes') + }) + it('has one registry source for every unified and plane item', () => { const unifiedIds = SETTINGS_SECTION_REGISTRY.flatMap(({ unified }) => unified ? [unified.id] : [] @@ -283,6 +329,7 @@ describe('settings navigation boundaries', () => { 'teammates', 'secrets', 'byok', + 'sandboxes', 'custom-tools', 'mcp', 'workflow-mcp-servers', @@ -299,6 +346,7 @@ describe('settings navigation boundaries', () => { 'teammates', 'secrets', 'byok', + 'sandboxes', 'custom-tools', 'mcp', 'workflow-mcp-servers', @@ -325,6 +373,7 @@ describe('settings navigation boundaries', () => { customBlocks: true, forks: true, inbox: true, + sandboxes: true, }, }) @@ -348,12 +397,14 @@ describe('settings navigation boundaries', () => { customBlocks: true, forks: true, inbox: true, + sandboxes: true, }, }) expect(items.map(({ id }) => id)).toEqual([ 'teammates', 'byok', + 'sandboxes', 'workflow-mcp-servers', 'recently-deleted', 'forks', diff --git a/apps/sim/components/settings/navigation.ts b/apps/sim/components/settings/navigation.ts index 08f12d531b5..3557160e1d1 100644 --- a/apps/sim/components/settings/navigation.ts +++ b/apps/sim/components/settings/navigation.ts @@ -24,7 +24,7 @@ import { Wrench, } from '@sim/emcn/icons' import { type PermissionType, permissionSatisfies } from '@sim/platform-authz/workspace' -import { McpIcon } from '@/components/icons' +import { CodeIcon, McpIcon } from '@/components/icons' import { getEnv, isTruthy } from '@/lib/core/config/env' import { isAccessControlEnabled, @@ -33,6 +33,7 @@ import { isDataRetentionEnabled, isHosted, isInboxEnabled, + isSandboxesEnabled, isSessionPoliciesEnabled, isSsoEnabled, isWhitelabelingEnabled, @@ -63,6 +64,7 @@ export type WorkspaceSettingsSection = | 'teammates' | 'secrets' | 'byok' + | 'sandboxes' | 'custom-tools' | 'mcp' | 'workflow-mcp-servers' @@ -110,6 +112,7 @@ export type UnifiedSettingsSection = | 'custom-tools' | 'workflow-mcp-servers' | 'inbox' + | 'sandboxes' | 'admin' | 'sessions' | 'data-retention' @@ -208,11 +211,31 @@ const SETTINGS_SELF_HOSTED_OVERRIDES = { dataDrains: isDataDrainsEnabled, dataRetention: isDataRetentionEnabled, inbox: isInboxEnabled, + sandboxes: isSandboxesEnabled, sessionPolicies: isSessionPoliciesEnabled, sso: isSsoEnabled, whitelabeling: isWhitelabelingEnabled, } as const +/** + * Whether this deployment can run remote sandboxes at all. + * + * Entitlement decides whether a workspace may *author* sandboxes; this decides + * whether anything could ever *run* one. Without a provider the tab is a dead + * end — you can define a dependency set that nothing will build and no Function + * block can select, because the picker is gated on this same pair of vars. + * + * It reads those browser twins rather than the server's `isRemoteSandboxEnabled` + * precisely so the two agree: that flag reads non-public vars, and this module + * renders on both sides. `NEXT_PUBLIC_E2B_ENABLED` is the pre-Daytona fallback, + * matching the picker's `showWhenEnvSet` order. + */ +function isSandboxExecutionAvailable(): boolean { + return ( + isTruthy(getEnv('NEXT_PUBLIC_SANDBOX_ENABLED')) || isTruthy(getEnv('NEXT_PUBLIC_E2B_ENABLED')) + ) +} + export const SETTINGS_NAVIGATION_BILLING_ENABLED = isTruthy(getEnv('NEXT_PUBLIC_BILLING_ENABLED')) type SettingsHrefSearchParams = Pick @@ -439,7 +462,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'enterprise', }, planes: { - workspace: { id: 'forks', group: 'enterprise', order: 9 }, + workspace: { id: 'forks', group: 'enterprise', order: 10 }, }, }, { @@ -526,7 +549,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'tools', }, planes: { - workspace: { id: 'custom-tools', group: 'tools', order: 3 }, + workspace: { id: 'custom-tools', group: 'tools', order: 4 }, }, }, { @@ -538,7 +561,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'tools', }, planes: { - workspace: { id: 'mcp', group: 'tools', order: 4 }, + workspace: { id: 'mcp', group: 'tools', order: 5 }, }, }, { @@ -560,7 +583,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'api-keys', description: 'Manage workspace API keys and personal-key policy.', group: 'system', - order: 6, + order: 7, }, }, }, @@ -573,7 +596,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'system', }, planes: { - workspace: { id: 'workflow-mcp-servers', group: 'tools', order: 5 }, + workspace: { id: 'workflow-mcp-servers', group: 'tools', order: 6 }, }, }, { @@ -589,6 +612,22 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] workspace: { id: 'byok', group: 'workspace', order: 2 }, }, }, + { + label: 'Sandboxes', + icon: CodeIcon, + docsLink: 'https://docs.sim.ai/workflows/blocks/function', + unified: { + id: 'sandboxes', + description: 'Install Python or npm packages for Function blocks to import.', + group: 'system', + requiresMax: true, + selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.sandboxes, + showWhenLocked: true, + }, + planes: { + workspace: { id: 'sandboxes', group: 'workspace', order: 3 }, + }, + }, { label: 'Chat keys', icon: HexSimple, @@ -614,7 +653,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] showWhenLocked: true, }, planes: { - workspace: { id: 'inbox', group: 'system', order: 7 }, + workspace: { id: 'inbox', group: 'system', order: 8 }, }, }, { @@ -626,7 +665,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'system', }, planes: { - workspace: { id: 'recently-deleted', group: 'system', order: 8 }, + workspace: { id: 'recently-deleted', group: 'system', order: 9 }, }, }, { @@ -724,7 +763,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.customBlocks, }, planes: { - workspace: { id: 'custom-blocks', group: 'enterprise', order: 10 }, + workspace: { id: 'custom-blocks', group: 'enterprise', order: 11 }, }, }, { @@ -758,6 +797,10 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] export function buildUnifiedSettingsNavigation(): UnifiedSettingsNavigationItem[] { return SETTINGS_SECTION_REGISTRY.flatMap(({ label, icon, docsLink, unified }) => { if (!unified) return [] + // Dropped here rather than in each consumer's filter: the sidebar's + // `selfHostedOverride` short-circuit would otherwise reveal the tab on a + // deployment that has the entitlement but no provider to run what it builds. + if (unified.id === 'sandboxes' && !isSandboxExecutionAvailable()) return [] const { group, ...item } = unified return [ { @@ -892,6 +935,20 @@ export interface WorkspaceSettingsEntitlements { customBlocks: boolean forks: boolean inbox: boolean + sandboxes: boolean +} + +/** + * Sections that stay visible without their entitlement, rendering a locked + * upgrade prompt instead of disappearing from the nav. Keyed by the entitlement + * that unlocks them, so adding a gated section is one entry rather than another + * hardcoded id check in {@link resolveWorkspaceNavigation}. + */ +const LOCKABLE_WORKSPACE_SECTIONS: Partial< + Record +> = { + inbox: 'inbox', + sandboxes: 'sandboxes', } interface ResolveWorkspaceNavigationOptions { @@ -910,6 +967,7 @@ const WORKSPACE_MUTATION_PERMISSION: Record [...sandboxKeys.all, 'list'] as const, + list: (workspaceId?: string) => [...sandboxKeys.lists(), workspaceId ?? ''] as const, +} + +export const SANDBOX_LIST_STALE_TIME = 30 * 1000 + +/** Poll cadence while any sandbox is still building; see {@link useSandboxes}. */ +export const SANDBOX_BUILD_POLL_INTERVAL = 3 * 1000 + +/** + * Bounds the build poll. A worker killed mid-build leaves a row `building` until + * the next save re-claims it, and without this a tab left open on that sandbox + * would poll forever. Covers a little over the 15-minute build cap. + */ +const MAX_BUILD_POLLS = 350 + +async function fetchSandboxes( + workspaceId: string, + signal?: AbortSignal +): Promise { + return requestJson(listSandboxesContract, { params: { id: workspaceId }, signal }) +} + +/** True while at least one sandbox has a build that has not reached a terminal state. */ +export function hasPendingBuild(sandboxes: readonly Sandbox[]): boolean { + return sandboxes.some( + (sandbox) => sandbox.buildStatus === 'pending' || sandbox.buildStatus === 'building' + ) +} + +/** + * Query options shared by the hook and the Function block's sandbox picker + * (`fetchWorkspaceSandboxOptions`), so both read one cache entry rather than two. + */ +export function getSandboxListQueryOptions(workspaceId: string) { + return { + queryKey: sandboxKeys.list(workspaceId), + queryFn: ({ signal }: { signal?: AbortSignal }) => fetchSandboxes(workspaceId, signal), + staleTime: SANDBOX_LIST_STALE_TIME, + } +} + +export function useSandboxes(workspaceId?: string) { + return useQuery({ + queryKey: sandboxKeys.list(workspaceId), + queryFn: ({ signal }) => fetchSandboxes(workspaceId as string, signal), + enabled: Boolean(workspaceId), + staleTime: SANDBOX_LIST_STALE_TIME, + // Builds are the only thing that changes without a user action, so the poll + // runs only while one is in flight and stops on the first terminal read. + refetchInterval: (query) => + query.state.data && + hasPendingBuild(query.state.data.sandboxes) && + query.state.dataUpdateCount < MAX_BUILD_POLLS + ? SANDBOX_BUILD_POLL_INTERVAL + : false, + }) +} + +type CreateSandboxParams = { + workspaceId: string +} & ContractBodyInput + +export function useCreateSandbox() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async ({ workspaceId, ...body }: CreateSandboxParams) => { + const data = await requestJson(createSandboxContract, { + params: { id: workspaceId }, + body, + }) + logger.info(`Created sandbox ${body.name} in workspace ${workspaceId}`) + return data + }, + onSettled: (_data, _error, variables) => + queryClient.invalidateQueries({ queryKey: sandboxKeys.list(variables.workspaceId) }), + }) +} + +type UpdateSandboxParams = { + workspaceId: string + sandboxId: string +} & ContractBodyInput + +export function useUpdateSandbox() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async ({ workspaceId, sandboxId, ...body }: UpdateSandboxParams) => { + const data = await requestJson(updateSandboxContract, { + params: { id: workspaceId, sandboxId }, + body, + }) + logger.info(`Updated sandbox ${sandboxId} in workspace ${workspaceId}`) + return data + }, + onSettled: (_data, _error, variables) => + queryClient.invalidateQueries({ queryKey: sandboxKeys.list(variables.workspaceId) }), + }) +} + +export function useDeleteSandbox() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async ({ workspaceId, sandboxId }: { workspaceId: string; sandboxId: string }) => { + const data = await requestJson(deleteSandboxContract, { + params: { id: workspaceId, sandboxId }, + }) + logger.info(`Deleted sandbox ${sandboxId} from workspace ${workspaceId}`) + return data + }, + onSettled: (_data, _error, variables) => + queryClient.invalidateQueries({ queryKey: sandboxKeys.list(variables.workspaceId) }), + }) +} diff --git a/apps/sim/lib/api/contracts/hotspots.ts b/apps/sim/lib/api/contracts/hotspots.ts index 564dbf63f53..eaa75f8f430 100644 --- a/apps/sim/lib/api/contracts/hotspots.ts +++ b/apps/sim/lib/api/contracts/hotspots.ts @@ -190,6 +190,12 @@ export const functionExecuteContract = defineRouteContract({ workspaceId: z.string().optional(), userId: z.string().optional(), isCustomTool: z.boolean().optional().default(false), + /** Workspace sandbox whose dependency set this execution runs against. */ + sandboxId: z.string().optional(), + /** `all` (default) or `selected`; see mountedSecrets. */ + secretScope: z.enum(['all', 'selected']).optional(), + /** Secret names this execution may read when secretScope is `selected`. */ + mountedSecrets: z.array(z.string()).optional(), _sandboxFiles: z .array( z.union([ diff --git a/apps/sim/lib/api/contracts/index.ts b/apps/sim/lib/api/contracts/index.ts index 958a4ed21f9..2001b85b8c2 100644 --- a/apps/sim/lib/api/contracts/index.ts +++ b/apps/sim/lib/api/contracts/index.ts @@ -21,6 +21,7 @@ export * from './media' export * from './permission-groups' export * from './pinned-items' export * from './primitives' +export * from './sandboxes' export * from './selectors' export * from './skills' export * from './storage-transfer' diff --git a/apps/sim/lib/api/contracts/sandboxes.ts b/apps/sim/lib/api/contracts/sandboxes.ts new file mode 100644 index 00000000000..39547edfdc7 --- /dev/null +++ b/apps/sim/lib/api/contracts/sandboxes.ts @@ -0,0 +1,151 @@ +import { z } from 'zod' +import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { type ContractJsonResponse, defineRouteContract } from '@/lib/api/contracts/types' + +export const sandboxLanguageSchema = z.enum(['javascript', 'python']) + +export const sandboxBuildStatusSchema = z.enum(['pending', 'building', 'ready', 'failed']) + +/** + * How the active deployment materializes dependencies. `runtime` deployments + * have no build to report, so the UI swaps the build-status chip for a note + * about the per-execution install cost. + */ +export const sandboxStrategySchema = z.enum(['prebuilt', 'runtime']) + +/** + * A structural payload guard, NOT the product limit. + * + * The client submits one entry per raw textarea line — blanks and `#` comments + * included — so that a rejection can be addressed back to the row the user typed + * it on. `validateDependencies` strips those before applying the real + * MAX_SANDBOX_DEPENDENCIES / MAX_DEPENDENCY_LENGTH caps and returns per-line + * `issues` the editor marks inline. Bounding this at the real limits instead + * would reject a legal list (50 packages plus a trailing newline is 51 entries) + * with a generic error carrying no `issues`, leaving the editor nothing to mark. + */ +const dependencyListSchema = z + .array(z.string().max(2000, 'a dependency line is unreasonably long')) + .max(1000, 'too many lines — paste a shorter dependency list') + +const sandboxNameSchema = z + .string() + .trim() + .min(1, 'Name is required') + .max(64, 'Name must be 64 characters or fewer') + +export const sandboxSchema = z.object({ + id: z.string(), + name: z.string(), + language: sandboxLanguageSchema, + dependencies: z.array(z.string()), + /** Absent under the `runtime` strategy, which has nothing to build. */ + buildStatus: sandboxBuildStatusSchema.nullable(), + /** Classified failure code from the build taxonomy. */ + errorCode: z.string().nullable(), + /** User-facing copy for the failure; never a raw traceback. */ + errorMessage: z.string().nullable(), + /** Installer log tail, shown behind a disclosure. */ + errorDetail: z.string().nullable(), + builtAt: z.string().nullable(), + createdAt: z.string(), + updatedAt: z.string(), +}) + +export type Sandbox = z.output + +/** A dependency line the server refused, addressed to the row the user typed it on. */ +export const sandboxDependencyIssueSchema = z.object({ + line: z.number().int().positive(), + value: z.string(), + reason: z.string(), +}) + +export type SandboxDependencyIssue = z.output + +const sandboxWorkspaceParamsSchema = z.object({ + id: workspaceIdSchema, +}) + +const sandboxResourceParamsSchema = z.object({ + id: workspaceIdSchema, + sandboxId: z.string().min(1, 'sandboxId is required'), +}) + +export const createSandboxBodySchema = z.object({ + name: sandboxNameSchema, + language: sandboxLanguageSchema, + /** One entry per submitted line, comments and blanks included, so rejections keep their row. */ + dependencies: dependencyListSchema, +}) + +export type CreateSandboxBody = z.input + +export const updateSandboxBodySchema = z + .object({ + name: sandboxNameSchema.optional(), + language: sandboxLanguageSchema.optional(), + dependencies: dependencyListSchema.optional(), + }) + .refine( + (body) => + body.name !== undefined || body.language !== undefined || body.dependencies !== undefined, + { message: 'Provide at least one of name, language, or dependencies' } + ) + +export type UpdateSandboxBody = z.input + +export const listSandboxesContract = defineRouteContract({ + method: 'GET', + path: '/api/workspaces/[id]/sandboxes', + params: sandboxWorkspaceParamsSchema, + response: { + mode: 'json', + schema: z.object({ + sandboxes: z.array(sandboxSchema), + strategy: sandboxStrategySchema, + /** False on a free plan: the list still renders, but read-only behind an upgrade prompt. */ + entitled: z.boolean(), + }), + }, +}) + +export const createSandboxContract = defineRouteContract({ + method: 'POST', + path: '/api/workspaces/[id]/sandboxes', + params: sandboxWorkspaceParamsSchema, + body: createSandboxBodySchema, + response: { + mode: 'json', + schema: z.object({ + sandbox: sandboxSchema, + }), + }, +}) + +export const updateSandboxContract = defineRouteContract({ + method: 'PATCH', + path: '/api/workspaces/[id]/sandboxes/[sandboxId]', + params: sandboxResourceParamsSchema, + body: updateSandboxBodySchema, + response: { + mode: 'json', + schema: z.object({ + sandbox: sandboxSchema, + }), + }, +}) + +export const deleteSandboxContract = defineRouteContract({ + method: 'DELETE', + path: '/api/workspaces/[id]/sandboxes/[sandboxId]', + params: sandboxResourceParamsSchema, + response: { + mode: 'json', + schema: z.object({ + success: z.literal(true), + }), + }, +}) + +export type SandboxListResponse = ContractJsonResponse diff --git a/apps/sim/lib/billing/client/plan-view.ts b/apps/sim/lib/billing/client/plan-view.ts index 6b363d49156..23a30f8a263 100644 --- a/apps/sim/lib/billing/client/plan-view.ts +++ b/apps/sim/lib/billing/client/plan-view.ts @@ -1,4 +1,4 @@ -import { CREDIT_TIERS } from '@/lib/billing/constants' +import { MAX_TIER_CREDITS } from '@/lib/billing/constants' import { getPlanTierCredits, isEnterprise, isFree, isPaid } from '@/lib/billing/plan-helpers' /** @@ -46,8 +46,6 @@ export interface PlanView { showCredits: boolean } -const MAX_TIER_CREDITS = CREDIT_TIERS[1].credits - /** Tier ordering used to derive upgrade/downgrade/highlight relationships. */ const PLAN_RANK: Record = { free: 0, pro: 1, max: 2, enterprise: 3 } diff --git a/apps/sim/lib/billing/client/utils.ts b/apps/sim/lib/billing/client/utils.ts index 91211902e71..b2e28b56ee3 100644 --- a/apps/sim/lib/billing/client/utils.ts +++ b/apps/sim/lib/billing/client/utils.ts @@ -4,7 +4,7 @@ */ import { DEFAULT_FREE_CREDITS } from '@/lib/billing/constants' -import { getPlanTierCredits, isEnterprise, isFree, isPro } from '@/lib/billing/plan-helpers' +import { isFree, isMaxTier, isPro } from '@/lib/billing/plan-helpers' import { hasUsableSubscriptionAccess } from '@/lib/billing/subscriptions/utils' import { USAGE_PILL_COLORS } from './consts' import type { BillingStatus, SubscriptionData, UsageData } from './types' @@ -54,8 +54,9 @@ export function getSubscriptionAccessState( const hasUsableTeamAccess = hasUsablePaidAccess && (status.isOrgScoped || status.isTeam || status.isEnterprise) const hasUsableEnterpriseAccess = hasUsablePaidAccess && status.isEnterprise - const hasUsableMaxAccess = - hasUsablePaidAccess && (getPlanTierCredits(status.plan) >= 25000 || isEnterprise(status.plan)) + // isMaxTier is the same predicate the server gates use, so a Max-gated surface + // can never render unlocked against an API that will refuse it. + const hasUsableMaxAccess = hasUsablePaidAccess && isMaxTier(status.plan) return { ...status, diff --git a/apps/sim/lib/billing/constants.ts b/apps/sim/lib/billing/constants.ts index d4e60cd74ee..a3c793e29d8 100644 --- a/apps/sim/lib/billing/constants.ts +++ b/apps/sim/lib/billing/constants.ts @@ -43,13 +43,24 @@ export const BILLING_LOCK_TIMEOUT_MS = 5_000 * Available credit tiers. Each tier maps a credit amount to the underlying dollar cost. * 1 credit = $0.005, so credits = dollars * 200. */ -export const CREDIT_TIERS = [ - { credits: 6000, dollars: 25, name: 'Pro' }, - { credits: 25000, dollars: 100, name: 'Max' }, -] as const +const PRO_CREDIT_TIER = { credits: 6000, dollars: 25, name: 'Pro' } as const +const MAX_CREDIT_TIER = { credits: 25000, dollars: 100, name: 'Max' } as const + +export const CREDIT_TIERS = [PRO_CREDIT_TIER, MAX_CREDIT_TIER] as const export type CreditTier = (typeof CREDIT_TIERS)[number] +/** + * Credit allocation at which a paid plan enters the Max tier. + * + * Derived from the tier table above so the threshold can never drift from it. + * Do not re-spell this number: every "is this Max?" decision goes through + * `isMaxTier` in `@/lib/billing/plan-helpers`, which reads this constant. The + * server feature gates and the client `hasUsableMaxAccess` derivation share that + * predicate precisely so the UI can never offer a feature the API refuses. + */ +export const MAX_TIER_CREDITS = MAX_CREDIT_TIER.credits + /** * Credits granted per dollar of plan spend. A credit is $0.005, so a dollar * buys 200 — the conversion behind both free-tier and daily-refresh credits. diff --git a/apps/sim/lib/billing/core/access.test.ts b/apps/sim/lib/billing/core/access.test.ts new file mode 100644 index 00000000000..9ae1dee42c9 --- /dev/null +++ b/apps/sim/lib/billing/core/access.test.ts @@ -0,0 +1,160 @@ +/** + * @vitest-environment node + */ +import { queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { getBillingEntityBlockStatus, getEffectiveBillingStatus } from '@/lib/billing/core/access' + +/** A clean `user_stats` row as `getEffectiveBillingStatus` selects it. */ +const UNBLOCKED_STATS = { blocked: false, blockedReason: null } + +describe('getEffectiveBillingStatus', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(() => { + resetDbChainMock() + }) + + it("reports the user's own block without consulting memberships", async () => { + queueTableRows(schemaMock.userStats, [{ blocked: true, blockedReason: 'payment_failed' }]) + + await expect(getEffectiveBillingStatus('user-1')).resolves.toEqual({ + billingBlocked: true, + billingBlockedReason: 'payment_failed', + blockedByOrgOwner: false, + }) + }) + + it('blocks a clean user whose organization owner is delinquent', async () => { + queueTableRows(schemaMock.userStats, [UNBLOCKED_STATS]) + queueTableRows(schemaMock.member, [{ organizationId: 'org-1' }]) + queueTableRows(schemaMock.member, [{ userId: 'owner-1' }]) + queueTableRows(schemaMock.userStats, [{ blocked: true, blockedReason: 'dispute' }]) + + await expect(getEffectiveBillingStatus('user-1')).resolves.toEqual({ + billingBlocked: true, + billingBlockedReason: 'dispute', + blockedByOrgOwner: true, + }) + }) + + it('allows a clean user who owns the organization they belong to', async () => { + queueTableRows(schemaMock.userStats, [UNBLOCKED_STATS]) + queueTableRows(schemaMock.member, [{ organizationId: 'org-1' }]) + queueTableRows(schemaMock.member, [{ userId: 'user-1' }]) + + await expect(getEffectiveBillingStatus('user-1')).resolves.toEqual({ + billingBlocked: false, + billingBlockedReason: null, + blockedByOrgOwner: false, + }) + }) +}) + +describe('getBillingEntityBlockStatus', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + afterAll(() => { + resetDbChainMock() + }) + + describe('personal payer', () => { + it('blocks a payer whose own row is blocked', async () => { + queueTableRows(schemaMock.userStats, [{ blocked: true, blockedReason: 'payment_failed' }]) + + await expect(getBillingEntityBlockStatus({ type: 'user', id: 'payer-1' })).resolves.toEqual({ + billingBlocked: true, + billingBlockedReason: 'payment_failed', + }) + }) + + /** + * The payer's own row is clean — only membership in a delinquent org blocks + * them. Reading `user_stats` directly would report "not blocked" here and + * disagree with every other entitlement gate, because `blockOrgMembers` + * fans out point-in-time and never marks a member who joins after the block. + */ + it('blocks a payer whose own row is clean but whose organization owner is delinquent', async () => { + queueTableRows(schemaMock.userStats, [UNBLOCKED_STATS]) + queueTableRows(schemaMock.member, [{ organizationId: 'org-1' }]) + queueTableRows(schemaMock.member, [{ userId: 'owner-1' }]) + queueTableRows(schemaMock.userStats, [{ blocked: true, blockedReason: 'payment_failed' }]) + + await expect(getBillingEntityBlockStatus({ type: 'user', id: 'payer-1' })).resolves.toEqual({ + billingBlocked: true, + billingBlockedReason: 'payment_failed', + }) + }) + + it('does not widen the payer block shape with blockedByOrgOwner', async () => { + queueTableRows(schemaMock.userStats, [UNBLOCKED_STATS]) + queueTableRows(schemaMock.member, []) + + const status = await getBillingEntityBlockStatus({ type: 'user', id: 'payer-1' }) + + expect(Object.keys(status).sort()).toEqual(['billingBlocked', 'billingBlockedReason']) + }) + + it('allows a clean payer with no memberships', async () => { + queueTableRows(schemaMock.userStats, [UNBLOCKED_STATS]) + queueTableRows(schemaMock.member, []) + + await expect(getBillingEntityBlockStatus({ type: 'user', id: 'payer-1' })).resolves.toEqual({ + billingBlocked: false, + billingBlockedReason: null, + }) + }) + }) + + describe('organization payer', () => { + /** + * An organization's debt is its owner's own debt. Re-deriving through the + * owner's memberships would let some unrelated org the owner belongs to + * block this organization's workspaces. + */ + it("reads the owner's own row and stops there", async () => { + queueTableRows(schemaMock.member, [{ userId: 'owner-1' }]) + queueTableRows(schemaMock.userStats, [{ billingBlocked: false, billingBlockedReason: null }]) + queueTableRows(schemaMock.member, [{ organizationId: 'unrelated-org' }]) + queueTableRows(schemaMock.userStats, [{ blocked: true, blockedReason: 'dispute' }]) + + await expect( + getBillingEntityBlockStatus({ type: 'organization', id: 'org-1' }) + ).resolves.toEqual({ + billingBlocked: false, + billingBlockedReason: null, + }) + }) + + it("blocks when the owner's own row is blocked", async () => { + queueTableRows(schemaMock.member, [{ userId: 'owner-1' }]) + queueTableRows(schemaMock.userStats, [ + { billingBlocked: true, billingBlockedReason: 'payment_failed' }, + ]) + + await expect( + getBillingEntityBlockStatus({ type: 'organization', id: 'org-1' }) + ).resolves.toEqual({ + billingBlocked: true, + billingBlockedReason: 'payment_failed', + }) + }) + + it('is not blocked when the organization has no owner row', async () => { + queueTableRows(schemaMock.member, []) + + await expect( + getBillingEntityBlockStatus({ type: 'organization', id: 'org-1' }) + ).resolves.toEqual({ + billingBlocked: false, + billingBlockedReason: null, + }) + }) + }) +}) diff --git a/apps/sim/lib/billing/core/access.ts b/apps/sim/lib/billing/core/access.ts index 283fb06a593..89a558c202f 100644 --- a/apps/sim/lib/billing/core/access.ts +++ b/apps/sim/lib/billing/core/access.ts @@ -16,7 +16,16 @@ export interface BillingEntityBlockStatus { billingBlockedReason: 'payment_failed' | 'dispute' | null } -async function getUserBillingBlockStatus(userId: string): Promise { +/** + * Reads one user's own `user_stats` row, without re-deriving the block that + * their organization membership would imply. + * + * Only the organization branch of {@link getBillingEntityBlockStatus} wants this + * narrow read: an organization's debt is its owner's own debt, and some other + * org the owner merely belongs to is not this organization's problem. Callers + * asking whether a *user* is blocked want {@link getEffectiveBillingStatus}. + */ +async function getUserStatsBlockStatus(userId: string): Promise { const [stats] = await db .select({ billingBlocked: userStats.billingBlocked, @@ -34,14 +43,27 @@ async function getUserBillingBlockStatus(userId: string): Promise { if (billingEntity.type === 'user') { - return getUserBillingBlockStatus(billingEntity.id) + const { billingBlocked, billingBlockedReason } = await getEffectiveBillingStatus( + billingEntity.id + ) + return { billingBlocked, billingBlockedReason } } const [owner] = await db @@ -58,7 +80,7 @@ export async function getBillingEntityBlockStatus(billingEntity: { return { billingBlocked: false, billingBlockedReason: null } } - return getUserBillingBlockStatus(owner.userId) + return getUserStatsBlockStatus(owner.userId) } /** diff --git a/apps/sim/lib/billing/core/subscription.test.ts b/apps/sim/lib/billing/core/subscription.test.ts index 758658d95a6..79bac7b6619 100644 --- a/apps/sim/lib/billing/core/subscription.test.ts +++ b/apps/sim/lib/billing/core/subscription.test.ts @@ -11,6 +11,7 @@ const { mockCheckEnterprisePlan, mockGetPlanTierCredits, mockHasUsableSubscriptionAccess, + mockGetEffectiveBillingStatus, } = vi.hoisted(() => ({ mockGetHighestPrioritySubscription: vi.fn(), mockGetHighestPriorityPersonalSubscription: vi.fn(), @@ -18,10 +19,11 @@ const { mockCheckEnterprisePlan: vi.fn(), mockGetPlanTierCredits: vi.fn(), mockHasUsableSubscriptionAccess: vi.fn(), + mockGetEffectiveBillingStatus: vi.fn(), })) vi.mock('@/lib/billing/core/access', () => ({ - getEffectiveBillingStatus: vi.fn(), + getEffectiveBillingStatus: mockGetEffectiveBillingStatus, isOrganizationBillingBlocked: vi.fn(), })) @@ -32,7 +34,9 @@ vi.mock('@/lib/billing/core/plan', () => ({ vi.mock('@/lib/billing/plan-helpers', () => ({ getPlanTierCredits: mockGetPlanTierCredits, - isEnterprise: vi.fn().mockReturnValue(false), + isEnterprise: (plan: string | null | undefined) => plan === 'enterprise', + isMaxTier: (plan: string | null | undefined) => + mockGetPlanTierCredits(plan) >= 25000 || plan === 'enterprise', isOrgPlan: (plan: string | null | undefined) => plan === 'enterprise' || plan === 'team' || Boolean(plan?.startsWith('team_')), isPro: vi.fn(), @@ -40,13 +44,14 @@ vi.mock('@/lib/billing/plan-helpers', () => ({ sqlIsPaid: vi.fn(() => ({ type: 'sqlIsPaid' })), })) +/** Mirrors the production sets exactly — a mock that widens them would let a gate regress unnoticed. */ vi.mock('@/lib/billing/subscriptions/utils', () => ({ checkEnterprisePlan: mockCheckEnterprisePlan, checkProPlan: vi.fn(), checkTeamPlan: vi.fn(), - ENTITLED_SUBSCRIPTION_STATUSES: ['active', 'trialing'], + ENTITLED_SUBSCRIPTION_STATUSES: ['active', 'past_due'], hasUsableSubscriptionAccess: mockHasUsableSubscriptionAccess, - USABLE_SUBSCRIPTION_STATUSES: ['active', 'trialing'], + USABLE_SUBSCRIPTION_STATUSES: ['active'], })) vi.mock('@/lib/workspaces/permissions/utils', () => ({ @@ -58,6 +63,7 @@ import { getOrganizationIdForSubscriptionReference, hasPaidSubscription, hasWorkspaceLiveSyncAccess, + hasWorkspaceSandboxAccess, isWorkspaceOnEnterprisePlan, syncSubscriptionPlan, } from '@/lib/billing/core/subscription' @@ -201,23 +207,62 @@ describe('isWorkspaceOnEnterprisePlan', () => { billedAccountUserId: 'owner-1', organizationId: null, }) + mockHasUsableSubscriptionAccess.mockImplementation( + (status: string | null, billingBlocked: boolean) => status === 'active' && !billingBlocked + ) + mockGetEffectiveBillingStatus.mockResolvedValue({ + billingBlocked: false, + billingBlockedReason: null, + blockedByOrgOwner: false, + }) }) it('uses only the exact personal subscription for a personal workspace payer', async () => { mockGetHighestPriorityPersonalSubscription.mockResolvedValue({ referenceId: 'owner-1', plan: 'enterprise', + status: 'active', }) mockGetHighestPrioritySubscription.mockResolvedValue({ referenceId: 'unrelated-org', plan: 'enterprise', + status: 'active', }) - mockCheckEnterprisePlan.mockReturnValue(true) await expect(isWorkspaceOnEnterprisePlan('ws-1')).resolves.toBe(true) expect(mockGetHighestPriorityPersonalSubscription).toHaveBeenCalledWith('owner-1') expect(mockGetHighestPrioritySubscription).not.toHaveBeenCalled() }) + + // The organization branch has always required an `active`, unblocked payer via + // `isOrganizationOnEnterprisePlan`. The personal branch used to skip both checks, + // so a delinquent personal enterprise payer kept the feature while an org in the + // identical state lost it. These two pin the branches to the same policy. + it('denies a personal enterprise payer whose subscription is only past_due', async () => { + mockGetHighestPriorityPersonalSubscription.mockResolvedValue({ + referenceId: 'owner-1', + plan: 'enterprise', + status: 'past_due', + }) + + await expect(isWorkspaceOnEnterprisePlan('ws-1')).resolves.toBe(false) + }) + + it('denies a personal enterprise payer who is billing-blocked through their org owner', async () => { + mockGetHighestPriorityPersonalSubscription.mockResolvedValue({ + referenceId: 'owner-1', + plan: 'enterprise', + status: 'active', + }) + mockGetEffectiveBillingStatus.mockResolvedValue({ + billingBlocked: true, + billingBlockedReason: 'payment_failed', + blockedByOrgOwner: true, + }) + + await expect(isWorkspaceOnEnterprisePlan('ws-1')).resolves.toBe(false) + expect(mockGetEffectiveBillingStatus).toHaveBeenCalledWith('owner-1') + }) }) describe('hasWorkspaceLiveSyncAccess', () => { @@ -231,7 +276,11 @@ describe('hasWorkspaceLiveSyncAccess', () => { mockHasUsableSubscriptionAccess.mockImplementation( (status: string | null, billingBlocked: boolean) => status === 'active' && !billingBlocked ) - dbChainMockFns.limit.mockResolvedValue([{ billingBlocked: false }]) + mockGetEffectiveBillingStatus.mockResolvedValue({ + billingBlocked: false, + billingBlockedReason: null, + blockedByOrgOwner: false, + }) }) it('allows live sync from the exact Max workspace payer', async () => { @@ -264,4 +313,84 @@ describe('hasWorkspaceLiveSyncAccess', () => { expect(mockGetHighestPriorityPersonalSubscription).toHaveBeenCalledWith('workspace-owner') expect(mockGetHighestPrioritySubscription).not.toHaveBeenCalled() }) + + // The payer's own row is clean; only their membership in a delinquent org + // blocks them. Reading `userStats.billingBlocked` directly would let this + // through whenever `blockOrgMembers`' fan-out is stale — a member who joined + // after the block, or whose row a sibling org's unblock already cleared. + it('denies a paid personal payer who is blocked by their org owner', async () => { + mockGetHighestPriorityPersonalSubscription.mockResolvedValue({ + referenceId: 'workspace-owner', + plan: 'pro_25000', + status: 'active', + }) + mockGetPlanTierCredits.mockReturnValue(25000) + mockGetEffectiveBillingStatus.mockResolvedValue({ + billingBlocked: true, + billingBlockedReason: 'payment_failed', + blockedByOrgOwner: true, + }) + + await expect(hasWorkspaceLiveSyncAccess('workspace-host')).resolves.toBe(false) + expect(mockGetEffectiveBillingStatus).toHaveBeenCalledWith('workspace-owner') + }) +}) + +/** + * Sandboxes are an enterprise feature, so `SANDBOXES_ENABLED` must win over the + * plan gate the way `INBOX_ENABLED` does. Both cases run with billing enabled — + * the `!isBillingEnabled` bail would otherwise answer every one of them, hiding + * whether the override is wired at all. + */ +describe('hasWorkspaceSandboxAccess', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isBillingEnabled: true, isHosted: true, isSandboxesEnabled: false }) + mockGetWorkspaceWithOwner.mockResolvedValue({ + id: 'workspace-host', + billedAccountUserId: 'workspace-owner', + organizationId: null, + }) + mockHasUsableSubscriptionAccess.mockImplementation( + (status: string | null, billingBlocked: boolean) => status === 'active' && !billingBlocked + ) + mockGetEffectiveBillingStatus.mockResolvedValue({ + billingBlocked: false, + billingBlockedReason: null, + blockedByOrgOwner: false, + }) + }) + + afterAll(() => setEnvFlags({ isSandboxesEnabled: true })) + + it('grants access from the self-hosted override without resolving a payer', async () => { + setEnvFlags({ isSandboxesEnabled: true }) + + await expect(hasWorkspaceSandboxAccess('workspace-host')).resolves.toBe(true) + expect(mockGetWorkspaceWithOwner).not.toHaveBeenCalled() + expect(mockGetHighestPriorityPersonalSubscription).not.toHaveBeenCalled() + }) + + it('falls back to the Max plan gate when the override is unset', async () => { + mockGetHighestPriorityPersonalSubscription.mockResolvedValue({ + referenceId: 'workspace-owner', + plan: 'pro_25000', + status: 'active', + }) + mockGetPlanTierCredits.mockReturnValue(25000) + + await expect(hasWorkspaceSandboxAccess('workspace-host')).resolves.toBe(true) + expect(mockGetHighestPriorityPersonalSubscription).toHaveBeenCalledWith('workspace-owner') + }) + + it('denies a sub-Max payer when the override is unset', async () => { + mockGetHighestPriorityPersonalSubscription.mockResolvedValue({ + referenceId: 'workspace-owner', + plan: 'pro_6000', + status: 'active', + }) + mockGetPlanTierCredits.mockReturnValue(6000) + + await expect(hasWorkspaceSandboxAccess('workspace-host')).resolves.toBe(false) + }) }) diff --git a/apps/sim/lib/billing/core/subscription.ts b/apps/sim/lib/billing/core/subscription.ts index 9352d882357..d4306a28b34 100644 --- a/apps/sim/lib/billing/core/subscription.ts +++ b/apps/sim/lib/billing/core/subscription.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import { member, organization, subscription, user, userStats } from '@sim/db/schema' +import { member, organization, subscription, user } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { isOrgAdminRole } from '@sim/platform-authz/workspace' import { and, eq, inArray, sql } from 'drizzle-orm' @@ -9,7 +9,7 @@ import { getHighestPrioritySubscription, } from '@/lib/billing/core/plan' import { - getPlanTierCredits, + isMaxTier, isOrgPlan, isEnterprise as isPlanEnterprise, isPro as isPlanPro, @@ -29,6 +29,7 @@ import { isBillingEnabled, isHosted, isInboxEnabled, + isSandboxesEnabled, isSsoEnabled, } from '@/lib/core/config/env-flags' import { getBaseUrl } from '@/lib/core/utils/urls' @@ -500,44 +501,118 @@ export async function hasSSOAccess(userId: string): Promise { } /** - * Check whether a workspace is entitled to the Access Control (Permission Groups) - * feature. Entitlement follows the workspace's `billedAccountUserId`: + * Check whether a workspace is entitled to workspace-scoped enterprise features + * — today, copilot BYOK. Entitlement follows the workspace's billing entity: * - self-hosted override honored via ACCESS_CONTROL_ENABLED, OR * - billing disabled, OR * - the workspace belongs to an enterprise-plan organization (org-mode), OR * - the billed user has an individual enterprise subscription (personal workspace). + * + * Org-scoped Access Control (Permission Groups) gates on + * {@link isOrganizationOnEnterprisePlan} instead — it has no workspace to resolve. */ export async function isWorkspaceOnEnterprisePlan(workspaceId: string): Promise { try { if (!isBillingEnabled) return true if (isAccessControlEnabled && !isHosted) return true - const { getWorkspaceWithOwner } = await import('@/lib/workspaces/permissions/utils') - const ws = await getWorkspaceWithOwner(workspaceId, { includeArchived: true }) - if (!ws) return false + return await hasWorkspaceTierAccess(workspaceId, isPlanEnterprise) + } catch (error) { + logger.error('Error checking workspace enterprise plan status', { error, workspaceId }) + return false + } +} + +/** + * How a workspace tier gate treats subscription status and billing-blocked state. + * + * - `'active-use'` — the payer must hold an `active` subscription and must not be + * billing-blocked. Correct for gating use of a feature. + * - `'retention'` — `active` and `past_due` both count, and block state is + * ignored, so a transient payment failure never triggers destructive teardown of + * already-provisioned infrastructure. Only reconciliation guards want this. + */ +type WorkspaceTierIntent = 'active-use' | 'retention' + +interface WorkspaceTierAccessOptions { + intent?: WorkspaceTierIntent + /** + * Result when the workspace row no longer exists. Teardown guards pass `true` + * so a missing workspace never reads as "safe to destroy". + */ + onMissingWorkspace?: boolean +} + +/** + * Whether the workspace's payer is on a plan satisfying `isTierEntitled`. + * + * Entitlement follows the workspace's billing entity — not the acting user — so + * any workspace admin (including an external member) qualifies when the + * workspace's organization, or its billed account for personal workspaces, is on + * a qualifying plan. + * + * This is the single payer resolution behind every workspace-scoped tier gate. + * Callers supply only the tier predicate and their own feature's env override; + * keeping the org/personal fork here is what stops the gates from drifting apart + * as billing edge cases are handled. + * + * The personal branch reads `getEffectiveBillingStatus`, NOT `userStats.billingBlocked` + * directly. Both express the same shipped policy — `blockOrgMembers` fans a + * delinquent org's block out to every member's own row, so membership in a + * delinquent org blocks you on personal resources too — but the fan-out is a + * point-in-time write and goes stale: nothing marks a member who joins an + * already-blocked org, and `unblockOrgMembers` clears the row even when a second + * delinquent org still covers them. Re-deriving from membership is what makes + * the read agree with the policy in those cases. + */ +async function hasWorkspaceTierAccess( + workspaceId: string, + isTierEntitled: (plan: string) => boolean, + options: WorkspaceTierAccessOptions = {} +): Promise { + const { intent = 'active-use', onMissingWorkspace = false } = options + + const { getWorkspaceWithOwner } = await import('@/lib/workspaces/permissions/utils') + const ws = await getWorkspaceWithOwner(workspaceId, { includeArchived: true }) + if (!ws) return onMissingWorkspace + if (intent === 'retention') { if (ws.organizationId) { - return isOrganizationOnEnterprisePlan(ws.organizationId) + const { getOrganizationSubscription } = await import('@/lib/billing/core/billing') + const orgSub = await getOrganizationSubscription(ws.organizationId) + return !!orgSub && isTierEntitled(orgSub.plan) } const billedSub = await getHighestPriorityPersonalSubscription(ws.billedAccountUserId) - return !!billedSub && checkEnterprisePlan(billedSub) - } catch (error) { - logger.error('Error checking workspace enterprise plan status', { error, workspaceId }) - return false + return !!billedSub && isTierEntitled(billedSub.plan) + } + + if (ws.organizationId) { + const [billingBlocked, orgSub] = await Promise.all([ + isOrganizationBillingBlocked(ws.organizationId), + getOrganizationSubscriptionUsable(ws.organizationId), + ]) + if (!orgSub) return false + if (!hasUsableSubscriptionAccess(orgSub.status, billingBlocked)) return false + return isTierEntitled(orgSub.plan) } -} -const MAX_PLAN_CREDITS = 25000 + const [billedSub, billingStatus] = await Promise.all([ + getHighestPriorityPersonalSubscription(ws.billedAccountUserId), + getEffectiveBillingStatus(ws.billedAccountUserId), + ]) + if (!billedSub) return false + if (!hasUsableSubscriptionAccess(billedSub.status, billingStatus.billingBlocked)) return false + return isTierEntitled(billedSub.plan) +} /** - * Whether a plan tier entitles the inbox (Sim Mailer) feature: a Max tier - * (credits >= 25000, covering `pro_25000` and `team_25000`) or any enterprise - * plan. Subscription status (usable vs entitled) is gated by callers before this - * runs — the predicate is tier-only. + * Whether the workspace's payer is on a usable Max-or-Enterprise subscription. + * Shared by the inbox (Sim Mailer), live sync, and custom sandboxes, which all + * sit on the same entitlement tier. */ -function isInboxEntitledPlan(plan: string): boolean { - return getPlanTierCredits(plan) >= MAX_PLAN_CREDITS || isPlanEnterprise(plan) +async function hasMaxTierWorkspaceAccess(workspaceId: string): Promise { + return hasWorkspaceTierAccess(workspaceId, isMaxTier) } /** @@ -557,24 +632,7 @@ export async function hasWorkspaceInboxAccess(workspaceId: string): Promise { try { if (!isHosted || !isBillingEnabled) return true - - const { getWorkspaceWithOwner } = await import('@/lib/workspaces/permissions/utils') - const ws = await getWorkspaceWithOwner(workspaceId, { includeArchived: true }) - if (!ws) return false - - if (ws.organizationId) { - const [billingBlocked, orgSub] = await Promise.all([ - isOrganizationBillingBlocked(ws.organizationId), - getOrganizationSubscriptionUsable(ws.organizationId), - ]) - if (!orgSub) return false - if (!hasUsableSubscriptionAccess(orgSub.status, billingBlocked)) return false - return isInboxEntitledPlan(orgSub.plan) - } - - const [billedSub, billedStatusRows] = await Promise.all([ - getHighestPriorityPersonalSubscription(ws.billedAccountUserId), - db - .select({ billingBlocked: userStats.billingBlocked }) - .from(userStats) - .where(eq(userStats.userId, ws.billedAccountUserId)) - .limit(1), - ]) - if (!billedSub) return false - if ( - !hasUsableSubscriptionAccess(billedSub.status, Boolean(billedStatusRows[0]?.billingBlocked)) - ) { - return false - } - return isInboxEntitledPlan(billedSub.plan) + return await hasMaxTierWorkspaceAccess(workspaceId) } catch (error) { logger.error('Error checking workspace live sync access', { error, workspaceId }) return false } } +/** + * Checks whether the exact workspace payer can create and edit custom sandboxes. + * + * Same entitlement as the inbox (Sim Mailer), and the same shape: the + * `SANDBOXES_ENABLED` self-hosted override wins first, then a deployment + * without billing is unrestricted, and otherwise the workspace payer must hold + * a usable Max or Enterprise subscription. Builds cost provider compute and + * storage, so this deliberately sits above the plain paid tier. + * + * This gates creating and editing only. Execution deliberately does not consult + * it (see `resolveWorkspaceSandbox`), so a workspace that downgrades keeps + * running the sandboxes it already built. + */ +export async function hasWorkspaceSandboxAccess(workspaceId: string): Promise { + try { + if (isSandboxesEnabled) return true + if (!isBillingEnabled) return true + return await hasMaxTierWorkspaceAccess(workspaceId) + } catch (error) { + logger.error('Error checking workspace sandbox access', { error, workspaceId }) + return false + } +} + /** * Send welcome email for Pro and Team plan subscriptions */ diff --git a/apps/sim/lib/billing/core/workspace-access.ts b/apps/sim/lib/billing/core/workspace-access.ts index 7625dca0850..2470c4a866c 100644 --- a/apps/sim/lib/billing/core/workspace-access.ts +++ b/apps/sim/lib/billing/core/workspace-access.ts @@ -31,6 +31,10 @@ export interface WorkspaceOwnerSubscriptionAccess { /** * Resolves the workspace-selected organization or personal payer and returns * its exact subscription access fields. + * + * Block state comes from {@link getBillingEntityBlockStatus}, so a personal + * payer who is blocked only through a delinquent organization loses paid access + * here too — the same answer the server-side entitlement gates give. */ export async function getWorkspaceOwnerSubscriptionAccess( workspaceId: string diff --git a/apps/sim/lib/billing/enterprise-provisioning.ts b/apps/sim/lib/billing/enterprise-provisioning.ts index 368f7135305..73e9e358ab0 100644 --- a/apps/sim/lib/billing/enterprise-provisioning.ts +++ b/apps/sim/lib/billing/enterprise-provisioning.ts @@ -21,10 +21,11 @@ import { import { acquireUserBillingIdentityLock } from '@/lib/billing/organizations/billing-identity-lock' import { acquireOrganizationMutationLock } from '@/lib/billing/organizations/membership' import { requireStripeClient } from '@/lib/billing/stripe-client' +import { TERMINAL_SUBSCRIPTION_STATUSES } from '@/lib/billing/subscriptions/utils' import { withEnterpriseReconciliationLease } from '@/lib/billing/webhooks/enterprise-reconciliation-lease' import { enqueueOutboxEvent, type OutboxHandler } from '@/lib/core/outbox/service' -const TERMINAL_SUBSCRIPTION_STATUSES = new Set(['canceled', 'incomplete_expired']) +const TERMINAL_STATUSES = new Set(TERMINAL_SUBSCRIPTION_STATUSES) function metadataRecord(value: unknown): Record { return value && typeof value === 'object' && !Array.isArray(value) @@ -33,7 +34,7 @@ function metadataRecord(value: unknown): Record { } function isNonterminalSubscriptionStatus(status: string | null | undefined): boolean { - return !status || !TERMINAL_SUBSCRIPTION_STATUSES.has(status) + return !status || !TERMINAL_STATUSES.has(status) } function subscriptionOrganizationId(metadata: unknown): string | null { diff --git a/apps/sim/lib/billing/max-tier-parity.test.ts b/apps/sim/lib/billing/max-tier-parity.test.ts new file mode 100644 index 00000000000..23f488f246b --- /dev/null +++ b/apps/sim/lib/billing/max-tier-parity.test.ts @@ -0,0 +1,108 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { getSubscriptionAccessState } from '@/lib/billing/client/utils' +import { MAX_TIER_CREDITS } from '@/lib/billing/constants' +import { getPlanTierCredits, isMaxTier } from '@/lib/billing/plan-helpers' + +/** + * Every plan name the product can put on a subscription row. + * + * `pro`/`team` are the legacy bare names; the rest are the `{type}_{credits}` + * form. A new tier added to `CREDIT_TIERS` must be added here too. + */ +const ALL_PLANS = [ + 'free', + 'pro', + 'pro_6000', + 'pro_25000', + 'team', + 'team_6000', + 'team_25000', + 'enterprise', +] as const + +/** + * `isMaxTier` is the sole definition of the Max entitlement. It is consumed by the + * server gates (`hasWorkspaceInboxAccess`, `hasWorkspaceLiveSyncAccess`, + * `hasWorkspaceSandboxAccess`) and by the client `hasUsableMaxAccess` that decides + * whether the settings sidebar renders Sandboxes and Sim Mailer unlocked. + * + * Those two must agree for every plan. When they disagreed, the sidebar offered + * features the API answered 403 for, and the personal-workspace cap inverted so + * Max-for-Teams and Enterprise got a smaller allowance than plain Pro. + */ +describe('Max tier parity', () => { + it('admits exactly the plans at or above the Max credit tier, plus enterprise', () => { + const maxPlans = ALL_PLANS.filter((plan) => isMaxTier(plan)) + + expect(maxPlans).toEqual(['pro_25000', 'team_25000', 'enterprise']) + }) + + it('includes both the individual and team plan at the Max credit allocation', () => { + expect(isMaxTier('pro_25000')).toBe(true) + expect(isMaxTier('team_25000')).toBe(true) + }) + + it('treats enterprise as Max even though it carries no credit suffix', () => { + expect(getPlanTierCredits('enterprise')).toBe(0) + expect(isMaxTier('enterprise')).toBe(true) + }) + + it('excludes every plan below the Max credit allocation', () => { + for (const plan of ['free', 'pro', 'pro_6000', 'team', 'team_6000']) { + expect(isMaxTier(plan)).toBe(false) + } + }) + + it('derives its threshold from the tier table rather than a literal', () => { + expect(isMaxTier(`pro_${MAX_TIER_CREDITS}`)).toBe(true) + expect(isMaxTier(`pro_${MAX_TIER_CREDITS - 1}`)).toBe(false) + }) + + it('handles a missing plan without throwing', () => { + expect(isMaxTier(null)).toBe(false) + expect(isMaxTier(undefined)).toBe(false) + expect(isMaxTier('')).toBe(false) + }) + + /** + * The client gate layers subscription status and billing-blocked state on top of + * the tier predicate, so with an active, unblocked payer the two must return the + * same answer for every plan. A divergence here is the render-unlocked-then-403 + * bug class. + */ + it('matches the client hasUsableMaxAccess derivation for every plan', () => { + for (const plan of ALL_PLANS) { + const client = getSubscriptionAccessState({ + plan, + status: 'active', + isPaid: true, + billingBlocked: false, + }) + + expect(client.hasUsableMaxAccess, `plan ${plan}`).toBe(isMaxTier(plan)) + } + }) + + it('denies Max on the client for a blocked or non-active payer even at the Max tier', () => { + expect( + getSubscriptionAccessState({ + plan: 'pro_25000', + status: 'active', + isPaid: true, + billingBlocked: true, + }).hasUsableMaxAccess + ).toBe(false) + + expect( + getSubscriptionAccessState({ + plan: 'pro_25000', + status: 'past_due', + isPaid: true, + billingBlocked: false, + }).hasUsableMaxAccess + ).toBe(false) + }) +}) diff --git a/apps/sim/lib/billing/plan-helpers.ts b/apps/sim/lib/billing/plan-helpers.ts index e905c671399..99ef6382d92 100644 --- a/apps/sim/lib/billing/plan-helpers.ts +++ b/apps/sim/lib/billing/plan-helpers.ts @@ -16,6 +16,7 @@ import { CREDIT_TIERS, DEFAULT_PRO_TIER_COST_LIMIT, DEFAULT_TEAM_TIER_COST_LIMIT, + MAX_TIER_CREDITS, } from '@/lib/billing/constants' export type PlanCategory = 'free' | 'pro' | 'team' | 'enterprise' @@ -25,8 +26,19 @@ export function isPro(plan: string | null | undefined): boolean { return plan === 'pro' || plan.startsWith('pro_') } -export function isMax(plan: string | null | undefined): boolean { - return isPro(plan) && getPlanTierCredits(plan) >= 25000 +/** + * Whether a plan is Max tier or above: any paid plan at or above the Max credit + * allocation (covering both `pro_25000` and `team_25000`), or any enterprise plan. + * + * Tier-only — subscription status and billing-blocked state are the caller's + * responsibility. This is the single definition of "Max" in the codebase: the + * server feature gates (inbox, live sync, sandboxes), the client + * `hasUsableMaxAccess` derivation, and the personal-workspace cap all route + * through it, so a Max-gated surface can never render unlocked against a server + * that will refuse it. + */ +export function isMaxTier(plan: string | null | undefined): boolean { + return getPlanTierCredits(plan) >= MAX_TIER_CREDITS || isEnterprise(plan) } export function isTeam(plan: string | null | undefined): boolean { @@ -107,7 +119,7 @@ export function getPlanType(plan: string | null | undefined): PlanCategory { export function getPlanTypeForLimits(plan: string | null | undefined): PlanCategory { if (plan === 'pro' || plan === 'team') return getPlanType(plan) if (isPro(plan) || isTeam(plan)) { - return getPlanTierCredits(plan) >= 25000 ? 'team' : 'pro' + return getPlanTierCredits(plan) >= MAX_TIER_CREDITS ? 'team' : 'pro' } return getPlanType(plan) } @@ -136,13 +148,17 @@ export function getValidPlanNames(type: 'pro' | 'team'): string[] { /** * SQL-level plan filters for Drizzle queries. * These are the SQL equivalents of the JS helpers above. + * + * The `_` in the plan-name separator is escaped because it is a single-character + * wildcard in SQL `LIKE`. Unescaped, `'pro_%'` would also match `proX…`, making + * these filters admit a wider set than their JS counterparts. */ export function sqlIsPro(column: AnyColumn): SQL | undefined { - return or(eq(column, 'pro'), like(column, 'pro_%')) + return or(eq(column, 'pro'), like(column, 'pro\\_%')) } export function sqlIsTeam(column: AnyColumn): SQL | undefined { - return or(eq(column, 'team'), like(column, 'team_%')) + return or(eq(column, 'team'), like(column, 'team\\_%')) } export function sqlIsPaid(column: AnyColumn): SQL | undefined { diff --git a/apps/sim/lib/billing/webhooks/invoices.test.ts b/apps/sim/lib/billing/webhooks/invoices.test.ts index 8dcdcd52cdd..cdd2d96088c 100644 --- a/apps/sim/lib/billing/webhooks/invoices.test.ts +++ b/apps/sim/lib/billing/webhooks/invoices.test.ts @@ -57,7 +57,7 @@ vi.mock('@/lib/billing/stripe-client', () => stripeClientMock) vi.mock('@/lib/billing/stripe-payment-method', () => stripePaymentMethodMock) vi.mock('@/lib/billing/subscriptions/utils', () => ({ - ENTITLED_SUBSCRIPTION_STATUSES: ['active', 'trialing', 'past_due'], + ENTITLED_SUBSCRIPTION_STATUSES: ['active', 'past_due'], })) vi.mock('@/lib/billing/utils/decimal', () => ({ diff --git a/apps/sim/lib/core/config/enterprise-entitlements.ts b/apps/sim/lib/core/config/enterprise-entitlements.ts index ea82fea86c4..9ab83057f37 100644 --- a/apps/sim/lib/core/config/enterprise-entitlements.ts +++ b/apps/sim/lib/core/config/enterprise-entitlements.ts @@ -33,6 +33,7 @@ export type EnterpriseFeature = | 'forking' | 'inbox' | 'organizations' + | 'sandboxes' | 'sessionPolicies' | 'sso' | 'whitelabeling' @@ -58,6 +59,12 @@ export type EnterpriseFeature = * delete pass is gated here. Defaulting it on would start expiring logs on * upgrade against plan defaults the operator never chose. * + * `sandboxes` is `true` for the mirror-image reason: its gate already returns + * true whenever billing is off, exactly like `inbox`. A `false` here would make + * the settings-nav override disagree with the gate that actually answers the + * request. Self-hosted builds run on the operator's own E2B/Daytona + * credentials, so there is no Sim-side cost to withhold. + * * Do not "tidy" these to a uniform value. Each records observed prior behavior, * and changing one silently alters a live deployment on upgrade. */ @@ -69,6 +76,7 @@ export const ENTERPRISE_FEATURE_LEGACY_DEFAULTS: Readonly { + const key = `route:${bucketName}:workspace:${workspaceId}` + const { allowed, resetAt } = await rateLimiter.checkRateLimitDirect(key, config) + if (allowed) return null + logger.warn('Workspace rate limit exceeded', { bucket: bucketName, workspaceId }) + return buildRateLimitResponse(resetAt) +} + /** * Apply a per-user limit when a userId is present, else fall back to per-IP. * Use for routes whose auth path may legitimately resolve without a userId diff --git a/apps/sim/lib/execution/remote-sandbox/build-errors.ts b/apps/sim/lib/execution/remote-sandbox/build-errors.ts new file mode 100644 index 00000000000..16ca66c6322 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/build-errors.ts @@ -0,0 +1,154 @@ +import { registryFor, type SandboxLanguage } from '@/lib/execution/remote-sandbox/sandbox-spec' + +/** + * Classified reasons a dependency set failed to materialize. The code is what + * gets stored and reported on; the rendered copy is what the settings UI shows, + * so a user never meets a raw pip traceback as the primary message. + */ +export type SandboxBuildErrorCode = + | 'package_not_found' + | 'version_not_found' + | 'resolution_conflict' + | 'build_timeout' + | 'install_failed' + | 'provider_error' + | 'rate_limited' + +export interface SandboxBuildError { + code: SandboxBuildErrorCode + /** User-facing copy. Never a traceback. */ + message: string + /** The dependency the failure named, when one could be extracted. */ + dependency?: string +} + +/** How much installer output to keep as the disclosure detail. */ +export const BUILD_LOG_TAIL_CHARS = 4000 + +/** + * Keeps the tail of installer output rather than the head: pip and npm print the + * actual failure last, after pages of resolution noise. + */ +export function tailBuildLog(output: string): string { + const trimmed = output.trim() + if (trimmed.length <= BUILD_LOG_TAIL_CHARS) return trimmed + return `…\n${trimmed.slice(-BUILD_LOG_TAIL_CHARS)}` +} + +interface Matcher { + code: SandboxBuildErrorCode + match: RegExp + render: (language: SandboxLanguage, groups: RegExpMatchArray) => SandboxBuildError +} + +/** + * Ordered installer-output matchers. Order matters: a version miss also mentions + * the package name, so the more specific pattern has to win. + */ +const MATCHERS: readonly Matcher[] = [ + { + // `Could not find a version that satisfies the requirement foo==9 (from versions: 1, 2)` + code: 'version_not_found', + match: + /could not find a version that satisfies the requirement ([^\s(]+)[^\n(]*\(from versions:\s*([^)]*)\)/i, + render: (language, groups) => { + const available = groups[2]?.trim() + const known = available && available.toLowerCase() !== 'none' + if (!known) { + return { + code: 'package_not_found', + dependency: groups[1], + message: `Package "${groups[1]}" was not found on ${registryFor(language)}.`, + } + } + return { + code: 'version_not_found', + dependency: groups[1], + message: `"${groups[1]}" has no version matching that specifier. Available: ${available}.`, + } + }, + }, + { + // npm: `notarget No matching version found for axios@^99.0.0` + code: 'version_not_found', + match: /no matching version found for (\S+)/i, + render: (_language, groups) => ({ + code: 'version_not_found', + dependency: groups[1], + message: `"${groups[1]}" has no version matching that specifier.`, + }), + }, + { + // npm: `404 Not Found - GET https://registry.npmjs.org/does-not-exist` + code: 'package_not_found', + match: /404 not found[^\n]*?\/([^/\s'"]+)\s*$/im, + render: (language, groups) => ({ + code: 'package_not_found', + dependency: groups[1], + message: `Package "${groups[1]}" was not found on ${registryFor(language)}.`, + }), + }, + { + code: 'package_not_found', + match: /no matching distribution found for (\S+)/i, + render: (language, groups) => ({ + code: 'package_not_found', + dependency: groups[1], + message: `Package "${groups[1]}" was not found on ${registryFor(language)}.`, + }), + }, + { + code: 'resolution_conflict', + match: + /(resolutionimpossible|conflicting dependencies|eresolve unable to resolve dependency tree|the conflict is caused by)/i, + render: () => ({ + code: 'resolution_conflict', + message: + 'These dependencies require incompatible versions of a shared package. Relax or remove a version pin.', + }), + }, + { + code: 'rate_limited', + match: /\b429\b|too many requests|rate ?limit/i, + render: (language) => ({ + code: 'rate_limited', + message: `${registryFor(language)} rate-limited the install. Try again in a few minutes.`, + }), + }, +] + +/** + * Maps raw installer output onto the taxonomy. Anything unrecognized falls back + * to `install_failed` with the log retained rather than being dropped — an + * unclassifiable failure is still a failure the user has to be able to debug. + */ +export function classifyInstallOutput( + language: SandboxLanguage, + output: string +): SandboxBuildError { + for (const matcher of MATCHERS) { + const groups = output.match(matcher.match) + if (groups) return matcher.render(language, groups) + } + return { + code: 'install_failed', + message: 'Installation failed. See the log below.', + } +} + +/** Copy for the failures that have no installer output to classify. */ +export function buildTimeoutError(minutes: number): SandboxBuildError { + return { + code: 'build_timeout', + message: `The build took longer than ${minutes} minutes and was stopped.`, + } +} + +export function providerBuildError(detail?: string): SandboxBuildError { + return { + code: 'provider_error', + message: detail + ? `The sandbox provider rejected the build. Try again shortly. (${detail})` + : 'The sandbox provider rejected the build. Try again shortly.', + } +} diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index 5c3f0adb39d..1fbb85b74be 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -10,6 +10,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { CodeLanguage } from '@/lib/execution/languages' const { + mockResolveSandbox, + mockProvisionRuntime, mockEnv, mockE2BCreate, mockE2BRunCode, @@ -30,6 +32,8 @@ const { mockGetSessionCommand, mockDeleteSession, } = vi.hoisted(() => ({ + mockResolveSandbox: vi.fn(), + mockProvisionRuntime: vi.fn(), mockEnv: { SANDBOX_PROVIDER: 'e2b' as string | undefined, PI_SANDBOX_LIFETIME_MS: undefined as string | undefined, @@ -69,6 +73,12 @@ vi.mock('@daytona/sdk', () => ({ }, })) vi.mock('@/lib/core/config/env', () => ({ env: mockEnv })) +vi.mock('@/lib/execution/remote-sandbox/resolve', () => ({ + resolveWorkspaceSandbox: mockResolveSandbox, + provisionRuntimeDependencies: mockProvisionRuntime, + invalidateSandboxResolution: vi.fn(), + RUNTIME_INSTALL_TIMEOUT_MS: 240_000, +})) import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' import { @@ -77,6 +87,8 @@ import { SIM_RESULT_PREFIX, withPiSandbox, } from '@/lib/execution/remote-sandbox' +import { daytonaProvider } from '@/lib/execution/remote-sandbox/daytona' +import { e2bProvider } from '@/lib/execution/remote-sandbox/e2b' import { PI_SANDBOX_MAX_LIFETIME_MS, PI_SANDBOX_MIN_LIFETIME_MS, @@ -132,6 +144,8 @@ beforeEach(() => { delete: mockDelete, }) mockExecuteCommand.mockResolvedValue({ result: '', exitCode: 0 }) + mockResolveSandbox.mockResolvedValue(null) + mockProvisionRuntime.mockResolvedValue(undefined) mockExecuteSessionCommand.mockResolvedValue({ cmdId: 'cmd_1' }) mockGetSessionCommand.mockResolvedValue({ exitCode: 0 }) }) @@ -322,6 +336,149 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => { }) }) +describe('custom dependency sets', () => { + it.each(PROVIDERS)('honors imageRef for code executions [%s]', async (provider) => { + useProvider(provider) + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + mockResolveSandbox.mockResolvedValue({ + id: 'sbx-1', + name: 'etl', + language: CodeLanguage.Python, + dependencies: ['pandas'], + strategy: 'prebuilt', + imageRef: 'sim-sbx-abc', + }) + + await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + if (provider === 'e2b') { + expect(mockE2BCreate).toHaveBeenCalledWith('sim-sbx-abc', expect.anything()) + } else { + expect(mockDaytonaCreate).toHaveBeenCalledWith( + expect.objectContaining({ snapshot: 'sim-sbx-abc' }) + ) + } + }) + + it.each(PROVIDERS)('refuses to let a sandbox displace the doc image [%s]', async (provider) => { + useProvider(provider) + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + // Even if resolution somehow yields a ref, the provider gates on the kind. + mockResolveSandbox.mockResolvedValue({ + id: 'sbx-1', + name: 'etl', + language: CodeLanguage.Python, + dependencies: ['pandas'], + strategy: 'prebuilt', + imageRef: 'sim-sbx-abc', + }) + + await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 1000, + sandboxKind: 'doc', + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + if (provider === 'e2b') { + expect(mockE2BCreate).toHaveBeenCalledWith('mothership-docs', expect.anything()) + } else { + expect(mockDaytonaCreate).toHaveBeenCalledWith( + expect.objectContaining({ snapshot: 'mothership-docs:v1' }) + ) + } + }) + + it.each(PROVIDERS)( + 'spends the runtime install out of the caller budget, not on top of it [%s]', + async (provider) => { + useProvider(provider) + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + mockResolveSandbox.mockResolvedValue({ + id: 'sbx-1', + name: 'etl', + language: CodeLanguage.Python, + dependencies: ['pandas'], + strategy: 'runtime', + }) + + // Well under the 240s ceiling, so the caller's budget is what binds. + await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 60_000, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + // 60s budget minus the 15s reserved for the code itself. + expect(mockProvisionRuntime).toHaveBeenCalledWith(expect.anything(), expect.anything(), { + timeoutMs: 45_000, + }) + } + ) + + it.each(PROVIDERS)( + 'caps the install at its own ceiling when the budget is generous [%s]', + async (provider) => { + useProvider(provider) + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + mockResolveSandbox.mockResolvedValue({ + id: 'sbx-1', + name: 'etl', + language: CodeLanguage.Python, + dependencies: ['pandas'], + strategy: 'runtime', + }) + + await executeInSandbox({ + code: 'x', + language: CodeLanguage.Python, + timeoutMs: 900_000, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + expect(mockProvisionRuntime).toHaveBeenCalledWith(expect.anything(), expect.anything(), { + timeoutMs: 240_000, + }) + } + ) + + it.each(PROVIDERS)('uses the env template when nothing is selected [%s]', async (provider) => { + useProvider(provider) + stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`) + + await executeInSandbox({ code: 'x', language: CodeLanguage.Python, timeoutMs: 1000 }) + + if (provider === 'e2b') { + expect(mockE2BCreate).toHaveBeenCalledWith('mothership-shell', expect.anything()) + } else { + expect(mockDaytonaCreate).toHaveBeenCalledWith( + expect.objectContaining({ snapshot: 'mothership-shell:v1' }) + ) + } + // No selection means no install step, on either strategy. + expect(mockE2BCommandsRun).not.toHaveBeenCalled() + expect(mockExecuteCommand).not.toHaveBeenCalled() + }) + + it('declares one strategy per provider, and only the prebuilt one can build', () => { + expect(e2bProvider.dependencyStrategy).toBe('prebuilt') + expect(e2bProvider.images).toBeDefined() + expect(daytonaProvider.dependencyStrategy).toBe('runtime') + expect(daytonaProvider.images).toBeUndefined() + }) +}) + describe('provider selection', () => { it('routes by SANDBOX_PROVIDER, defaulting to E2B when unset', async () => { stubCodeRun('e2b', `${SIM_RESULT_PREFIX}null`) diff --git a/apps/sim/lib/execution/remote-sandbox/daytona.ts b/apps/sim/lib/execution/remote-sandbox/daytona.ts index c1710757754..3f2c9047476 100644 --- a/apps/sim/lib/execution/remote-sandbox/daytona.ts +++ b/apps/sim/lib/execution/remote-sandbox/daytona.ts @@ -20,7 +20,13 @@ function toSeconds(timeoutMs: number): number { return Math.max(1, Math.ceil(timeoutMs / 1000)) } -function snapshotFor(kind: SandboxKind): string { +function snapshotFor(kind: SandboxKind, imageRef?: string): string { + // An operator-supplied snapshot may only displace the general shell image. + // `doc` and `pi` keep their vetted snapshots unconditionally, so nothing a + // workspace configures can land under the doc compiler or the coding agent. + if (imageRef && (kind === 'code' || kind === 'shell')) { + return imageRef + } // Mirrors the E2B provider's fail-closed behaviour: never let LLM-authored code // run in a provider default image just because a snapshot id is unset. const snapshot = @@ -56,7 +62,10 @@ class DaytonaSandboxHandle implements SandboxHandle { return this.sandbox.id } - async runCode(code: string, options: { timeoutMs: number }): Promise { + async runCode( + code: string, + options: { timeoutMs: number; envs?: Record } + ): Promise { // Python goes through CodeInterpreter because it reports a structured // `{ name, value, traceback }` error — the same shape E2B returns, which the // route's line-offset error formatting depends on. CodeInterpreter is @@ -65,6 +74,7 @@ class DaytonaSandboxHandle implements SandboxHandle { if (this.language === CodeLanguage.Python) { const result = await this.sandbox.codeInterpreter.runCode(code, { timeout: toSeconds(options.timeoutMs), + ...(options.envs ? { envs: options.envs } : {}), }) return { text: '', @@ -80,7 +90,11 @@ class DaytonaSandboxHandle implements SandboxHandle { } } - const result = await this.sandbox.process.codeRun(code, undefined, toSeconds(options.timeoutMs)) + const result = await this.sandbox.process.codeRun( + code, + options.envs ? { env: options.envs } : undefined, + toSeconds(options.timeoutMs) + ) const output: string = result.result ?? '' if (result.exitCode !== 0) { // `process.codeRun` has no structured error channel — the interpreter's @@ -242,14 +256,24 @@ function lastNonEmptyLine(output: string): string { return lines.length > 0 ? lines[lines.length - 1] : 'Execution failed' } +/** + * Daytona takes the `runtime` strategy and exposes no image builder. + * + * Prebuilt images are not available at any tier: the 30-active-snapshot quota + * lives on the organization rather than the plan, snapshots auto-deactivate + * after 14 days unused, and raising the quota needs Daytona support. Since + * builds are content-addressed and shared, 30 would be the ceiling for all of + * Sim rather than per workspace. Installing per execution consumes no quota. + */ export const daytonaProvider: SandboxProvider = { id: 'daytona', + dependencyStrategy: 'runtime', async create(kind: SandboxKind, options?: CreateSandboxOptions): Promise { const apiKey = env.DAYTONA_API_KEY if (!apiKey) { throw new Error('DAYTONA_API_KEY is required when the Daytona sandbox provider is selected') } - const snapshot = snapshotFor(kind) + const snapshot = snapshotFor(kind, options?.imageRef) const language = options?.language ?? CodeLanguage.Python logger.info('Creating Daytona sandbox', { kind, snapshot }) diff --git a/apps/sim/lib/execution/remote-sandbox/e2b.ts b/apps/sim/lib/execution/remote-sandbox/e2b.ts index 2f5dd11934d..24f6041d68a 100644 --- a/apps/sim/lib/execution/remote-sandbox/e2b.ts +++ b/apps/sim/lib/execution/remote-sandbox/e2b.ts @@ -1,21 +1,68 @@ -import type { Sandbox as E2BSandbox } from '@e2b/code-interpreter' +import type { Sandbox as E2BSandbox, Template as E2BTemplate } from '@e2b/code-interpreter' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { env } from '@/lib/core/config/env' import { CodeLanguage } from '@/lib/execution/languages' +import { classifyInstallOutput, tailBuildLog } from '@/lib/execution/remote-sandbox/build-errors' +import { + quoteDependency, + type SandboxSpec, + SIM_DEPS_DIR, + SIM_NODE_MODULES_DIR, +} from '@/lib/execution/remote-sandbox/sandbox-spec' import type { CreateSandboxOptions, RunCommandOptions, SandboxCodeResult, SandboxCommandResult, SandboxHandle, + SandboxImageBuild, + SandboxImageBuilder, + SandboxImageBuildStatus, SandboxKind, SandboxProvider, } from '@/lib/execution/remote-sandbox/types' const logger = createLogger('E2BSandboxProvider') -function templateFor(kind: SandboxKind): string | undefined { +/** The `Template` callable, threaded in so the SDK stays a dynamic import. */ +type TemplateFactory = typeof E2BTemplate + +/** + * Prefix for content-addressed dependency templates. Refs are account-global, so + * the prefix keeps them clearly attributable and out of the way of hand-built + * templates. + */ +const SANDBOX_TEMPLATE_PREFIX = 'sim-sbx-' + +/** + * How much of the spec hash goes into the ref. Template names are one namespace + * across the whole E2B account, and nothing detects a collision — two specs + * sharing a prefix would build to the same name and silently swap package sets + * between workspaces. 32 hex chars (128 bits) puts that out of reach. + */ +const SPEC_HASH_REF_LENGTH = 32 + +export function sandboxImageRef(specHash: string): string { + return `${SANDBOX_TEMPLATE_PREFIX}${specHash.slice(0, SPEC_HASH_REF_LENGTH)}` +} + +/** + * E2B's control-plane base. `ConnectionConfig` derives this internally but is + * not re-exported by `@e2b/code-interpreter`, and `e2b` itself is only a + * transitive dependency, so the one endpoint the SDK omits resolves it here. + */ +function e2bApiUrl(): string { + return `https://api.${env.E2B_DOMAIN || 'e2b.app'}` +} + +function templateFor(kind: SandboxKind, imageRef?: string): string | undefined { + // A workspace dependency set may only displace the general shell template. + // `doc` and `pi` keep their vetted images unconditionally, so a user's package + // list can never land under the document compiler or the coding agent. + if (imageRef && (kind === 'code' || kind === 'shell')) { + return imageRef + } // Document generation uses a dedicated template (python-pptx/docx/openpyxl/ // reportlab + fonts); shell/code execution use the general shell template. // Doc fails closed: never run LLM-authored Python in E2B's default template @@ -47,10 +94,14 @@ class E2BSandboxHandle implements SandboxHandle { return this.sandbox.sandboxId } - async runCode(code: string, options: { timeoutMs: number }): Promise { + async runCode( + code: string, + options: { timeoutMs: number; envs?: Record } + ): Promise { const execution = await this.sandbox.runCode(code, { language: this.language === CodeLanguage.Python ? 'python' : 'javascript', timeoutMs: options.timeoutMs, + ...(options.envs ? { envs: options.envs } : {}), }) // Kernel stream entries are chunks, not lines — each already carries its own @@ -111,14 +162,151 @@ class E2BSandboxHandle implements SandboxHandle { } } +/** + * Composes the dependency layer over the general shell template. `fromTemplate` + * means each build stacks only the new layer and inherits everything else, which + * is what makes a template per workspace sandbox cheap. + * + * Dependencies reach the installer as individually quoted argv entries. The SDK + * renders `pipInstall`/`npmInstall` by joining packages with spaces into one + * shell string, so an unquoted `django>=5.0` would redirect stdout into a file + * named `=5.0` and silently install an unpinned Django — hence + * {@link quoteDependency} and the hand-composed `runCmd`. + */ +function composeDependencyTemplate( + spec: SandboxSpec, + template: TemplateFactory, + baseTemplate: string +) { + const packages = spec.dependencies.map(quoteDependency).join(' ') + + if (spec.language === CodeLanguage.Python) { + return template() + .fromTemplate(baseTemplate) + .runCmd(`pip install --no-input --disable-pip-version-check ${packages}`, { user: 'root' }) + } + + // Unlike pip, an npm install is not automatically importable — Node resolves + // from NODE_PATH or the install location. Installing into a fixed prefix and + // baking NODE_PATH is what makes `require`/`import` find these packages. + return template() + .fromTemplate(baseTemplate) + .makeDir(SIM_DEPS_DIR, { user: 'root' }) + .runCmd(`npm install --prefix ${SIM_DEPS_DIR} --no-audit --no-fund --omit=dev ${packages}`, { + user: 'root', + }) + .setEnvs({ NODE_PATH: SIM_NODE_MODULES_DIR }) +} + +const e2bImages: SandboxImageBuilder = { + async startBuild(spec: SandboxSpec, specHash: string): Promise { + const apiKey = env.E2B_API_KEY + if (!apiKey) { + throw new Error('E2B_API_KEY is required to build a sandbox image') + } + // Resolved before the dynamic import so a misconfigured deployment fails + // without ever reaching the network. + const baseTemplate = env.MOTHERSHIP_E2B_TEMPLATE_ID + if (!baseTemplate) { + throw new Error('Sandbox builds are not configured (MOTHERSHIP_E2B_TEMPLATE_ID is unset)') + } + const imageRef = sandboxImageRef(specHash) + + const { Template } = await import('@e2b/code-interpreter') + const template = composeDependencyTemplate(spec, Template, baseTemplate) + const build = await Template.buildInBackground(template, imageRef, { apiKey }) + + logger.info('Started E2B sandbox image build', { + imageRef, + buildId: build.buildId, + language: spec.language, + dependencyCount: spec.dependencies.length, + }) + return { imageRef, buildId: build.buildId, providerImageId: build.templateId } + }, + + async getBuildStatus( + build: SandboxImageBuild, + spec: SandboxSpec + ): Promise { + const apiKey = env.E2B_API_KEY + if (!apiKey) { + throw new Error('E2B_API_KEY is required to poll a sandbox image build') + } + const { Template } = await import('@e2b/code-interpreter') + const status = await Template.getBuildStatus( + { templateId: build.providerImageId ?? build.imageRef, buildId: build.buildId }, + { apiKey } + ) + + const logs = status.logEntries.map((entry) => entry.message).join('\n') + if (status.status === 'ready') return { status: 'ready' } + if (status.status === 'error') { + return { + status: 'failed', + error: classifyInstallOutput(spec.language, `${status.reason?.message ?? ''}\n${logs}`), + logs: tailBuildLog(logs), + } + } + return { status: 'building' } + }, + + /** + * The SDK surfaces no template delete, so the retention sweep calls the REST + * endpoint directly. A non-2xx throws so the caller leaves the registry row in + * place and retries, rather than orphaning the remote template. + */ + /** + * E2B maps a 404 from the control plane to `NotFoundError`, and the only resource + * a create request names is the template — so a 404 there means the ref is gone. + * + * Its two subclasses are excluded because they describe other calls entirely: a + * missing file inside a running sandbox, or a sandbox that has already exited. + * Neither is reachable from a create. Everything else — auth, rate limit, + * transport — is deliberately not a missing image, since rebuilding on those + * would turn a provider outage into a build storm. + */ + async isMissingImage(error: unknown): Promise { + const { FileNotFoundError, NotFoundError, SandboxNotFoundError } = await import( + '@e2b/code-interpreter' + ) + return ( + error instanceof NotFoundError && + !(error instanceof SandboxNotFoundError) && + !(error instanceof FileNotFoundError) + ) + }, + + async deleteImage(build: SandboxImageBuild): Promise { + const apiKey = env.E2B_API_KEY + if (!apiKey) { + throw new Error('E2B_API_KEY is required to delete a sandbox image') + } + const templateId = build.providerImageId ?? build.imageRef + const response = await fetch(`${e2bApiUrl()}/templates/${encodeURIComponent(templateId)}`, { + method: 'DELETE', + headers: { 'X-API-KEY': apiKey }, + // The retention sweep deletes sequentially; without this one hung + // connection to the control plane stalls the whole cron handler. + signal: AbortSignal.timeout(30_000), + }) + // A template that is already gone is the state we wanted. + if (!response.ok && response.status !== 404) { + throw new Error(`E2B refused to delete template ${templateId} (${response.status})`) + } + }, +} + export const e2bProvider: SandboxProvider = { id: 'e2b', + dependencyStrategy: 'prebuilt', + images: e2bImages, async create(kind: SandboxKind, options?: CreateSandboxOptions): Promise { const apiKey = env.E2B_API_KEY if (!apiKey) { throw new Error('E2B_API_KEY is required when E2B is enabled') } - const templateName = templateFor(kind) + const templateName = templateFor(kind, options?.imageRef) logger.info('Creating E2B sandbox', { kind, template: templateName || '(default)' }) // E2B reaps a sandbox after `timeoutMs` (default five minutes). Omitted diff --git a/apps/sim/lib/execution/remote-sandbox/image-registry.test.ts b/apps/sim/lib/execution/remote-sandbox/image-registry.test.ts new file mode 100644 index 00000000000..c3b970684d9 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/image-registry.test.ts @@ -0,0 +1,438 @@ +/** + * @vitest-environment node + * + * Builds are addressed by content and shared across workspaces, so releasing one + * eagerly is only safe while nothing else references it. These cases pin that + * guard down, plus the failure modes that must leave the retention sweep a job to + * finish rather than losing the image silently. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDelete, mockInsert, mockSelect, mockDeleteImage, mockProviderStrategy } = vi.hoisted( + () => ({ + mockDelete: vi.fn(), + mockInsert: vi.fn(), + mockSelect: vi.fn(), + mockDeleteImage: vi.fn(), + mockProviderStrategy: { current: 'prebuilt' as 'prebuilt' | 'runtime' }, + }) +) + +vi.mock('@sim/db', () => ({ + db: { delete: mockDelete, insert: mockInsert, select: mockSelect }, +})) + +vi.mock('@sim/db/schema', () => ({ + sandboxImage: { + id: 'id', + provider: 'provider', + specHash: 'spec_hash', + status: 'status', + imageRef: 'image_ref', + buildId: 'build_id', + providerImageId: 'provider_image_id', + lastUsedAt: 'last_used_at', + createdAt: 'created_at', + updatedAt: 'updated_at', + }, + workspaceSandbox: { id: 'id', specHash: 'spec_hash' }, +})) + +vi.mock('drizzle-orm', () => ({ + and: (...args: unknown[]) => args, + eq: (...args: unknown[]) => args, + inArray: (...args: unknown[]) => args, + lt: (...args: unknown[]) => args, + notInArray: (...args: unknown[]) => args, + or: (...args: unknown[]) => args, + sql: (...args: unknown[]) => args, +})) + +vi.mock('@/lib/execution/remote-sandbox/provider', () => ({ + resolveProvider: () => ({ + id: 'e2b', + get dependencyStrategy() { + return mockProviderStrategy.current + }, + get images() { + return mockProviderStrategy.current === 'prebuilt' + ? { deleteImage: mockDeleteImage } + : undefined + }, + }), +})) + +import { + cleanupSandboxImages, + ensureSandboxImage, + FAILED_BUILD_RETRY_COOLDOWN_MS, + releaseSandboxImage, + sandboxBuildIdempotencyKey, +} from '@/lib/execution/remote-sandbox/image-registry' + +const READY_IMAGE = { + id: 'img-1', + status: 'ready', + spec: { language: 'python', dependencies: ['pandas'] }, + imageRef: 'sim-sbx-abc', + buildId: 'build-1', + providerImageId: 'tmpl-1', +} + +beforeEach(() => { + vi.clearAllMocks() + mockProviderStrategy.current = 'prebuilt' + mockDelete.mockReturnValue({ where: () => ({ returning: () => Promise.resolve([]) }) }) + mockInsert.mockReturnValue({ values: () => ({ onConflictDoNothing: () => Promise.resolve() }) }) + // Default: nothing re-adopted the hash, so the post-delete rebuild is a no-op and + // cases that are not about it stay on one `delete`. + mockSelect.mockReturnValue({ + from: () => ({ where: () => ({ limit: () => Promise.resolve([]) }) }), + }) + mockDeleteImage.mockResolvedValue(undefined) +}) + +/** Serializes the mocked predicate tree so a clause can be asserted by shape. */ +function predicateText(predicate: unknown): string { + if (predicate == null) return '' + if (Array.isArray(predicate)) return predicate.map(predicateText).join(' ') + if (typeof predicate === 'object') return JSON.stringify(predicate) + return String(predicate) +} + +/** + * True once a build upsert has been issued. Distinguished from the restore path by + * the conflict clause: a rebuild is `onConflictDoUpdate`, a restore is + * `onConflictDoNothing`. + */ +function captureUpsert(): () => boolean { + let seen = false + mockInsert.mockReturnValue({ + values: () => ({ + onConflictDoUpdate: () => { + seen = true + return { returning: () => Promise.resolve([]) } + }, + onConflictDoNothing: () => Promise.resolve(), + }), + }) + return () => seen +} + +/** Captures the conditional-delete predicate and what the claim resolves to. */ +function stubClaim(rows: unknown[]): () => unknown { + let captured: unknown + mockDelete.mockReturnValue({ + where: (predicate: unknown) => { + captured = predicate + return { returning: () => Promise.resolve(rows) } + }, + }) + return () => captured +} + +describe('releaseSandboxImage', () => { + it('deletes the provider image once the row is claimed', async () => { + stubClaim([READY_IMAGE]) + + await releaseSandboxImage('hash-1') + + expect(mockDeleteImage).toHaveBeenCalledWith({ + imageRef: 'sim-sbx-abc', + buildId: 'build-1', + providerImageId: 'tmpl-1', + }) + }) + + /** + * The bystander case: two workspaces declaring the same package list share one + * build, so one workspace's delete must not take the image out from under the + * other. The guard is the conditional delete itself — reading references in a + * separate statement left a window, spanning a provider network call, in which + * another workspace could adopt the hash between the check and the delete. + */ + it('claims only when no sandbox references the hash, in one statement', async () => { + const read = stubClaim([READY_IMAGE]) + + await releaseSandboxImage('hash-1') + + const clause = predicateText(read()) + expect(clause).toContain('not exists') + expect(clause).toContain('workspace_sandbox') + }) + + it('excludes an in-flight build from the claim rather than racing it', async () => { + const read = stubClaim([READY_IMAGE]) + + await releaseSandboxImage('hash-1') + + const clause = predicateText(read()) + expect(clause).toContain('pending') + expect(clause).toContain('building') + }) + + it('touches the provider only when the claim actually took a row', async () => { + stubClaim([]) + + await releaseSandboxImage('hash-1') + + expect(mockDeleteImage).not.toHaveBeenCalled() + }) + + it('no-ops under a runtime provider, which has no images to release', async () => { + mockProviderStrategy.current = 'runtime' + stubClaim([READY_IMAGE]) + + await releaseSandboxImage('hash-1') + + expect(mockDelete).not.toHaveBeenCalled() + expect(mockDeleteImage).not.toHaveBeenCalled() + }) + + /** + * Claiming before the provider call means a refusal would otherwise strand a + * template nothing points at, so the row goes back and the sweep inherits it. + */ + it('restores the claimed row when the provider refuses', async () => { + stubClaim([READY_IMAGE]) + mockDeleteImage.mockRejectedValue(new Error('E2B unreachable')) + + await expect(releaseSandboxImage('hash-1')).resolves.toBeUndefined() + + expect(mockInsert).toHaveBeenCalledTimes(1) + }) + + /** + * The adopter's row is new and healthy-looking, so nothing else would notice the + * template went out from under it — resolution only repairs a missing or + * `failed` row, and a `failed` one waits out the cooldown first. + */ + it('rebuilds a hash re-adopted while the provider delete was in flight', async () => { + stubClaim([READY_IMAGE]) + mockSelect.mockReturnValue({ + from: () => ({ where: () => ({ limit: () => Promise.resolve([{ id: 'img-new' }]) }) }), + }) + const enqueued = captureUpsert() + + await releaseSandboxImage('hash-1') + + expect(mockDeleteImage).toHaveBeenCalledTimes(1) + expect(enqueued()).toBe(true) + }) + + it('does not rebuild when nothing re-adopted the hash', async () => { + stubClaim([READY_IMAGE]) + mockSelect.mockReturnValue({ + from: () => ({ where: () => ({ limit: () => Promise.resolve([]) }) }), + }) + const enqueued = captureUpsert() + + await releaseSandboxImage('hash-1') + + expect(enqueued()).toBe(false) + }) + + /** + * Restoring belongs to a refused delete and nothing else. Once the template is + * gone, putting the row back would recreate a `ready` row pointing at nothing — + * the one state resolution cannot repair. + */ + it('does not restore the row when the post-delete rebuild fails', async () => { + stubClaim([READY_IMAGE]) + mockSelect.mockImplementation(() => { + throw new Error('registry unreachable') + }) + + await releaseSandboxImage('hash-1') + + expect(mockDeleteImage).toHaveBeenCalledTimes(1) + expect(mockInsert).not.toHaveBeenCalled() + }) + + /** + * A row claiming `ready` against a deleted template is the state resolution + * cannot repair, so a rebuild that does not take must not leave one behind. + * Dropping it converts the adopter into the missing-row case, which the next + * execution fixes on its own. + */ + it('drops the dead row when the rebuild cannot be scheduled', async () => { + stubClaim([READY_IMAGE]) + mockSelect.mockImplementation(() => { + throw new Error('registry unreachable') + }) + + await releaseSandboxImage('hash-1') + + // The claim itself plus the dead-row cleanup. + expect(mockDelete).toHaveBeenCalledTimes(2) + }) + + it('skips the provider when the claimed row never had an image', async () => { + stubClaim([{ ...READY_IMAGE, imageRef: null }]) + + await releaseSandboxImage('hash-1') + + expect(mockDeleteImage).not.toHaveBeenCalled() + expect(mockInsert).not.toHaveBeenCalled() + }) +}) + +/** + * The sweep reads a batch of candidates and then works through them a chunk of + * network calls at a time, so minutes can pass between the query and any one + * delete. Whether a row still qualifies has to be decided by the claim, not by + * that earlier read. + */ +describe('cleanupSandboxImages', () => { + const CANDIDATE = { id: 'img-1', specHash: 'hash-1', imageRef: 'sim-sbx-abc' } + + /** + * Nomination query only — later selects (the re-adoption check) resolve empty, so + * a candidate is not mistaken for its own adopter. + */ + function stubCandidates(rows: unknown[]) { + mockSelect.mockReturnValue({ + from: () => ({ where: () => ({ limit: () => Promise.resolve([]) }) }), + }) + mockSelect.mockReturnValueOnce({ + from: () => ({ where: () => ({ limit: () => Promise.resolve(rows) }) }), + }) + } + + it('skips a candidate a workspace adopted after the query ran', async () => { + stubCandidates([CANDIDATE]) + stubClaim([]) + + const result = await cleanupSandboxImages(30) + + expect(mockDeleteImage).not.toHaveBeenCalled() + expect(result).toEqual({ deleted: 0, failed: 0 }) + }) + + it('deletes the image for a candidate that still qualifies at claim time', async () => { + stubCandidates([CANDIDATE]) + stubClaim([READY_IMAGE]) + + const result = await cleanupSandboxImages(30) + + expect(mockDeleteImage).toHaveBeenCalledTimes(1) + expect(result).toEqual({ deleted: 1, failed: 0 }) + }) + + it('restores the row and counts a failure when the provider refuses', async () => { + stubCandidates([CANDIDATE]) + stubClaim([READY_IMAGE]) + mockDeleteImage.mockRejectedValue(new Error('E2B unreachable')) + + const result = await cleanupSandboxImages(30) + + expect(mockInsert).toHaveBeenCalledTimes(1) + expect(result).toEqual({ deleted: 0, failed: 1 }) + }) + + it('keeps the retention cutoff in the claim, not just the candidate query', async () => { + stubCandidates([CANDIDATE]) + const read = stubClaim([READY_IMAGE]) + + await cleanupSandboxImages(30) + + expect(hasTimeBound(read())).toBe(true) + }) +}) + +/** True when any leaf of the mocked predicate tree is a `Date`, i.e. a time bound. */ +function hasTimeBound(predicate: unknown): boolean { + if (predicate instanceof Date) return true + return Array.isArray(predicate) && predicate.some(hasTimeBound) +} + +/** + * The repair path fires once per execution, so re-claiming a failed row on sight + * would let a per-minute schedule enqueue a per-minute build of a package list + * that will never resolve. A save is a person asking again and must not wait. + */ +describe('ensureSandboxImage failed-build cooldown', () => { + const SPEC = { language: 'python' as const, dependencies: ['pandas'] } + + /** Captures the conflict predicate; the empty `returning` means "nothing claimed". */ + function captureSetWhere(): () => unknown { + let captured: unknown + mockInsert.mockReturnValue({ + values: () => ({ + onConflictDoUpdate: (config: { setWhere: unknown }) => { + captured = config.setWhere + return { returning: () => Promise.resolve([]) } + }, + }), + }) + return () => captured + } + + it('bounds the failed branch by time when a cooldown is requested', async () => { + const read = captureSetWhere() + + await ensureSandboxImage(SPEC, 'hash-1', { + minFailureAgeMs: FAILED_BUILD_RETRY_COOLDOWN_MS, + }) + + const [failedBranch] = read() as unknown[] + expect(hasTimeBound(failedBranch)).toBe(true) + }) + + it('leaves the failed branch unbounded for a save, so a person retries at once', async () => { + const read = captureSetWhere() + + await ensureSandboxImage(SPEC, 'hash-1') + + const [failedBranch] = read() as unknown[] + expect(hasTimeBound(failedBranch)).toBe(false) + }) + + /** + * A row that reached `ready` before the delete landed is the permanent case: + * resolution repairs a missing or `failed` row, never one claiming to be ready, + * so its dead `imageRef` would survive until someone re-saved the sandbox. + */ + it('reclaims a ready row when the image is known to be gone', async () => { + const read = captureSetWhere() + + await ensureSandboxImage(SPEC, 'hash-1', { imageKnownGone: true }) + + const branch = predicateText((read() as unknown[])[0]) + expect(branch).toContain('status') + // Excludes only in-flight statuses, so `ready` and `failed` both qualify. + expect(branch).toContain('pending') + expect(branch).not.toContain('failed') + }) +}) + +/** + * The claim already collapses concurrent saves, so the trigger key only has to + * distinguish attempts. Keyed by spec alone it suppressed the next legitimate one + * instead — Trigger.dev returns the finished run, the row stays `pending` with no + * worker, and nothing can re-claim it until it goes stale. + */ +describe('sandboxBuildIdempotencyKey', () => { + it('differs between two attempts at the same spec', () => { + const first = sandboxBuildIdempotencyKey('e2b', 'hash-1', new Date(1_000)) + const second = sandboxBuildIdempotencyKey('e2b', 'hash-1', new Date(2_000)) + + expect(first).not.toBe(second) + }) + + it('still collapses a duplicate delivery of one attempt', () => { + const attemptAt = new Date(1_000) + + expect(sandboxBuildIdempotencyKey('e2b', 'hash-1', attemptAt)).toBe( + sandboxBuildIdempotencyKey('e2b', 'hash-1', attemptAt) + ) + }) + + it('keeps providers apart for the same content address', () => { + const attemptAt = new Date(1_000) + + expect(sandboxBuildIdempotencyKey('e2b', 'hash-1', attemptAt)).not.toBe( + sandboxBuildIdempotencyKey('daytona', 'hash-1', attemptAt) + ) + }) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/image-registry.ts b/apps/sim/lib/execution/remote-sandbox/image-registry.ts new file mode 100644 index 00000000000..be6dc008fae --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/image-registry.ts @@ -0,0 +1,592 @@ +import { db } from '@sim/db' +import { sandboxImage } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { backoffWithJitter } from '@sim/utils/retry' +import { and, eq, inArray, lt, notInArray, or, type SQL, sql } from 'drizzle-orm' +import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { runDetached } from '@/lib/core/utils/background' +import { + buildTimeoutError, + providerBuildError, + type SandboxBuildError, +} from '@/lib/execution/remote-sandbox/build-errors' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' +import { invalidateSandboxResolution } from '@/lib/execution/remote-sandbox/resolve' +import type { SandboxSpec } from '@/lib/execution/remote-sandbox/sandbox-spec' +import type { + SandboxImageBuild, + SandboxImageBuilder, + SandboxImageStatus, +} from '@/lib/execution/remote-sandbox/types' + +const logger = createLogger('SandboxImageRegistry') + +/** Ceiling on how long one build may run before it is called failed. */ +export const BUILD_POLL_CAP_MS = 15 * 60 * 1000 + +/** A `building` row older than this is assumed abandoned and may be re-claimed. */ +const STALE_BUILD_MS = BUILD_POLL_CAP_MS * 2 + +const POLL_BASE_MS = 3_000 +const POLL_MAX_MS = 20_000 + +export interface SandboxImageBuildPayload { + provider: string + specHash: string +} + +/** + * Collapses concurrent saves of the same spec into one build. Content-addressed, + * so two workspaces declaring identical dependencies share the key as well as + * the resulting image. + */ +export function sandboxBuildIdempotencyKey( + provider: string, + specHash: string, + attemptAt: Date +): string { + return `sandbox-image-${provider}-${specHash}-${attemptAt.getTime()}` +} + +/** + * How long a failed build is left alone when the caller is an automatic repair + * rather than a person. Long enough that a per-minute schedule cannot turn a + * permanently broken package list into a per-minute build, short enough that a + * transient registry outage recovers within the hour. + */ +export const FAILED_BUILD_RETRY_COOLDOWN_MS = 10 * 60 * 1000 + +export interface EnsureSandboxImageOptions { + /** + * Ignore a `failed` row younger than this instead of re-claiming it. + * + * Omitted means retry immediately, which is right for a save: a person editing + * a package list is explicitly asking for another attempt. Automatic callers + * pass {@link FAILED_BUILD_RETRY_COOLDOWN_MS} — they fire once per execution, + * and a build that fails in seconds would otherwise be re-enqueued by every + * run of a scheduled workflow forever. + */ + minFailureAgeMs?: number + /** + * Reclaim a `ready` row as well as a failed one. + * + * Only the release path sets this, and only once it has deleted an image out + * from under a hash something re-adopted. That row looks healthy while its + * `imageRef` points at nothing, and resolution cannot tell: it repairs a row + * that is missing or `failed`, never one claiming to be ready, so without this + * the sandbox stays broken until someone re-saves it by hand. + * + * An in-flight build is still left alone. It either recreates the template it + * was already building or fails into the normal repair path, and resetting it + * would only add a duplicate build. + */ + imageKnownGone?: boolean +} + +/** + * Which settled row a save may re-claim: any of them when the image is known to be + * gone, otherwise a failed one — immediately for a person, after the cooldown for + * an automatic caller. + */ +function settledRebuildBranch(options: EnsureSandboxImageOptions): SQL | undefined { + if (options.imageKnownGone) { + return notInArray(sandboxImage.status, ['pending', 'building']) + } + if (options.minFailureAgeMs) { + return and( + eq(sandboxImage.status, 'failed'), + lt(sandboxImage.updatedAt, new Date(Date.now() - options.minFailureAgeMs)) + ) + } + return eq(sandboxImage.status, 'failed') +} + +/** + * Ensures a build exists for `spec`, enqueueing one only when the registry has + * no row or the last attempt failed. A `ready` or in-flight row is left alone, + * which is what makes an unchanged save cost nothing. + * + * No-ops under a `runtime` provider: it installs per execution and never touches + * this registry. + */ +export async function ensureSandboxImage( + spec: SandboxSpec, + specHash: string, + options: EnsureSandboxImageOptions = {} +): Promise { + const provider = resolveProvider() + if (provider.dependencyStrategy !== 'prebuilt' || !provider.images) return + // Nothing to install means nothing to build — and `pip install` with no + // requirement exits non-zero, so a build here would fail permanently. + if (spec.dependencies.length === 0) return + + const inserted = await db + .insert(sandboxImage) + .values({ + id: generateId(), + provider: provider.id, + specHash, + spec, + status: 'pending', + }) + .onConflictDoUpdate({ + target: [sandboxImage.provider, sandboxImage.specHash], + set: { + status: 'pending', + errorCode: null, + errorMessage: null, + errorDetail: null, + updatedAt: new Date(), + }, + // A failed build is retryable, immediately for a person and after a cooldown + // for an automatic caller. `pending` and `building` become re-claimable once + // stale: an enqueue that threw (provider outage), a detached run that died + // with the process, or a worker killed mid-build would otherwise strand the + // row forever, and no later save could revive it because the content address + // never changes. + setWhere: or( + settledRebuildBranch(options), + and( + inArray(sandboxImage.status, ['pending', 'building']), + lt(sandboxImage.updatedAt, new Date(Date.now() - STALE_BUILD_MS)) + ) + ), + }) + .returning({ + id: sandboxImage.id, + status: sandboxImage.status, + updatedAt: sandboxImage.updatedAt, + }) + + // No row returned means the conflict target matched but `setWhere` rejected the + // update — an existing `ready`, `pending`, or `building` row. Nothing to do. + if (inserted.length === 0) return + + await enqueueSandboxImageBuild({ provider: provider.id, specHash }, inserted[0].updatedAt) +} + +async function enqueueSandboxImageBuild( + payload: SandboxImageBuildPayload, + attemptAt: Date +): Promise { + if (!isTriggerDevEnabled) { + runDetached('sandbox-image-build', () => runSandboxImageBuild(payload)) + return + } + // Dynamically imported so Trigger.dev stays out of the web bundle's static graph. + const [{ sandboxImageBuildTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([ + import('@/background/sandbox-image-build'), + import('@trigger.dev/sdk'), + import('@/lib/core/async-jobs/region'), + ]) + await tasks.trigger('sandbox-image-build', payload, { + // Keyed by the claim, not by the spec alone. Concurrent saves are already + // collapsed by the conditional update above — only one caller gets a row back, + // so only one reaches here. A spec-only key would instead suppress the *next* + // legitimate attempt: Trigger.dev returns the finished run rather than starting + // one, leaving the row `pending` with no worker and unclaimable until it goes + // stale. That is half an hour of a save-to-retry doing nothing. + idempotencyKey: sandboxBuildIdempotencyKey(payload.provider, payload.specHash, attemptAt), + // Still short, so even a duplicate delivery of one attempt cannot linger. + idempotencyKeyTTL: '5m', + tags: [`sandboxSpec:${payload.specHash}`], + region: await resolveTriggerRegion(), + }) +} + +async function writeFailure( + payload: SandboxImageBuildPayload, + error: SandboxBuildError, + detail?: string +): Promise { + await db + .update(sandboxImage) + .set({ + status: 'failed', + errorCode: error.code, + errorMessage: error.message, + errorDetail: detail ?? null, + updatedAt: new Date(), + }) + .where( + and(eq(sandboxImage.provider, payload.provider), eq(sandboxImage.specHash, payload.specHash)) + ) + invalidateSandboxResolution() +} + +/** + * Drives one build to a terminal state: claim the row, start the provider build, + * poll with backoff to {@link BUILD_POLL_CAP_MS}, then write `ready` or `failed`. + * + * The claim is conditional on the row still being `pending`, so a re-delivered + * task (Trigger.dev at-least-once, or a detached retry) is a no-op rather than a + * second concurrent build. + */ +export async function runSandboxImageBuild(payload: SandboxImageBuildPayload): Promise { + const provider = resolveProvider() + if (provider.id !== payload.provider || !provider.images) { + logger.warn('Skipping sandbox image build for a provider this deployment does not serve', { + requested: payload.provider, + active: provider.id, + }) + return + } + + const claimed = await db + .update(sandboxImage) + .set({ status: 'building', updatedAt: new Date() }) + .where( + and( + eq(sandboxImage.provider, payload.provider), + eq(sandboxImage.specHash, payload.specHash), + eq(sandboxImage.status, 'pending') + ) + ) + .returning({ spec: sandboxImage.spec }) + + if (claimed.length === 0) { + logger.info('Sandbox image build already claimed, skipping', { specHash: payload.specHash }) + return + } + const spec = claimed[0].spec as SandboxSpec + + let build: SandboxImageBuild + try { + build = await provider.images.startBuild(spec, payload.specHash) + } catch (error) { + logger.error('Failed to start sandbox image build', toError(error)) + await writeFailure(payload, providerBuildError(getErrorMessage(error))) + return + } + + await db + .update(sandboxImage) + .set({ + imageRef: build.imageRef, + buildId: build.buildId, + providerImageId: build.providerImageId ?? null, + updatedAt: new Date(), + }) + .where( + and(eq(sandboxImage.provider, payload.provider), eq(sandboxImage.specHash, payload.specHash)) + ) + + const deadline = Date.now() + BUILD_POLL_CAP_MS + for (let attempt = 1; Date.now() < deadline; attempt++) { + await sleep(backoffWithJitter(attempt, null, { baseMs: POLL_BASE_MS, maxMs: POLL_MAX_MS })) + let status: Awaited> + try { + status = await provider.images.getBuildStatus(build, spec) + } catch (error) { + // A transient poll failure is not a build failure — keep polling until the + // cap, and let the timeout be the thing that gives up. + logger.warn('Sandbox image build poll failed', { specHash: payload.specHash, error }) + continue + } + + if (status.status === 'ready') { + await db + .update(sandboxImage) + .set({ + status: 'ready', + errorCode: null, + errorMessage: null, + errorDetail: null, + updatedAt: new Date(), + }) + .where( + and( + eq(sandboxImage.provider, payload.provider), + eq(sandboxImage.specHash, payload.specHash) + ) + ) + invalidateSandboxResolution() + logger.info('Sandbox image build ready', { + specHash: payload.specHash, + imageRef: build.imageRef, + }) + return + } + + if (status.status === 'failed') { + await writeFailure(payload, status.error ?? providerBuildError(), status.logs ?? undefined) + logger.warn('Sandbox image build failed', { + specHash: payload.specHash, + code: status.error?.code, + }) + return + } + } + + await writeFailure(payload, buildTimeoutError(BUILD_POLL_CAP_MS / 60_000)) + logger.warn('Sandbox image build timed out', { specHash: payload.specHash }) +} + +/** + * Deletes the provider image behind a spec hash once no sandbox references it. + * + * Called when a sandbox is deleted, and when an edit re-points one at a new + * content address — both leave the previous build unreferenced. Without it the + * image sits in provider storage until the retention sweep, which is up to + * `SANDBOX_IMAGE_RETENTION_DAYS` of paying to store something nothing can select. + * + * Builds are keyed by content, not by workspace, so two workspaces declaring the + * same package list share one image and deleting on the strength of one + * workspace's action would break the other. The reference check is therefore part + * of the same statement that removes the row: reading references first and + * deleting second left a window — wide, because a provider delete is a network + * call — in which another workspace could adopt the hash, inherit a `ready` row, + * and have its next run fail against a template already on its way out. Winning + * the conditional delete is what proves nothing referenced the hash. + * + * Claiming the row before the provider call means a provider that then refuses + * would strand a template nothing points at, so the row is put back and the + * retention sweep inherits the retry. An in-flight build is left alone rather + * than raced; the sweep collects it once it settles. + * + * Best-effort by contract: failures are logged and swallowed, because this runs + * after the mutation it follows has already committed and must never turn a + * successful delete into an error. + */ +export async function releaseSandboxImage(specHash: string): Promise { + const provider = resolveProvider() + if (provider.dependencyStrategy !== 'prebuilt' || !provider.images) return + + try { + const outcome = await claimAndDeleteImage(provider.id, provider.images, specHash) + if (outcome === 'released') { + invalidateSandboxResolution() + logger.info('Released unreferenced sandbox image', { specHash }) + } + } catch (error) { + logger.warn('Failed to release sandbox image; the retention sweep will retry', { + specHash, + error: getErrorMessage(error), + }) + } +} + +type ImageClaimOutcome = 'released' | 'skipped' | 'failed' + +/** + * Rebuilds a hash that was adopted while its image was being deleted. + * + * Claiming removes the row, so between that and the provider call finishing a + * workspace can declare the same package list, get a fresh row, and start a build + * under the same content-derived `imageRef` — which the in-flight delete then + * removes. The window is inherent: the registry row and the provider template are + * two systems with no shared transaction, so it can be made small but not zero. + * + * What is avoidable is the adopter being left broken. Its row is new and + * healthy-looking, so nothing else would notice — and a row that reached `ready` + * before the delete landed is worse than slow, it is permanent: resolution repairs + * a row that is missing or `failed`, never one claiming to be ready, so its + * `imageRef` would point at nothing until someone re-saved the sandbox by hand. + * `imageKnownGone` is what lets this reclaim that row. A build still in flight is + * left alone, since it either recreates the template or fails into the normal + * repair path. + */ +async function rebuildIfReadopted( + providerId: string, + specHash: string, + spec: SandboxSpec +): Promise { + try { + const [readopted] = await db + .select({ id: sandboxImage.id }) + .from(sandboxImage) + .where(and(eq(sandboxImage.provider, providerId), eq(sandboxImage.specHash, specHash))) + .limit(1) + if (!readopted) return + + logger.warn('Sandbox image was re-adopted mid-delete; rebuilding it now', { specHash }) + await ensureSandboxImage(spec, specHash, { imageKnownGone: true }) + } catch (error) { + // The template is gone and the rebuild did not take, so the adopter's row now + // claims a `ready` image that does not exist — the one state resolution cannot + // repair, because it only rebuilds a row that is missing or `failed`. Dropping + // the row converts that into the missing case, which the next execution fixes + // on its own. Swallowing instead would leave the sandbox broken until someone + // re-saved it by hand. + logger.warn('Failed to rebuild a re-adopted sandbox image; dropping the dead row', { + specHash, + error: getErrorMessage(error), + }) + try { + await db + .delete(sandboxImage) + .where(and(eq(sandboxImage.provider, providerId), eq(sandboxImage.specHash, specHash))) + } catch (cleanupError) { + logger.error('Could not drop a sandbox image row pointing at a deleted template', { + specHash, + error: getErrorMessage(cleanupError), + }) + } + } +} + +/** + * Takes ownership of a spec hash's registry row and deletes the image behind it. + * + * Both callers that remove an image go through here, because the ordering is the + * whole contract and having written it twice is what let the two paths drift: + * + * 1. The unreferenced check is part of the `DELETE`, not a query before it. + * Winning the delete is the proof nothing referenced the hash. Reading first + * and deleting second leaves a window — spanning a provider network call — + * where another workspace declares the same package list, inherits the `ready` + * row without rebuilding, and loses the template underneath it. + * 2. The provider delete runs only after the claim, so two sweeps or a sweep and + * a release cannot both issue it. + * 3. A provider that refuses gets the row put back, since claiming first would + * otherwise strand a template nothing points at and no later pass would find + * it. Restoring is what keeps the retry on the sweep. + * + * `extraConditions` lets the sweep add its retention cutoff to the same claim + * rather than trusting the candidate query it ran earlier. + */ +async function claimAndDeleteImage( + providerId: string, + images: SandboxImageBuilder, + specHash: string, + extraConditions: SQL[] = [] +): Promise { + const [claimed] = await db + .delete(sandboxImage) + .where( + and( + eq(sandboxImage.provider, providerId), + eq(sandboxImage.specHash, specHash), + notInArray(sandboxImage.status, ['pending', 'building']), + sql`not exists (select 1 from workspace_sandbox ws where ws.spec_hash = ${specHash})`, + ...extraConditions + ) + ) + .returning({ + id: sandboxImage.id, + spec: sandboxImage.spec, + status: sandboxImage.status, + imageRef: sandboxImage.imageRef, + buildId: sandboxImage.buildId, + providerImageId: sandboxImage.providerImageId, + }) + if (!claimed) return 'skipped' + if (!claimed.imageRef) return 'released' + + try { + await images.deleteImage({ + imageRef: claimed.imageRef, + buildId: claimed.buildId ?? '', + providerImageId: claimed.providerImageId ?? undefined, + }) + } catch (error) { + await db + .insert(sandboxImage) + .values({ + id: claimed.id, + provider: providerId, + specHash, + spec: claimed.spec, + status: claimed.status as SandboxImageStatus, + imageRef: claimed.imageRef, + buildId: claimed.buildId, + providerImageId: claimed.providerImageId, + }) + .onConflictDoNothing() + logger.warn('Provider refused a sandbox image delete; restored the row for retry', { + specHash, + error: getErrorMessage(error), + }) + return 'failed' + } + + // Deliberately past the catch above. The template is gone for good by now, so a + // failure here must not restore the row: that path puts back a `ready` row whose + // imageRef points at nothing, which is the one state resolution cannot repair. + await rebuildIfReadopted(providerId, specHash, claimed.spec as SandboxSpec) + return 'released' +} + +/** Most rows one sweep will touch. The next run picks up whatever is left. */ +const CLEANUP_BATCH_LIMIT = 200 + +/** Provider deletes issued at once. Bounded so a sweep cannot stampede the API. */ +const CLEANUP_CONCURRENCY = 8 + +/** + * Removes build rows that no `workspace_sandbox` still references and that have + * gone unused past the retention window. + * + * The query below only nominates candidates. Every row is re-checked and claimed + * atomically by {@link claimAndDeleteImage} before its image is touched, because + * this sweep reads up to {@link CLEANUP_BATCH_LIMIT} rows and then works through + * them a chunk of network calls at a time — leaving minutes in which a workspace + * could declare one of those package lists, inherit the `ready` row, and lose the + * template underneath it. A candidate that stops qualifying in that window simply + * fails its claim and is skipped. + * + * A provider delete that fails restores the row, so the next sweep retries rather + * than orphaning a remote template nothing points at any more. + * + * Progress is committed per chunk. A sweep that is killed part-way — by a route + * timeout or a redeploy — therefore keeps what it already deleted, instead of + * re-issuing every provider delete next run and never draining the backlog. + */ +export async function cleanupSandboxImages(retentionDays: number): Promise<{ + deleted: number + failed: number +}> { + const provider = resolveProvider() + if (provider.dependencyStrategy !== 'prebuilt' || !provider.images) { + return { deleted: 0, failed: 0 } + } + const images = provider.images + + const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000) + const stale = await db + .select({ + id: sandboxImage.id, + specHash: sandboxImage.specHash, + imageRef: sandboxImage.imageRef, + buildId: sandboxImage.buildId, + providerImageId: sandboxImage.providerImageId, + }) + .from(sandboxImage) + .where( + and( + eq(sandboxImage.provider, provider.id), + sql`coalesce(${sandboxImage.lastUsedAt}, ${sandboxImage.createdAt}) < ${cutoff}`, + sql`not exists (select 1 from workspace_sandbox ws where ws.spec_hash = ${sandboxImage.specHash})` + ) + ) + .limit(CLEANUP_BATCH_LIMIT) + + if (stale.length === CLEANUP_BATCH_LIMIT) { + logger.info('Sandbox image sweep hit its batch limit; the rest waits for the next run', { + limit: CLEANUP_BATCH_LIMIT, + }) + } + + let deleted = 0 + let failed = 0 + + for (let offset = 0; offset < stale.length; offset += CLEANUP_CONCURRENCY) { + const chunk = stale.slice(offset, offset + CLEANUP_CONCURRENCY) + const outcomes = await Promise.all( + chunk.map((row) => + claimAndDeleteImage(provider.id, images, row.specHash, [ + sql`coalesce(${sandboxImage.lastUsedAt}, ${sandboxImage.createdAt}) < ${cutoff}`, + ]) + ) + ) + + deleted += outcomes.filter((outcome) => outcome === 'released').length + failed += outcomes.filter((outcome) => outcome === 'failed').length + } + + if (deleted > 0) invalidateSandboxResolution() + return { deleted, failed } +} diff --git a/apps/sim/lib/execution/remote-sandbox/index.ts b/apps/sim/lib/execution/remote-sandbox/index.ts index 1bbc80dabbe..c99a6333e55 100644 --- a/apps/sim/lib/execution/remote-sandbox/index.ts +++ b/apps/sim/lib/execution/remote-sandbox/index.ts @@ -1,9 +1,14 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { env } from '@/lib/core/config/env' -import { daytonaProvider } from '@/lib/execution/remote-sandbox/daytona' -import { e2bProvider } from '@/lib/execution/remote-sandbox/e2b' import { resolvePiSandboxLifetimeMs } from '@/lib/execution/remote-sandbox/pi-lifetime' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' +import { + provisionRuntimeDependencies, + type ResolvedSandbox, + RUNTIME_INSTALL_TIMEOUT_MS, + repairMissingSandboxImage, + resolveWorkspaceSandbox, +} from '@/lib/execution/remote-sandbox/resolve' import type { CreateSandboxOptions, SandboxCommandResult, @@ -12,8 +17,6 @@ import type { SandboxFile, SandboxHandle, SandboxKind, - SandboxProvider, - SandboxProviderId, SandboxShellExecutionRequest, } from '@/lib/execution/remote-sandbox/types' @@ -26,43 +29,6 @@ export type { const logger = createLogger('RemoteSandbox') -/** - * The known sandbox providers. Keyed by {@link SandboxProviderId}, so adding an - * adapter is one entry here plus one member on the id union — the type makes an - * unhandled provider a compile error, not a runtime surprise. - */ -const PROVIDERS: Record = { - e2b: e2bProvider, - daytona: daytonaProvider, -} - -const DEFAULT_PROVIDER: SandboxProviderId = 'e2b' - -/** - * Resolves which provider serves this execution from the `SANDBOX_PROVIDER` env - * var (defaulting to {@link DEFAULT_PROVIDER}). - * - * Selection is deliberately resolved ONCE, before the sandbox is created, and is - * never revisited mid-execution: user code has side effects (HTTP calls, S3 - * writes, DB mutations), so retrying a partially-executed run on another provider - * could duplicate them. Changing providers is a config change — set - * `SANDBOX_PROVIDER` and redeploy; in-flight executions are unaffected. - */ -function resolveProvider(): SandboxProvider { - // Normalize casing identically to env-flags' availability gate — otherwise a - // value like `Daytona` would pass the gate (which lowercases) but miss this - // lowercase-keyed map and throw at create time. - const configured = env.SANDBOX_PROVIDER?.toLowerCase() - if (!configured) return PROVIDERS[DEFAULT_PROVIDER] - const provider = PROVIDERS[configured as SandboxProviderId] - if (!provider) { - throw new Error( - `Unknown SANDBOX_PROVIDER "${env.SANDBOX_PROVIDER}" (expected one of: ${Object.keys(PROVIDERS).join(', ')})` - ) - } - return provider -} - async function createSandbox( kind: SandboxKind, options?: CreateSandboxOptions @@ -73,6 +39,31 @@ async function createSandbox( return sandbox } +/** + * Creates a sandbox, turning "that image is gone" into a rebuild rather than a + * failure the author has to resolve by hand. + * + * Create is the only step that observes whether the provider image really exists, + * which is why the repair hangs off it: the registry row and the remote template + * are two systems with no shared transaction, so keeping them in step is always + * best-effort, while checking at the point of use is not. Any other failure is + * rethrown untouched. + */ +async function createSelectedSandbox( + kind: SandboxKind, + options: CreateSandboxOptions, + selected: ResolvedSandbox | null +): Promise { + try { + return await createSandbox(kind, options) + } catch (error) { + if (!selected) throw error + const rebuilding = await repairMissingSandboxImage(selected, error) + if (!rebuilding) throw error + throw new Error(rebuilding) + } +} + /** * Materializes sandbox input files before user code runs. `content` entries are written inline; * `url` entries are fetched from inside the sandbox via `curl` — their bytes never pass through the @@ -267,19 +258,87 @@ async function collectExportedFiles( } } +/** + * Floor on what is left for the user's code after a runtime install. Below this + * the run is not worth attempting — but reporting "your code timed out" would + * still be the wrong story, so the install's own budget is capped to leave it. + */ +const MIN_CODE_BUDGET_MS = 15_000 + +/** + * How long a runtime dependency install may take before it must yield to the + * code it is installing for. Capped by {@link RUNTIME_INSTALL_TIMEOUT_MS} and by + * whatever the caller's budget leaves after reserving {@link MIN_CODE_BUDGET_MS}. + */ +function installBudgetMs(timeoutMs: number): number { + return Math.max(0, Math.min(RUNTIME_INSTALL_TIMEOUT_MS, timeoutMs - MIN_CODE_BUDGET_MS)) +} + +/** + * Installs a runtime sandbox's dependencies out of the caller's budget and + * reports what it spent, so the code that follows can be given the remainder. + * + * Deliberately times ONLY the install. Mount materialization is not deducted — + * it predates this accounting and can legitimately run long for a large + * presigned fetch, so charging it here would shorten the code budget of existing + * prebuilt-strategy workflows that never had an install step at all. + */ +async function provisionWithinBudget( + sandbox: SandboxHandle, + selected: ResolvedSandbox | null, + timeoutMs: number +): Promise { + if (!selected) return 0 + const startedAt = Date.now() + await provisionRuntimeDependencies(sandbox, selected, { timeoutMs: installBudgetMs(timeoutMs) }) + return Date.now() - startedAt +} + +/** What remains of the caller's budget once the install has taken its share. */ +function remainingBudgetMs(timeoutMs: number, installMs: number): number { + return Math.max(MIN_CODE_BUDGET_MS, timeoutMs - installMs) +} + export async function executeInSandbox( req: SandboxExecutionRequest ): Promise { const { code, language, timeoutMs } = req + const kind = req.sandboxKind ?? 'code' + + // Resolved before the sandbox is created so a selection that cannot be honored + // fails without spending a provider create. + const selected = await resolveWorkspaceSandbox({ + kind, + language, + workspaceId: req.workspaceId, + sandboxId: req.sandboxId, + }) - const sandbox = await createSandbox(req.sandboxKind ?? 'code', { language }) + const sandbox = await createSelectedSandbox( + kind, + { language, imageRef: selected?.imageRef }, + selected + ) const sandboxId = sandbox.sandboxId try { - // Inside the try so a failed mount still kills the sandbox via the finally below. + // Inside the try so a failed install or mount still kills the sandbox via the + // finally below. Dependencies land before the inputs so user code and its + // mounts always see a complete environment. + // + // The install is spent OUT OF the caller's budget, not on top of it. Our + // caller aborts the whole request at `timeoutMs` (see `tools/index.ts`), so + // an install that ran to its own separate 240s ceiling would blow past that + // and surface a bare "Request timed out" instead of the classified install + // error. Under the prebuilt strategy provisioning returns immediately, so + // this arithmetic is a no-op there. + const installMs = await provisionWithinBudget(sandbox, selected, timeoutMs) await writeSandboxInputs(sandbox, req.sandboxFiles, {}) - const execution = await sandbox.runCode(code, { timeoutMs }) + const execution = await sandbox.runCode(code, { + timeoutMs: remainingBudgetMs(timeoutMs, installMs), + ...(selected?.envs ? { envs: selected.envs } : {}), + }) if (execution.error) { const errorMessage = `${execution.error.name}: ${execution.error.value}` @@ -338,20 +397,33 @@ export async function executeShellInSandbox( req: SandboxShellExecutionRequest ): Promise { const { code, envs, timeoutMs } = req + const kind = req.sandboxKind ?? 'shell' + + // No language is passed: a shell execution runs commands rather than a language + // runtime, so whichever language the sandbox carries is the one it installs. + const selected = await resolveWorkspaceSandbox({ + kind, + workspaceId: req.workspaceId, + sandboxId: req.sandboxId, + }) - const sandbox = await createSandbox(req.sandboxKind ?? 'shell') + const sandbox = await createSelectedSandbox(kind, { imageRef: selected?.imageRef }, selected) const sandboxId = sandbox.sandboxId try { - // Inside the try so a failed mount still kills the sandbox via the finally below. + // Inside the try so a failed install or mount still kills the sandbox via the + // finally below. The install shares the caller's budget rather than adding to + // it — see the note in `executeInSandbox`. + const installMs = await provisionWithinBudget(sandbox, selected, timeoutMs) await writeSandboxInputs(sandbox, req.sandboxFiles, { rootUser: true }) const result = await sandbox.runCommand(code, { envs: { + ...selected?.envs, ...envs, PATH: '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/bin', }, - timeoutMs, + timeoutMs: remainingBudgetMs(timeoutMs, installMs), rootUser: true, }) diff --git a/apps/sim/lib/execution/remote-sandbox/provider.ts b/apps/sim/lib/execution/remote-sandbox/provider.ts new file mode 100644 index 00000000000..1d662194dfd --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/provider.ts @@ -0,0 +1,41 @@ +import { env } from '@/lib/core/config/env' +import { daytonaProvider } from '@/lib/execution/remote-sandbox/daytona' +import { e2bProvider } from '@/lib/execution/remote-sandbox/e2b' +import type { SandboxProvider, SandboxProviderId } from '@/lib/execution/remote-sandbox/types' + +/** + * The known sandbox providers. Keyed by {@link SandboxProviderId}, so adding an + * adapter is one entry here plus one member on the id union — the type makes an + * unhandled provider a compile error, not a runtime surprise. + */ +const PROVIDERS: Record = { + e2b: e2bProvider, + daytona: daytonaProvider, +} + +const DEFAULT_PROVIDER: SandboxProviderId = 'e2b' + +/** + * Resolves which provider serves this execution from the `SANDBOX_PROVIDER` env + * var (defaulting to {@link DEFAULT_PROVIDER}). + * + * Selection is deliberately resolved ONCE, before the sandbox is created, and is + * never revisited mid-execution: user code has side effects (HTTP calls, S3 + * writes, DB mutations), so retrying a partially-executed run on another provider + * could duplicate them. Changing providers is a config change — set + * `SANDBOX_PROVIDER` and redeploy; in-flight executions are unaffected. + */ +export function resolveProvider(): SandboxProvider { + // Normalize casing identically to env-flags' availability gate — otherwise a + // value like `Daytona` would pass the gate (which lowercases) but miss this + // lowercase-keyed map and throw at create time. + const configured = env.SANDBOX_PROVIDER?.toLowerCase() + if (!configured) return PROVIDERS[DEFAULT_PROVIDER] + const provider = PROVIDERS[configured as SandboxProviderId] + if (!provider) { + throw new Error( + `Unknown SANDBOX_PROVIDER "${env.SANDBOX_PROVIDER}" (expected one of: ${Object.keys(PROVIDERS).join(', ')})` + ) + } + return provider +} diff --git a/apps/sim/lib/execution/remote-sandbox/resolve.test.ts b/apps/sim/lib/execution/remote-sandbox/resolve.test.ts new file mode 100644 index 00000000000..928374e65fe --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/resolve.test.ts @@ -0,0 +1,433 @@ +/** + * @vitest-environment node + * + * Resolution is the fail-closed boundary: a selection that cannot be honored has + * to surface as an explicit error, never as a baffling ModuleNotFoundError + * inside the user's code. These cases pin that contract down. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { CodeLanguage } from '@/lib/execution/languages' + +const { mockSelect, mockUpdate, mockProviderStrategy, mockEnsureSandboxImage, mockIsMissingImage } = + vi.hoisted(() => ({ + mockSelect: vi.fn(), + mockUpdate: vi.fn(), + mockProviderStrategy: { current: 'prebuilt' as 'prebuilt' | 'runtime' }, + mockEnsureSandboxImage: vi.fn(), + mockIsMissingImage: vi.fn(), + })) + +vi.mock('@/lib/execution/remote-sandbox/image-registry', () => ({ + ensureSandboxImage: mockEnsureSandboxImage, + FAILED_BUILD_RETRY_COOLDOWN_MS: 600_000, +})) + +vi.mock('@sim/db', () => ({ + db: { + select: mockSelect, + update: mockUpdate, + }, +})) + +vi.mock('@sim/db/schema', () => ({ + workspaceSandbox: { + id: 'id', + workspaceId: 'workspace_id', + name: 'name', + language: 'language', + dependencies: 'dependencies', + specHash: 'spec_hash', + }, + sandboxImage: { + provider: 'provider', + specHash: 'spec_hash', + status: 'status', + imageRef: 'image_ref', + errorMessage: 'error_message', + lastUsedAt: 'last_used_at', + }, +})) + +vi.mock('drizzle-orm', () => ({ + and: (...args: unknown[]) => args, + eq: (...args: unknown[]) => args, +})) + +vi.mock('@/lib/execution/remote-sandbox/provider', () => ({ + resolveProvider: () => ({ + id: 'e2b', + get dependencyStrategy() { + return mockProviderStrategy.current + }, + get images() { + return mockProviderStrategy.current === 'prebuilt' + ? { isMissingImage: mockIsMissingImage } + : undefined + }, + }), +})) + +import { + invalidateSandboxResolution, + provisionRuntimeDependencies, + repairMissingSandboxImage, + resolveWorkspaceSandbox, +} from '@/lib/execution/remote-sandbox/resolve' + +/** Queues the rows each successive `db.select()` chain resolves to. */ +function queueSelects(...results: unknown[][]) { + mockSelect.mockReset() + for (const rows of results) { + mockSelect.mockReturnValueOnce({ + from: () => ({ where: () => ({ limit: () => Promise.resolve(rows) }) }), + }) + } +} + +const SANDBOX_ROW = { + id: 'sbx-1', + name: 'bigquery-etl', + language: 'python', + dependencies: ['pandas'], + specHash: 'hash-1', +} + +beforeEach(() => { + vi.clearAllMocks() + invalidateSandboxResolution() + mockProviderStrategy.current = 'prebuilt' + mockUpdate.mockReturnValue({ set: () => ({ where: () => Promise.resolve() }) }) + mockEnsureSandboxImage.mockResolvedValue(undefined) +}) + +describe('resolveWorkspaceSandbox', () => { + it('returns null when nothing is selected, leaving current behavior unchanged', async () => { + const resolved = await resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + }) + expect(resolved).toBeNull() + expect(mockSelect).not.toHaveBeenCalled() + }) + + it.each(['doc', 'pi'] as const)('ignores a selection for the %s kind', async (kind) => { + const resolved = await resolveWorkspaceSandbox({ + kind, + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + expect(resolved).toBeNull() + expect(mockSelect).not.toHaveBeenCalled() + }) + + it('passes the ready image ref under the prebuilt strategy', async () => { + queueSelects([SANDBOX_ROW], [{ status: 'ready', imageRef: 'sim-sbx-abc', errorMessage: null }]) + + const resolved = await resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + expect(resolved).toMatchObject({ strategy: 'prebuilt', imageRef: 'sim-sbx-abc' }) + // Python needs no NODE_PATH; only JavaScript does. + expect(resolved?.envs).toBeUndefined() + }) + + it('carries NODE_PATH for javascript so installed packages resolve', async () => { + queueSelects( + [{ ...SANDBOX_ROW, language: 'javascript', dependencies: ['axios'] }], + [{ status: 'ready', imageRef: 'sim-sbx-abc', errorMessage: null }] + ) + + const resolved = await resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.JavaScript, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + expect(resolved?.envs?.NODE_PATH).toContain('node_modules') + }) + + it.each([ + ['building', /still building/], + ['pending', /still building/], + ['failed', /failed to build/], + ])('fails closed when the build is %s', async (status, expected) => { + queueSelects([SANDBOX_ROW], [{ status, imageRef: null, errorMessage: 'pandas not found' }]) + + await expect( + resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + ).rejects.toThrow(expected) + }) + + it('fails closed when the build row does not exist yet', async () => { + queueSelects([SANDBOX_ROW], []) + + await expect( + resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + ).rejects.toThrow(/no completed build/) + }) + + /** + * A sandbox whose definition is fine but whose image is gone must not require + * the user to re-save it in Settings to become runnable again. Resolution + * re-queues the build so the next execution succeeds on its own. + */ + it.each([ + ['a failed build', [{ status: 'failed', imageRef: null, errorMessage: 'pandas not found' }]], + ['a missing build row', []], + ])('re-queues the build for %s', async (_label, imageRows) => { + queueSelects([SANDBOX_ROW], imageRows) + + await expect( + resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + ).rejects.toThrow(/queued/) + + // The cooldown is what stops a scheduled workflow re-enqueueing a build that + // fails in seconds on every single run. + expect(mockEnsureSandboxImage).toHaveBeenCalledWith( + { language: 'python', dependencies: ['pandas'] }, + 'hash-1', + { minFailureAgeMs: 600_000 } + ) + }) + + it('does not re-queue while a healthy build is still in flight', async () => { + queueSelects([SANDBOX_ROW], [{ status: 'building', imageRef: null, errorMessage: null }]) + + await expect( + resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + ).rejects.toThrow(/still building/) + + // The registry's own conflict guard decides whether a stale in-flight row is + // re-claimable, so calling it here is safe — but the message must not + // promise a rebuild that the guard will refuse. + expect(mockEnsureSandboxImage).toHaveBeenCalled() + }) + + it('keeps the build error when the repair itself fails', async () => { + queueSelects([SANDBOX_ROW], [{ status: 'failed', imageRef: null, errorMessage: 'pandas gone' }]) + mockEnsureSandboxImage.mockRejectedValue(new Error('trigger.dev unreachable')) + + await expect( + resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + ).rejects.toThrow(/pandas gone/) + }) + + it('never re-queues when the image is usable', async () => { + queueSelects([SANDBOX_ROW], [{ status: 'ready', imageRef: 'sim-sbx-abc', errorMessage: null }]) + + await resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + expect(mockEnsureSandboxImage).not.toHaveBeenCalled() + }) + + it('rejects a deleted or cross-workspace sandbox', async () => { + queueSelects([]) + + await expect( + resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-other', + sandboxId: 'sbx-1', + }) + ).rejects.toThrow(/no longer exists in this workspace/) + }) + + it('rejects a language mismatch instead of installing the wrong dependency set', async () => { + queueSelects([SANDBOX_ROW], [{ status: 'ready', imageRef: 'sim-sbx-abc', errorMessage: null }]) + + await expect( + resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.JavaScript, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + ).rejects.toThrow(/installs python dependencies, but this block runs javascript/) + }) + + it('never touches the build registry under the runtime strategy', async () => { + mockProviderStrategy.current = 'runtime' + queueSelects([SANDBOX_ROW]) + + const resolved = await resolveWorkspaceSandbox({ + kind: 'code', + language: CodeLanguage.Python, + workspaceId: 'ws-1', + sandboxId: 'sbx-1', + }) + + expect(resolved).toMatchObject({ strategy: 'runtime' }) + expect(resolved?.imageRef).toBeUndefined() + expect(mockSelect).toHaveBeenCalledTimes(1) + expect(mockEnsureSandboxImage).not.toHaveBeenCalled() + }) +}) + +describe('provisionRuntimeDependencies', () => { + function fakeSandbox(commandResult: { stdout: string; stderr: string; exitCode: number }) { + return { + sandboxId: 'sb_1', + writeFile: vi.fn().mockResolvedValue(undefined), + runCommand: vi.fn().mockResolvedValue(commandResult), + runCode: vi.fn(), + readFile: vi.fn(), + kill: vi.fn(), + } + } + + const RUNTIME_PY = { + id: 'sbx-1', + name: 'etl', + language: CodeLanguage.Python, + dependencies: ['pandas'], + strategy: 'runtime' as const, + } + + it('writes the manifest through the filesystem API, never a shell argument', async () => { + const sandbox = fakeSandbox({ stdout: 'ok', stderr: '', exitCode: 0 }) + + await provisionRuntimeDependencies(sandbox, RUNTIME_PY) + + expect(sandbox.writeFile).toHaveBeenCalledWith('/tmp/sim-requirements.txt', 'pandas\n') + const [command] = sandbox.runCommand.mock.calls.at(-1) as [string] + expect(command).toContain('-r /tmp/sim-requirements.txt') + expect(command).not.toContain('pandas') + }) + + it('installs javascript packages from a package.json into the shared prefix', async () => { + const sandbox = fakeSandbox({ stdout: 'ok', stderr: '', exitCode: 0 }) + + await provisionRuntimeDependencies(sandbox, { + ...RUNTIME_PY, + language: CodeLanguage.JavaScript, + dependencies: ['axios@^1.7.0'], + }) + + const manifestCall = sandbox.writeFile.mock.calls[0] as [string, string] + expect(manifestCall[0]).toMatch(/package\.json$/) + expect(JSON.parse(manifestCall[1]).dependencies).toEqual({ axios: '^1.7.0' }) + }) + + it('aborts with the classified error so user code never runs half-installed', async () => { + const sandbox = fakeSandbox({ + stdout: '', + stderr: 'ERROR: No matching distribution found for pandsa', + exitCode: 1, + }) + + await expect(provisionRuntimeDependencies(sandbox, RUNTIME_PY)).rejects.toThrow( + /Package "pandsa" was not found on PyPI/ + ) + }) + + it('does nothing under the prebuilt strategy', async () => { + const sandbox = fakeSandbox({ stdout: '', stderr: '', exitCode: 0 }) + + await provisionRuntimeDependencies(sandbox, { ...RUNTIME_PY, strategy: 'prebuilt' }) + + expect(sandbox.writeFile).not.toHaveBeenCalled() + expect(sandbox.runCommand).not.toHaveBeenCalled() + }) + + it('uses a timeout of its own, so a slow install cannot eat the code budget', async () => { + const sandbox = fakeSandbox({ stdout: 'ok', stderr: '', exitCode: 0 }) + + await provisionRuntimeDependencies(sandbox, RUNTIME_PY) + + const [, options] = sandbox.runCommand.mock.calls.at(-1) as [string, { timeoutMs: number }] + expect(options.timeoutMs).toBeGreaterThan(0) + }) +}) + +/** + * The registry and the provider template are two systems with no shared + * transaction, so every attempt to keep them in step leaves some window. Create is + * the one step that observes the truth, which is why the repair hangs off it. + */ +describe('repairMissingSandboxImage', () => { + const SELECTED = { + id: 'sbx-1', + name: 'bigquery-etl', + language: CodeLanguage.Python, + dependencies: ['pandas'], + specHash: 'hash-1', + strategy: 'prebuilt' as const, + imageRef: 'sim-sbx-abc', + } + + it('rebuilds without a cooldown, because create observed the image is gone', async () => { + mockIsMissingImage.mockResolvedValue(true) + + const message = await repairMissingSandboxImage(SELECTED, new Error('404')) + + expect(message).toMatch(/being rebuilt/) + expect(mockEnsureSandboxImage).toHaveBeenCalledWith( + { language: 'python', dependencies: ['pandas'] }, + 'hash-1', + { imageKnownGone: true } + ) + }) + + /** + * The classifier has to stay narrow: treating an auth or rate-limit failure as a + * missing image would rebuild every sandbox on a provider outage. + */ + it('leaves any other provider failure alone', async () => { + mockIsMissingImage.mockResolvedValue(false) + + const message = await repairMissingSandboxImage(SELECTED, new Error('rate limited')) + + expect(message).toBeNull() + expect(mockEnsureSandboxImage).not.toHaveBeenCalled() + }) + + it('does nothing for a runtime-strategy sandbox, which has no image to miss', async () => { + mockIsMissingImage.mockResolvedValue(true) + + const message = await repairMissingSandboxImage( + { ...SELECTED, strategy: 'runtime', imageRef: undefined }, + new Error('404') + ) + + expect(message).toBeNull() + expect(mockEnsureSandboxImage).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/resolve.ts b/apps/sim/lib/execution/remote-sandbox/resolve.ts new file mode 100644 index 00000000000..87cd62486dc --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/resolve.ts @@ -0,0 +1,464 @@ +import { createLogger } from '@sim/logger' +import { CodeLanguage } from '@/lib/execution/languages' +import { classifyInstallOutput, tailBuildLog } from '@/lib/execution/remote-sandbox/build-errors' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' +import { + isSandboxLanguage, + renderDependencyManifest, + type SandboxLanguage, + SIM_DEPS_DIR, + SIM_NODE_MODULES_DIR, + SIM_PACKAGE_JSON_PATH, + SIM_REQUIREMENTS_PATH, +} from '@/lib/execution/remote-sandbox/sandbox-spec' +import type { + SandboxDependencyStrategy, + SandboxHandle, + SandboxKind, +} from '@/lib/execution/remote-sandbox/types' + +const logger = createLogger('SandboxResolve') + +/** + * The DB is reached lazily so the sandbox barrel stays importable without one. + * `withPiSandbox`, the copilot doc compilers, and `verify-sandbox-parity.ts` all + * pull in this module through `remote-sandbox/index.ts` but never select a + * workspace sandbox — a static `@sim/db` import would make every one of them + * throw at module load when `DATABASE_URL` is unset. + */ +async function sandboxDb() { + const [{ db }, schema, orm] = await Promise.all([ + import('@sim/db'), + import('@sim/db/schema'), + import('drizzle-orm'), + ]) + return { + db, + sandboxImage: schema.sandboxImage, + workspaceSandbox: schema.workspaceSandbox, + and: orm.and, + eq: orm.eq, + } +} + +/** + * Ceiling on a dependency install. + * + * This is an upper bound, not a separate allowance: the caller carves the actual + * budget out of the execution timeout it is itself willing to wait for (see + * `installBudgetMs` in the sandbox barrel). Letting the install run past that + * would only produce a bare client-side "Request timed out" in place of the + * classified installer error. + */ +export const RUNTIME_INSTALL_TIMEOUT_MS = 240_000 + +/** + * How long a resolved BUILD stays cached in-process. + * + * Only the content-addressed half is cached: `sandbox_image` is keyed by + * `(provider, specHash)`, and a spec hash names one immutable dependency set, so + * a hit can never describe the wrong packages. The `workspace_sandbox` row is + * re-read every time (one indexed lookup) because it is the mutable half — that + * is what makes a delete or an edit take effect immediately on EVERY replica + * rather than only the one that served the mutation. + */ +const IMAGE_TTL_MS = 30_000 + +/** `lastUsedAt` is a retention signal, not an audit trail — hourly is precise enough. */ +const LAST_USED_DEBOUNCE_MS = 60 * 60 * 1000 + +/** Only these kinds honor a workspace sandbox; see {@link resolveWorkspaceSandbox}. */ +const SANDBOX_AWARE_KINDS: ReadonlySet = new Set(['code', 'shell']) + +export interface ResolvedSandbox { + id: string + name: string + language: SandboxLanguage + dependencies: string[] + /** Content address of the package set, so a failed create can rebuild it. */ + specHash: string + strategy: SandboxDependencyStrategy + /** Provider image to create from. Present under the `prebuilt` strategy. */ + imageRef?: string + /** Environment the execution must carry for the dependencies to be importable. */ + envs?: Record +} + +/** A `sandbox_image` row, cached by its content address. */ +interface CachedImage { + status: string + imageRef: string | null + errorMessage: string | null +} + +interface CacheEntry { + expiresAt: number + value: CachedImage +} + +/** + * Both maps are process-lifetime and keyed by an unbounded space (every spec + * hash ever executed), so each drops its oldest entry rather than growing for + * the life of the worker. + */ +const IMAGE_CACHE_LIMIT = 1000 +const LAST_USED_CACHE_LIMIT = 1000 + +const imageCache = new Map() +const lastUsedWrites = new Map() + +/** + * JavaScript packages live outside the default resolution roots, so Node needs + * `NODE_PATH` to find them. Both strategies install into the same directory, so + * both hand back the same environment. + */ +function envsFor(language: SandboxLanguage): Record | undefined { + return language === CodeLanguage.JavaScript ? { NODE_PATH: SIM_NODE_MODULES_DIR } : undefined +} + +/** + * Records that a build was used, so the retention sweep can tell a live image + * from an abandoned one. Debounced and fire-and-forget: this is bookkeeping, and + * a failed write must never fail an execution. + */ +function touchImage(specHash: string, provider: string): void { + const key = `${provider}:${specHash}` + const now = Date.now() + const written = lastUsedWrites.get(key) + if (written && now - written < LAST_USED_DEBOUNCE_MS) return + if (lastUsedWrites.size >= LAST_USED_CACHE_LIMIT) lastUsedWrites.clear() + lastUsedWrites.set(key, now) + void sandboxDb() + .then(({ db, sandboxImage, and, eq }) => + db + .update(sandboxImage) + .set({ lastUsedAt: new Date() }) + .where(and(eq(sandboxImage.provider, provider), eq(sandboxImage.specHash, specHash))) + ) + .catch((error) => logger.warn('Failed to record sandbox image use', { specHash, error })) +} + +/** + * Resolves the sandbox an execution should run against, or `null` when none is + * selected (today's behavior: the env-configured template, no install step). + * + * Fails closed rather than degrading. A selection that cannot be honored — + * deleted, cross-workspace, wrong language, or a build that is not `ready` — + * throws with the reason, because the alternative is a baffling + * `ModuleNotFoundError` inside the user's code. + * + * Deliberately not plan-gated: the gate covers creating and editing sandboxes, + * so a workspace that downgrades keeps executing the ones it already has. + */ +export async function resolveWorkspaceSandbox(args: { + kind: SandboxKind + /** + * The language the caller will execute. Omitted by the shell path, which runs + * commands rather than a language runtime and so has nothing to mismatch. + */ + language?: CodeLanguage + workspaceId?: string + sandboxId?: string +}): Promise { + const { kind, language, workspaceId, sandboxId } = args + if (!sandboxId) return null + // `doc` and `pi` keep their vetted images unconditionally. + if (!SANDBOX_AWARE_KINDS.has(kind)) return null + if (!workspaceId) { + throw new Error('A sandbox was selected but this execution has no workspace to resolve it in') + } + + const provider = resolveProvider() + const { db, sandboxImage, workspaceSandbox, and, eq } = await sandboxDb() + const [row] = await db + .select({ + id: workspaceSandbox.id, + name: workspaceSandbox.name, + language: workspaceSandbox.language, + dependencies: workspaceSandbox.dependencies, + specHash: workspaceSandbox.specHash, + }) + .from(workspaceSandbox) + .where(and(eq(workspaceSandbox.id, sandboxId), eq(workspaceSandbox.workspaceId, workspaceId))) + .limit(1) + + if (!row) { + throw new Error( + `The selected sandbox no longer exists in this workspace. Pick another one, or clear the selection to run on the default image.` + ) + } + if (!isSandboxLanguage(row.language)) { + throw new Error(`Sandbox "${row.name}" has an unsupported language (${row.language})`) + } + + const base = { + id: row.id, + name: row.name, + language: row.language, + dependencies: row.dependencies ?? [], + specHash: row.specHash, + envs: envsFor(row.language), + } + + let resolved: ResolvedSandbox + if (base.dependencies.length === 0) { + // A sandbox with no packages declares nothing to build and nothing to + // install, so it resolves to the base image under either strategy. Looking + // for a build row here would fail closed on an image that never existed. + resolved = { ...base, strategy: provider.dependencyStrategy } + } else if (provider.dependencyStrategy === 'runtime') { + resolved = { ...base, strategy: 'runtime' } + } else { + const image = await readImage(provider.id, row.specHash) + + if (!image || image.status !== 'ready' || !image.imageRef) { + await scheduleImageRepair(base, row.specHash) + throw new Error(describeUnusableImage(row.name, image?.status, image?.errorMessage)) + } + touchImage(row.specHash, provider.id) + resolved = { ...base, strategy: 'prebuilt', imageRef: image.imageRef } + } + + assertLanguageMatches(resolved, language) + return resolved +} + +/** + * Reads a build row, memoized on its content address. + * + * Editing a sandbox produces a different hash and deleting one is caught by the + * `workspace_sandbox` read that always runs, so those cannot serve a stale hit. A + * non-ready row is NOT cached either: it is precisely the value that flips + * underneath us while a build completes, and caching it would keep a just-finished + * build unusable. + * + * A `ready` row is no longer strictly terminal, though, and this cache is + * per-process. `releaseSandboxImage` clears only the replica that ran it, so + * another replica can serve a cached `ready` image for up to {@link IMAGE_TTL_MS} + * after its template was deleted — and because the hit looks healthy, resolution + * hands back a dead `imageRef` instead of reaching the repair path. Sandbox + * creation then fails on that replica until the entry expires and the row read + * finds nothing. Bounded and self-healing, but real; closing it needs either + * cross-replica invalidation or a provider-error path that invalidates on + * "template not found". + */ +async function readImage(providerId: string, specHash: string): Promise { + const cacheKey = `${providerId}:${specHash}` + const cached = imageCache.get(cacheKey) + if (cached) { + if (cached.expiresAt > Date.now()) return cached.value + imageCache.delete(cacheKey) + } + + const { db, sandboxImage, and, eq } = await sandboxDb() + const [image] = await db + .select({ + status: sandboxImage.status, + imageRef: sandboxImage.imageRef, + errorMessage: sandboxImage.errorMessage, + }) + .from(sandboxImage) + .where(and(eq(sandboxImage.provider, providerId), eq(sandboxImage.specHash, specHash))) + .limit(1) + + if (image?.status === 'ready' && image.imageRef) { + if (imageCache.size >= IMAGE_CACHE_LIMIT) { + const oldest = imageCache.keys().next() + if (!oldest.done) imageCache.delete(oldest.value) + } + imageCache.set(cacheKey, { expiresAt: Date.now() + IMAGE_TTL_MS, value: image }) + } + return image +} + +/** + * Re-enqueues a build for a sandbox whose image is unusable. + * + * `ensureSandboxImage` otherwise runs only when a sandbox is saved, which left + * three states permanently stuck until someone re-saved it in Settings: a build + * that failed, a build whose worker died mid-flight, and — after switching a + * deployment from a `runtime` provider to a `prebuilt` one — every sandbox + * created while the old provider was active, since `runtime` writes no image + * rows at all. Repairing here costs an execution that was going to fail either + * way and lets the next one succeed, instead of making the user reconfigure a + * sandbox whose definition was never wrong. + * + * Rate-limited, unlike the save path. This fires once per execution, and a bad + * package name fails in seconds, so re-claiming a failed row on sight would let a + * per-minute schedule enqueue a per-minute build of something that will never + * succeed. The cooldown caps that at one attempt per window while a save — an + * explicit request from a person — still retries immediately. Executions arriving + * during a healthy build enqueue nothing either way. + * + * Imported dynamically for the same reason as {@link sandboxDb} — the registry + * pulls `@sim/db` into the static import graph, which this module keeps out of + * the executor bundle. A repair that fails must never replace the caller's + * message, which is the one naming the sandbox and its build error. + */ +async function scheduleImageRepair( + spec: { language: SandboxLanguage; dependencies: string[] }, + specHash: string, + options?: { imageKnownGone?: boolean } +): Promise { + try { + const { ensureSandboxImage, FAILED_BUILD_RETRY_COOLDOWN_MS } = await import( + '@/lib/execution/remote-sandbox/image-registry' + ) + await ensureSandboxImage( + { language: spec.language, dependencies: spec.dependencies }, + specHash, + // A create that just failed on a missing image has observed the truth, so it + // reclaims whatever the row says and skips the cooldown. Resolution reading a + // row it cannot verify only gets the rate-limited retry. + options?.imageKnownGone + ? { imageKnownGone: true } + : { minFailureAgeMs: FAILED_BUILD_RETRY_COOLDOWN_MS } + ) + } catch (error) { + logger.warn('Failed to schedule sandbox image repair', { specHash, error }) + } +} + +/** + * Repairs a sandbox whose image turned out to be gone when the provider was asked + * to create from it. + * + * This is the backstop the registry cannot be: the row and the provider template + * are two systems with no shared transaction, so every attempt to keep them in step + * leaves some window — a released image adopted mid-delete, a stale cache on another + * replica, a rebuild that did not take. Create is the one place that observes ground + * truth, so a `ready` row pointing at nothing repairs itself here on first use + * instead of needing someone to re-save the sandbox. + * + * Returns the message to fail this execution with, or `null` when the failure was + * anything else and must surface unchanged. + */ +export async function repairMissingSandboxImage( + selected: ResolvedSandbox, + error: unknown +): Promise { + if (selected.strategy !== 'prebuilt' || !selected.imageRef) return null + + const provider = resolveProvider() + if (!provider.images) return null + if (!(await provider.images.isMissingImage(error))) return null + + invalidateSandboxResolution() + await scheduleImageRepair(selected, selected.specHash, { imageKnownGone: true }) + logger.warn('Sandbox image was missing at create; rebuilding it', { + sandbox: selected.name, + specHash: selected.specHash, + }) + + return `Sandbox "${selected.name}" is being rebuilt because its image is no longer available. Run again in a moment.` +} + +function assertLanguageMatches(sandbox: ResolvedSandbox, language?: CodeLanguage): void { + if (!language || sandbox.language === language) return + throw new Error( + `Sandbox "${sandbox.name}" installs ${sandbox.language} dependencies, but this block runs ${language}. Select a ${language} sandbox or clear the selection.` + ) +} + +function describeUnusableImage( + name: string, + status: string | undefined, + errorMessage: string | null | undefined +): string { + if (status === 'failed') { + return `Sandbox "${name}" failed to build: ${errorMessage ?? 'installation failed'}. A rebuild has been queued — run again in a moment. If it keeps failing, fix its dependencies in Settings → Sandboxes.` + } + if (status === 'pending' || status === 'building') { + return `Sandbox "${name}" is still building. Wait for it to finish, then run again.` + } + return `Sandbox "${name}" has no completed build yet. A build has been queued — run again in a moment.` +} + +/** + * Clears the in-process build cache. + * + * Best-effort only, and no longer load-bearing: it clears one process, so on a + * multi-replica deployment the others keep their entries. Correctness comes from + * what is NOT cached — the `workspace_sandbox` row is re-read on every resolve, + * and the cache is keyed by content address, so a stale entry can only ever + * describe a build that is still exactly what its hash says it is. This just + * lets the replica that served a mutation pick up a rebuild a little sooner. + */ +export function invalidateSandboxResolution(): void { + imageCache.clear() +} + +function installCommandFor(language: SandboxLanguage): string { + if (language === CodeLanguage.Python) { + return `pip install --no-input --disable-pip-version-check -r ${SIM_REQUIREMENTS_PATH}` + } + // `--prefix` is the install target; the manifest is copied in as root first, + // because the filesystem API cannot write into a root-owned directory. + return `cp ${SIM_PACKAGE_JSON_PATH} ${SIM_DEPS_DIR}/package.json && npm install --prefix ${SIM_DEPS_DIR} --no-audit --no-fund --omit=dev` +} + +/** + * Installs a runtime-strategy sandbox's dependencies before user code runs. + * + * The dependency list reaches the sandbox as a file written through the + * filesystem API, never interpolated into a shell command, so a package name is + * never parsed as shell syntax. The installer's own output is returned to the + * caller rather than merged into the execution's stdout, so a package whose name + * contains the `__SIM_RESULT__` marker cannot corrupt the parsed result. + * + * A non-zero exit throws: user code must never run against a half-installed + * environment and report a confusing `ModuleNotFoundError` instead of the real + * installation failure. + */ +export async function provisionRuntimeDependencies( + sandbox: SandboxHandle, + resolved: ResolvedSandbox, + options?: { timeoutMs?: number } +): Promise { + if (resolved.strategy !== 'runtime' || resolved.dependencies.length === 0) return + + const installTimeoutMs = options?.timeoutMs ?? RUNTIME_INSTALL_TIMEOUT_MS + if (installTimeoutMs <= 0) { + throw new Error( + `Sandbox "${resolved.name}" installs its packages at run time, which needs more time than this block's timeout allows. Raise the block's timeout and try again.` + ) + } + + const manifest = renderDependencyManifest({ + language: resolved.language, + dependencies: resolved.dependencies, + }) + const manifestPath = + resolved.language === CodeLanguage.Python ? SIM_REQUIREMENTS_PATH : SIM_PACKAGE_JSON_PATH + + await sandbox.writeFile(manifestPath, manifest) + if (resolved.language === CodeLanguage.JavaScript) { + await sandbox.runCommand(`mkdir -p ${SIM_DEPS_DIR}`, { timeoutMs: 30_000, rootUser: true }) + } + + const started = Date.now() + const result = await sandbox.runCommand(installCommandFor(resolved.language), { + timeoutMs: installTimeoutMs, + rootUser: true, + }) + + if (result.exitCode !== 0) { + // Daytona merges both streams into stdout, so fall back to it for the real output. + const output = result.stderr || result.stdout || `installer exited ${result.exitCode}` + const classified = classifyInstallOutput(resolved.language, output) + logger.error('Runtime dependency install failed', { + sandboxId: sandbox.sandboxId, + sandbox: resolved.name, + code: classified.code, + exitCode: result.exitCode, + }) + throw new Error(`${classified.message}\n\n${tailBuildLog(output)}`) + } + + logger.info('Installed sandbox dependencies at run time', { + sandboxId: sandbox.sandboxId, + sandbox: resolved.name, + dependencyCount: resolved.dependencies.length, + durationMs: Date.now() - started, + }) +} diff --git a/apps/sim/lib/execution/remote-sandbox/sandbox-spec.test.ts b/apps/sim/lib/execution/remote-sandbox/sandbox-spec.test.ts new file mode 100644 index 00000000000..ba8589a93af --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/sandbox-spec.test.ts @@ -0,0 +1,190 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { CodeLanguage } from '@/lib/execution/languages' +import { + canonicalizeDependencies, + hashSandboxSpec, + MAX_SANDBOX_DEPENDENCIES, + parseJsDependency, + quoteDependency, + renderDependencyManifest, + validateDependencies, +} from '@/lib/execution/remote-sandbox/sandbox-spec' + +const PY = CodeLanguage.Python +const JS = CodeLanguage.JavaScript + +function accepted(language: typeof PY | typeof JS, input: string): string[] { + const result = validateDependencies(language, input) + if (!result.ok) { + throw new Error(`expected acceptance, got: ${JSON.stringify(result.issues)}`) + } + return result.dependencies +} + +function rejection(language: typeof PY | typeof JS, input: string) { + const result = validateDependencies(language, input) + if (result.ok) throw new Error(`expected rejection of ${input}`) + return result.issues +} + +describe('validateDependencies (python)', () => { + it.each([ + 'Django==5.0', + 'google-cloud-bigquery[pandas]>=3', + 'pandas', + 'pyairtable>=3.0', + 'requests>=2.0,<3.0', + 'numpy~=1.26', + 'urllib3!=2.0.0', + ])('accepts %s', (value) => { + expect(accepted(PY, value)).toHaveLength(1) + }) + + it.each([ + ['git+https://github.com/psf/requests', 'URLs and VCS references are not allowed'], + ['-e .', 'installer flags are not allowed (remove the leading dash)'], + ['--index-url http://evil', 'installer flags are not allowed (remove the leading dash)'], + ['foo; rm -rf /', 'a dependency cannot contain spaces'], + ['../local-package', 'local paths are not allowed'], + ['https://example.com/pkg.whl', 'URLs and VCS references are not allowed'], + ])('rejects %s', (value, reason) => { + const issues = rejection(PY, value) + expect(issues).toHaveLength(1) + expect(issues[0].reason).toBe(reason) + }) + + it('reports the offending line number against the submitted row', () => { + const issues = rejection(PY, ['pandas', '', '# a comment', 'git+https://evil', 'requests']) + expect(issues).toHaveLength(1) + expect(issues[0].line).toBe(4) + expect(issues[0].value).toBe('git+https://evil') + }) + + it('strips comments and blank lines', () => { + expect(accepted(PY, '# deps\n\npandas\n\n # trailing\nrequests\n')).toEqual([ + 'pandas', + 'requests', + ]) + }) + + it('rejects more than the dependency cap and marks every entry past it', () => { + const list = Array.from({ length: MAX_SANDBOX_DEPENDENCIES + 1 }, (_, i) => `pkg-${i}`) + const issues = rejection(PY, list) + expect(issues).toHaveLength(1) + expect(issues[0].line).toBe(MAX_SANDBOX_DEPENDENCIES + 1) + }) + + it('rejects an over-long entry', () => { + const issues = rejection(PY, `pkg${'a'.repeat(300)}`) + expect(issues[0].reason).toContain('longer than') + }) +}) + +describe('validateDependencies (javascript)', () => { + it.each([ + 'axios@^1.7.0', + '@aws-sdk/client-s3', + '@aws-sdk/client-s3@^3.600.0', + 'zod', + 'lodash@4.17.21', + 'left-pad@latest', + ])('accepts %s', (value) => { + expect(accepted(JS, value)).toHaveLength(1) + }) + + it.each([ + ['git+https://github.com/axios/axios', 'URLs and VCS references are not allowed'], + ['file:../local', 'local and alias specifiers are not allowed'], + ['link:../sibling', 'local and alias specifiers are not allowed'], + ['workspace:*', 'local and alias specifiers are not allowed'], + ['npm:alias@1.0.0', 'local and alias specifiers are not allowed'], + ['axios@>=1.2 <2', 'a dependency cannot contain spaces'], + ])('rejects %s', (value, reason) => { + const issues = rejection(JS, value) + expect(issues[0].reason).toBe(reason) + }) +}) + +describe('canonicalization and hashing', () => { + it('is stable under reordering', () => { + const a = hashSandboxSpec({ language: PY, dependencies: ['requests', 'pandas'] }) + const b = hashSandboxSpec({ language: PY, dependencies: ['pandas', 'requests'] }) + expect(a).toBe(b) + }) + + it('normalizes python names per PEP 503, so casing and separators do not fork a build', () => { + const a = hashSandboxSpec({ language: PY, dependencies: ['Google_Cloud.BigQuery'] }) + const b = hashSandboxSpec({ language: PY, dependencies: ['google-cloud-bigquery'] }) + expect(a).toBe(b) + expect(canonicalizeDependencies(PY, ['Google_Cloud.BigQuery'])).toEqual([ + 'google-cloud-bigquery', + ]) + }) + + it('leaves npm names verbatim, because the registry serves React and react separately', () => { + expect(canonicalizeDependencies(JS, ['React'])).toEqual(['React']) + expect(hashSandboxSpec({ language: JS, dependencies: ['React'] })).not.toBe( + hashSandboxSpec({ language: JS, dependencies: ['react'] }) + ) + }) + + it('de-duplicates', () => { + expect(canonicalizeDependencies(PY, ['pandas', 'pandas', 'PANDAS'])).toEqual(['pandas']) + }) + + it('hashes the same list differently under different languages', () => { + expect(hashSandboxSpec({ language: PY, dependencies: ['lodash'] })).not.toBe( + hashSandboxSpec({ language: JS, dependencies: ['lodash'] }) + ) + }) + + it('preserves the version specifier while normalizing only the name', () => { + expect(canonicalizeDependencies(PY, ['Google_Cloud.BigQuery[Pandas]>=3.0'])).toEqual([ + 'google-cloud-bigquery[Pandas]>=3.0', + ]) + }) +}) + +describe('shell quoting', () => { + it('quotes specifier characters that a shell would otherwise interpret', () => { + expect(quoteDependency('django>=5.0')).toBe("'django>=5.0'") + }) + + it('refuses to quote a value that never passed validation', () => { + expect(() => quoteDependency("foo' ; rm -rf /")).toThrow(/unvalidated dependency/) + }) +}) + +describe('manifest rendering', () => { + it('renders a requirements.txt for python', () => { + expect(renderDependencyManifest({ language: PY, dependencies: ['requests', 'pandas'] })).toBe( + 'pandas\nrequests\n' + ) + }) + + it('renders a package.json dependency map for javascript', () => { + const manifest = renderDependencyManifest({ + language: JS, + dependencies: ['axios@^1.7.0', '@aws-sdk/client-s3', 'zod@3.23.8'], + }) + expect(JSON.parse(manifest).dependencies).toEqual({ + '@aws-sdk/client-s3': '*', + axios: '^1.7.0', + zod: '3.23.8', + }) + }) + + it('splits a scoped name from its range without mistaking the scope for one', () => { + expect(parseJsDependency('@aws-sdk/client-s3')).toEqual({ + name: '@aws-sdk/client-s3', + range: '*', + }) + expect(parseJsDependency('@aws-sdk/client-s3@^3.0.0')).toEqual({ + name: '@aws-sdk/client-s3', + range: '^3.0.0', + }) + }) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/sandbox-spec.ts b/apps/sim/lib/execution/remote-sandbox/sandbox-spec.ts new file mode 100644 index 00000000000..ae93ddeb88f --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/sandbox-spec.ts @@ -0,0 +1,294 @@ +import { createHash } from 'node:crypto' +import { CodeLanguage } from '@/lib/execution/languages' + +/** + * The languages a sandbox can carry dependencies for. Shell is excluded: a shell + * execution has no package manager of its own, it borrows whichever sandbox the + * `code` kind resolved. + */ +export type SandboxLanguage = `${CodeLanguage.JavaScript}` | `${CodeLanguage.Python}` + +export const SANDBOX_LANGUAGES = [CodeLanguage.JavaScript, CodeLanguage.Python] as const + +export function isSandboxLanguage(value: string): value is SandboxLanguage { + return value === CodeLanguage.JavaScript || value === CodeLanguage.Python +} + +/** + * The package registry a language installs from, named as a user would recognize + * it. Single source of truth so validation errors, build failures, and the wand + * prompt all call it the same thing. + */ +export function registryFor(language: SandboxLanguage): string { + return language === CodeLanguage.Python ? 'PyPI' : 'npm' +} + +/** + * A canonical, content-addressable dependency set. `dependencies` is always the + * output of {@link canonicalizeDependencies}, so two specs that mean the same + * thing are byte-identical and therefore hash identically. + */ +export interface SandboxSpec { + language: SandboxLanguage + dependencies: string[] +} + +/** Upper bound on how many packages one sandbox may declare. */ +export const MAX_SANDBOX_DEPENDENCIES = 50 + +/** Upper bound on the length of a single dependency entry. */ +export const MAX_DEPENDENCY_LENGTH = 200 + +/** + * Where the installed JavaScript packages live inside a sandbox, for both the + * prebuilt (baked into the E2B template) and runtime (installed per execution) + * strategies. Unlike pip, an npm install is not automatically importable — Node + * resolves from `NODE_PATH` or the install location — so producer and consumer + * must agree on one directory. They read it from here. + */ +export const SIM_DEPS_DIR = '/opt/sim-deps' + +export const SIM_NODE_MODULES_DIR = `${SIM_DEPS_DIR}/node_modules` + +/** + * Where the runtime strategy writes the dependency manifests it installs from. + * + * Both live under `/tmp`, not {@link SIM_DEPS_DIR}: the install target is created + * as root, while `SandboxHandle.writeFile` goes through the provider filesystem + * API as the sandbox's default user — writing the manifest into the root-owned + * directory fails with EACCES. + */ +export const SIM_REQUIREMENTS_PATH = '/tmp/sim-requirements.txt' + +export const SIM_PACKAGE_JSON_PATH = '/tmp/sim-package.json' + +/** A rejected dependency line, addressed back to the row the user typed it on. */ +export interface DependencyIssue { + /** 1-indexed line in the submitted list, so the editor can mark the exact row. */ + line: number + value: string + reason: string +} + +export type DependencyValidation = + | { ok: true; dependencies: string[] } + | { ok: false; issues: DependencyIssue[] } + +/** + * PEP 508 distribution name followed by optional extras and an optional PEP 440 + * version specifier set. Deliberately whitespace-free — see + * {@link quoteDependency} for why every accepted character has to be inert. + */ +const PYTHON_SPECIFIER = '(===|==|!=|<=|>=|~=|<|>)[A-Za-z0-9][A-Za-z0-9.*+!-]*' +const PYTHON_DEPENDENCY_PATTERN = new RegExp( + '^[A-Za-z0-9][A-Za-z0-9._-]*' + + '(\\[[A-Za-z0-9][A-Za-z0-9._-]*(,[A-Za-z0-9][A-Za-z0-9._-]*)*\\])?' + + `(${PYTHON_SPECIFIER}(,${PYTHON_SPECIFIER})*)?$` +) + +/** An npm package name, optionally scoped, with an optional `@range` suffix. */ +const JS_DEPENDENCY_PATTERN = + /^(@[a-z0-9][a-z0-9._-]*\/)?[a-zA-Z0-9][a-zA-Z0-9._-]*(@[\^~<>=]{0,2}[A-Za-z0-9*][A-Za-z0-9.*+-]*)?$/ + +/** + * Prefixes and substrings rejected ahead of the shape check purely so the error + * names the actual problem. The patterns above would reject all of these anyway. + */ +const DEPENDENCY_REJECTIONS: ReadonlyArray<{ match: RegExp; reason: string }> = [ + { match: /^-/, reason: 'installer flags are not allowed (remove the leading dash)' }, + { match: /^git\+|:\/\//, reason: 'URLs and VCS references are not allowed' }, + { + match: /^(file|link|workspace|npm|portal|patch):/i, + reason: 'local and alias specifiers are not allowed', + }, + { match: /[\s]/, reason: 'a dependency cannot contain spaces' }, + { match: /[;&|`$(){}"'\\]/, reason: 'contains characters that are not valid in a package name' }, + { match: /^[./~]/, reason: 'local paths are not allowed' }, +] + +interface SourceLine { + line: number + value: string +} + +/** + * Splits a submitted dependency list into meaningful entries while keeping each + * one addressed to the row it was typed on, so a rejection can be marked inline + * rather than reported against a shifted index. + */ +function readDependencyLines(input: string | readonly string[]): SourceLine[] { + const rawLines = typeof input === 'string' ? input.split('\n') : input + const entries: SourceLine[] = [] + rawLines.forEach((raw, index) => { + const value = (raw ?? '').trim() + if (!value || value.startsWith('#')) return + entries.push({ line: index + 1, value }) + }) + return entries +} + +/** + * Normalizes a Python distribution name per PEP 503, which PyPI treats as + * case- and separator-insensitive. `Google_Cloud.BigQuery` and + * `google-cloud-bigquery` are the same distribution, so they must reach the same + * spec hash and share one build. + * + * npm names get no such treatment: the registry serves `React` and `react` as + * distinct packages, so folding case there could install something the user did + * not ask for. + */ +function normalizePythonName(name: string): string { + return name.toLowerCase().replace(/[-_.]+/g, '-') +} + +function canonicalizeEntry(language: SandboxLanguage, value: string): string { + if (language !== CodeLanguage.Python) return value + const boundary = value.search(/[[<>=!~]/) + if (boundary === -1) return normalizePythonName(value) + return normalizePythonName(value.slice(0, boundary)) + value.slice(boundary) +} + +/** + * Reduces a validated dependency list to its canonical form: normalized, + * de-duplicated, and sorted. Two lists that install the same thing collapse to + * the same array, which is what makes {@link hashSandboxSpec} content-addressed. + */ +export function canonicalizeDependencies( + language: SandboxLanguage, + dependencies: readonly string[] +): string[] { + const seen = new Set() + for (const dependency of dependencies) { + seen.add(canonicalizeEntry(language, dependency.trim())) + } + return [...seen].sort() +} + +/** + * Validates a submitted dependency list and returns its canonical form. + * + * This is the security boundary for the whole feature. Every accepted entry is + * eventually handed to pip or npm, on the E2B build path as a shell argument, so + * the grammar here is a strict allowlist rather than a denylist: no whitespace, + * no quoting, no redirection, no path or URL forms. Everything downstream — + * {@link quoteDependency}, the runtime manifests — assumes that invariant holds. + */ +export function validateDependencies( + language: SandboxLanguage, + input: string | readonly string[] +): DependencyValidation { + const entries = readDependencyLines(input) + const issues: DependencyIssue[] = [] + + // Reported once, against the first line over the cap. Marking every remaining + // row would bury the real message under hundreds of identical ones and bloat + // the response for a paste that is simply too long. + const overflow = entries[MAX_SANDBOX_DEPENDENCIES] + if (overflow) { + issues.push({ + line: overflow.line, + value: overflow.value, + reason: `a sandbox can declare at most ${MAX_SANDBOX_DEPENDENCIES} dependencies (this list has ${entries.length})`, + }) + } + + const pattern = + language === CodeLanguage.Python ? PYTHON_DEPENDENCY_PATTERN : JS_DEPENDENCY_PATTERN + const registryName = registryFor(language) + + for (const entry of entries.slice(0, MAX_SANDBOX_DEPENDENCIES)) { + if (entry.value.length > MAX_DEPENDENCY_LENGTH) { + issues.push({ + line: entry.line, + value: entry.value, + reason: `a dependency cannot be longer than ${MAX_DEPENDENCY_LENGTH} characters`, + }) + continue + } + + const rejection = DEPENDENCY_REJECTIONS.find((candidate) => candidate.match.test(entry.value)) + if (rejection) { + issues.push({ line: entry.line, value: entry.value, reason: rejection.reason }) + continue + } + + if (!pattern.test(entry.value)) { + issues.push({ + line: entry.line, + value: entry.value, + reason: `not a valid ${registryName} package name or version specifier`, + }) + } + } + + if (issues.length > 0) return { ok: false, issues } + return { + ok: true, + dependencies: canonicalizeDependencies( + language, + entries.map((e) => e.value) + ), + } +} + +/** + * Content address for a canonical spec. Keyed on language as well as + * dependencies so the same package list under Python and JavaScript never + * collides onto one build. + */ +export function hashSandboxSpec(spec: SandboxSpec): string { + const canonical = JSON.stringify({ + language: spec.language, + dependencies: canonicalizeDependencies(spec.language, spec.dependencies), + }) + return createHash('sha256').update(canonical, 'utf-8').digest('hex') +} + +/** + * Quotes a validated dependency for an argv position in a shell command. + * + * The E2B template builder composes `pip install`/`npm install` as a shell + * string, so an unquoted `django>=5.0` would redirect stdout to a file named + * `=5.0` and silently install an unpinned Django. Validation already rejects + * quotes and whitespace, so wrapping in single quotes is total — but the guard + * stays, because this function is what makes that assumption load-bearing. + */ +export function quoteDependency(dependency: string): string { + // Quotes and whitespace would break out of the argv slot; a leading dash would + // stay inside it and be read by pip/npm as a FLAG (`--index-url=...`, `-t/`) + // rather than a package. Specs are re-read from `sandbox_image.spec` JSONB long + // after validation ran, so this check cannot lean on the writer having run it. + if (/^-/.test(dependency) || /['"\s]/.test(dependency)) { + throw new Error(`Refusing to shell-quote an unvalidated dependency: ${dependency}`) + } + return `'${dependency}'` +} + +/** + * Splits an npm spec into the name and range halves a `package.json` + * `dependencies` map needs. The scope's leading `@` is skipped when looking for + * the range separator so `@aws-sdk/client-s3` is not read as a range. + */ +export function parseJsDependency(dependency: string): { name: string; range: string } { + const separator = dependency.lastIndexOf('@') + if (separator <= 0) return { name: dependency, range: '*' } + return { name: dependency.slice(0, separator), range: dependency.slice(separator + 1) || '*' } +} + +/** + * Renders the canonical list as the manifest the runtime strategy installs from. + * The bytes are delivered through the sandbox filesystem API, never a shell, so + * this is the second line of defense behind {@link validateDependencies}. + */ +export function renderDependencyManifest(spec: SandboxSpec): string { + const dependencies = canonicalizeDependencies(spec.language, spec.dependencies) + if (spec.language === CodeLanguage.Python) { + return `${dependencies.join('\n')}\n` + } + const map: Record = {} + for (const dependency of dependencies) { + const { name, range } = parseJsDependency(dependency) + map[name] = range + } + return `${JSON.stringify({ name: 'sim-sandbox-deps', private: true, dependencies: map }, null, 2)}\n` +} diff --git a/apps/sim/lib/execution/remote-sandbox/types.ts b/apps/sim/lib/execution/remote-sandbox/types.ts index 69772f49e47..e155b2adb75 100644 --- a/apps/sim/lib/execution/remote-sandbox/types.ts +++ b/apps/sim/lib/execution/remote-sandbox/types.ts @@ -1,4 +1,6 @@ import type { CodeLanguage } from '@/lib/execution/languages' +import type { SandboxBuildError } from '@/lib/execution/remote-sandbox/build-errors' +import type { SandboxSpec } from '@/lib/execution/remote-sandbox/sandbox-spec' /** * Which vetted image a sandbox runs in. Every kind fails closed when its @@ -30,6 +32,10 @@ export interface SandboxExecutionRequest { * (mothership-docs) that has python-pptx/docx/openpyxl/reportlab installed. */ sandboxKind?: 'code' | 'doc' + /** Scope for {@link sandboxId}; a sandbox from another workspace is rejected. */ + workspaceId?: string + /** Workspace sandbox whose dependency set this execution runs against. */ + sandboxId?: string } export interface SandboxShellExecutionRequest { @@ -45,6 +51,10 @@ export interface SandboxShellExecutionRequest { * they run in the doc image (mothership-docs). */ sandboxKind?: 'shell' | 'doc' + /** Scope for {@link sandboxId}; a sandbox from another workspace is rejected. */ + workspaceId?: string + /** Workspace sandbox whose dependency set this execution runs against. */ + sandboxId?: string } export interface SandboxExecutionResult { @@ -106,7 +116,10 @@ export interface SandboxHandle { * override — passing `javascript` to its `codeRun` executes the source through * Python instead. We create one sandbox per execution, so binding costs nothing. */ - runCode(code: string, options: { timeoutMs: number }): Promise + runCode( + code: string, + options: { timeoutMs: number; envs?: Record } + ): Promise runCommand(command: string, options: RunCommandOptions): Promise readFile(path: string): Promise /** @@ -121,6 +134,13 @@ export interface SandboxHandle { export interface CreateSandboxOptions { /** Bound at creation — see {@link SandboxHandle.runCode}. */ language?: CodeLanguage + /** + * Provider image to create from, overriding the env-configured template. + * Honored for `code` and `shell` only: `doc` and `pi` keep their vetted images + * unconditionally, so a user's dependency set can never displace the document + * compiler's or the coding agent's. + */ + imageRef?: string /** * How long the provider may keep the sandbox alive before reaping it. Only * E2B honours this: its default is five minutes, so anything longer-running @@ -131,7 +151,61 @@ export interface CreateSandboxOptions { lifetimeMs?: number } +/** + * How a provider materializes a custom dependency set. + * + * `prebuilt` bakes it into a reusable image ahead of time (E2B, whose templates + * have no count limit and layer cheaply). `runtime` installs it inside the + * sandbox before user code runs (Daytona, whose 30-snapshot organization quota + * does not scale with tier and so cannot hold per-workspace images). + */ +export type SandboxDependencyStrategy = 'prebuilt' | 'runtime' + +export type SandboxImageStatus = 'pending' | 'building' | 'ready' | 'failed' + +/** Handle to an in-flight or completed provider build. */ +export interface SandboxImageBuild { + /** The value passed back as {@link CreateSandboxOptions.imageRef}. */ + imageRef: string + buildId: string + /** Provider-side image identifier, when it differs from the human-facing ref. */ + providerImageId?: string +} + +export interface SandboxImageBuildStatus { + status: SandboxImageStatus + error?: SandboxBuildError + /** Provider log tail, kept for the failure disclosure. */ + logs?: string +} + +/** + * Build side of a `prebuilt` provider. Split from {@link SandboxProvider} so a + * `runtime` provider simply omits it and the type makes that unambiguous. + */ +export interface SandboxImageBuilder { + startBuild(spec: SandboxSpec, specHash: string): Promise + /** `spec` is carried through so a failure can be classified against the right registry. */ + getBuildStatus(build: SandboxImageBuild, spec: SandboxSpec): Promise + /** Removes a built image from the provider. Used by the retention sweep. */ + deleteImage(build: SandboxImageBuild): Promise + /** + * Whether a failure from {@link SandboxProvider.create} means the image itself + * is gone, rather than anything else that can go wrong reaching the provider. + * + * Only a `prebuilt` provider can answer this, and it is what makes a dead + * `imageRef` self-correcting: the registry and the provider are two systems with + * no shared transaction, so instead of trying to keep them in step, the create + * that observes the truth reports it. Must stay narrow — treating an auth or + * rate-limit failure as a missing image would rebuild on every outage. + */ + isMissingImage(error: unknown): Promise +} + export interface SandboxProvider { readonly id: SandboxProviderId + readonly dependencyStrategy: SandboxDependencyStrategy + /** Present exactly when {@link dependencyStrategy} is `prebuilt`. */ + readonly images?: SandboxImageBuilder create(kind: SandboxKind, options?: CreateSandboxOptions): Promise } diff --git a/apps/sim/lib/execution/remote-sandbox/wand-enricher.ts b/apps/sim/lib/execution/remote-sandbox/wand-enricher.ts new file mode 100644 index 00000000000..16c038b5003 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/wand-enricher.ts @@ -0,0 +1,170 @@ +import { db } from '@sim/db' +import { workspaceSandbox } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { and, eq } from 'drizzle-orm' +import { CodeLanguage } from '@/lib/execution/languages' +import { + MAX_SANDBOX_DEPENDENCIES, + parseJsDependency, + registryFor, + type SandboxLanguage, +} from '@/lib/execution/remote-sandbox/sandbox-spec' + +const logger = createLogger('SandboxWandEnricher') + +/** Registry lookups are a nice-to-have; a slow one must never delay generation. */ +const REGISTRY_TIMEOUT_MS = 2_000 + +/** + * Package metadata barely changes, so a process-lifetime cache is enough — but it + * is keyed by an unbounded space (every package any workspace ever declares), so + * it evicts oldest-first rather than growing forever. + */ +const METADATA_CACHE_LIMIT = 500 +const metadataCache = new Map() + +function rememberMetadata(key: string, metadata: PackageMetadata | null): void { + if (metadataCache.size >= METADATA_CACHE_LIMIT) { + const oldest = metadataCache.keys().next() + if (!oldest.done) metadataCache.delete(oldest.value) + } + metadataCache.set(key, metadata) +} + +interface PackageMetadata { + name: string + version?: string + summary?: string +} + +/** Strips the version specifier, leaving the bare distribution/package name. */ +function bareName(language: SandboxLanguage, dependency: string): string { + if (language === CodeLanguage.Python) { + const boundary = dependency.search(/[[<>=!~]/) + return boundary === -1 ? dependency : dependency.slice(0, boundary) + } + return parseJsDependency(dependency).name +} + +async function fetchJson(url: string, signal: AbortSignal): Promise { + // boundary-raw-fetch: external-origin request to a public package registry + const response = await fetch(url, { signal, headers: { accept: 'application/json' } }) + if (!response.ok) return null + return response.json() +} + +/** + * Fetches name, resolved version, and one-line summary from the public registry. + * + * `/api/wand` is a single-shot completion with no tools, so live documentation + * retrieval is not available. This grounds *which* library and *which* version + * without an LLM tool loop. Any failure returns null and the caller degrades to + * the bare name — a registry hiccup must not fail the generation. + */ +async function fetchPackageMetadata( + language: SandboxLanguage, + name: string, + signal: AbortSignal +): Promise { + const cacheKey = `${language}:${name}` + const cached = metadataCache.get(cacheKey) + if (cached !== undefined) return cached + + let metadata: PackageMetadata | null = null + try { + if (language === CodeLanguage.Python) { + const body = (await fetchJson( + `https://pypi.org/pypi/${encodeURIComponent(name)}/json`, + signal + )) as { info?: { name?: string; version?: string; summary?: string } } | null + if (body?.info) { + metadata = { + name: body.info.name ?? name, + version: body.info.version, + summary: body.info.summary ?? undefined, + } + } + } else { + // `/latest` rather than the full packument: the root document carries every + // published version's manifest and runs to megabytes for a popular package, + // which would blow the timeout budget below just to read three fields. + const body = (await fetchJson( + `https://registry.npmjs.org/${name.split('/').map(encodeURIComponent).join('/')}/latest`, + signal + )) as { name?: string; version?: string; description?: string } | null + if (body?.name) { + metadata = { + name: body.name, + version: body.version, + summary: body.description ?? undefined, + } + } + } + } catch (error) { + logger.warn('Package registry lookup failed', { name, error }) + } + + // Only successes are cached. A miss here is usually the shared abort deadline + // below firing, and caching that would permanently degrade the prompt for a + // package that is perfectly resolvable on the next attempt. + if (metadata) rememberMetadata(cacheKey, metadata) + return metadata +} + +function describe(dependency: string, metadata: PackageMetadata | null): string { + if (!metadata) return `- ${dependency}` + const version = metadata.version ? ` (latest ${metadata.version})` : '' + const summary = metadata.summary ? ` — ${metadata.summary}` : '' + return `- ${dependency}${version}${summary}` +} + +/** + * Wand enricher that tells the model which packages the block can actually + * import. With no sandbox selected it returns null, leaving today's prompt + * unchanged. + */ +export async function enrichSandboxPackages( + workspaceId: string | null, + context: Record +): Promise { + const sandboxId = context.sandboxId as string | undefined + if (!sandboxId || !workspaceId) return null + + const [sandbox] = await db + .select({ + name: workspaceSandbox.name, + language: workspaceSandbox.language, + dependencies: workspaceSandbox.dependencies, + }) + .from(workspaceSandbox) + .where(and(eq(workspaceSandbox.id, sandboxId), eq(workspaceSandbox.workspaceId, workspaceId))) + .limit(1) + + if (!sandbox) return null + const language = sandbox.language as SandboxLanguage + const dependencies = (sandbox.dependencies ?? []).slice(0, MAX_SANDBOX_DEPENDENCIES) + if (dependencies.length === 0) return null + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), REGISTRY_TIMEOUT_MS) + let metadata: (PackageMetadata | null)[] + try { + metadata = await Promise.all( + dependencies.map((dependency) => + fetchPackageMetadata(language, bareName(language, dependency), controller.signal).catch( + () => null + ) + ) + ) + } finally { + clearTimeout(timer) + } + + const registry = registryFor(language) + const lines = dependencies.map((dependency, index) => describe(dependency, metadata[index])) + return [ + `This block runs in the "${sandbox.name}" sandbox, which has these ${registry} packages installed:`, + ...lines, + 'You may import any of them. Do NOT import a package that is not on this list — it is not installed and the code will fail.', + ].join('\n') +} diff --git a/apps/sim/lib/execution/remote-sandbox/workspace-sandboxes.ts b/apps/sim/lib/execution/remote-sandbox/workspace-sandboxes.ts new file mode 100644 index 00000000000..bbea96adcda --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/workspace-sandboxes.ts @@ -0,0 +1,206 @@ +import { db } from '@sim/db' +import { sandboxImage, workspaceSandbox } from '@sim/db/schema' +import { and, eq, inArray } from 'drizzle-orm' +import type { Sandbox } from '@/lib/api/contracts/sandboxes' +import { ensureSandboxImage } from '@/lib/execution/remote-sandbox/image-registry' +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' +import { invalidateSandboxResolution } from '@/lib/execution/remote-sandbox/resolve' +import { + type DependencyIssue, + hashSandboxSpec, + type SandboxLanguage, + validateDependencies, +} from '@/lib/execution/remote-sandbox/sandbox-spec' +import type { SandboxDependencyStrategy } from '@/lib/execution/remote-sandbox/types' + +/** 403 copy for a workspace whose plan does not include sandbox authoring. */ +export const MAX_PLAN_REQUIRED = 'Sandboxes require an active Max or Enterprise plan.' + +export const SANDBOX_ADMIN_REQUIRED = 'Only workspace admins can manage sandboxes' + +/** + * The unique index that actually arbitrates sandbox-name collisions. Named here + * so a write path can recognize losing the race and answer 409 rather than 500. + */ +export const WORKSPACE_SANDBOX_NAME_INDEX = 'workspace_sandbox_workspace_name_unique' + +/** + * Builds cost provider compute, so every mutation shares one per-workspace + * budget rather than giving each admin a full allowance of their own. + */ +export const SANDBOX_MUTATION_LIMIT = { + maxTokens: 20, + refillRate: 10, + refillIntervalMs: 60_000, +} as const + +/** Thrown when a submitted dependency list has lines the editor should mark. */ +export class SandboxDependencyError extends Error { + constructor(readonly issues: DependencyIssue[]) { + super(issues[0]?.reason ?? 'Invalid dependency list') + this.name = 'SandboxDependencyError' + } +} + +export interface SandboxSpecUpdate { + language: SandboxLanguage + dependencies: string[] + specHash: string +} + +/** + * Validates a submitted list against the target language and returns the + * canonical spec. Called on every write, including a language change, so a list + * that was valid Python does not survive a switch to JavaScript unchecked. + */ +export function buildSpecUpdate( + language: SandboxLanguage, + submitted: readonly string[] +): SandboxSpecUpdate { + const validation = validateDependencies(language, submitted) + if (!validation.ok) throw new SandboxDependencyError(validation.issues) + return { + language, + dependencies: validation.dependencies, + specHash: hashSandboxSpec({ language, dependencies: validation.dependencies }), + } +} + +export function currentSandboxStrategy(): SandboxDependencyStrategy { + return resolveProvider().dependencyStrategy +} + +interface SandboxRow { + id: string + name: string + language: string + dependencies: string[] | null + createdAt: Date + updatedAt: Date +} + +interface ImageRow { + status: string + errorCode: string | null + errorMessage: string | null + errorDetail: string | null + updatedAt: Date +} + +function toSandbox(row: SandboxRow, image: ImageRow | undefined): Sandbox { + return { + id: row.id, + name: row.name, + language: row.language as Sandbox['language'], + dependencies: row.dependencies ?? [], + // A runtime-strategy deployment has no build, so the status is genuinely absent + // rather than pending — the UI branches on that to explain the tradeoff. + buildStatus: (image?.status as Sandbox['buildStatus']) ?? null, + errorCode: image?.errorCode ?? null, + errorMessage: image?.errorMessage ?? null, + errorDetail: image?.errorDetail ?? null, + builtAt: image?.status === 'ready' ? image.updatedAt.toISOString() : null, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + } +} + +/** + * Joins the build registry onto a set of sandbox rows. + * + * `sandbox_image` is content-addressed and shared across every workspace, so it + * MUST be filtered by the spec hashes actually in play — selecting the whole + * provider's rows would read the entire platform's build table (log tails and + * all) to render one workspace's few sandboxes. + */ +async function attachBuildStatus(rows: (SandboxRow & { specHash: string })[]): Promise { + const provider = resolveProvider() + if (provider.dependencyStrategy !== 'prebuilt' || rows.length === 0) { + return rows.map((row) => toSandbox(row, undefined)) + } + + const specHashes = [...new Set(rows.map((row) => row.specHash))] + const images = await db + .select({ + specHash: sandboxImage.specHash, + status: sandboxImage.status, + errorCode: sandboxImage.errorCode, + errorMessage: sandboxImage.errorMessage, + errorDetail: sandboxImage.errorDetail, + updatedAt: sandboxImage.updatedAt, + }) + .from(sandboxImage) + .where(and(eq(sandboxImage.provider, provider.id), inArray(sandboxImage.specHash, specHashes))) + + const byHash = new Map(images.map((image) => [image.specHash, image])) + return rows.map((row) => toSandbox(row, byHash.get(row.specHash))) +} + +const SANDBOX_COLUMNS = { + id: workspaceSandbox.id, + name: workspaceSandbox.name, + language: workspaceSandbox.language, + dependencies: workspaceSandbox.dependencies, + specHash: workspaceSandbox.specHash, + createdAt: workspaceSandbox.createdAt, + updatedAt: workspaceSandbox.updatedAt, +} as const + +/** + * Lists a workspace's sandboxes with their build status. Under a runtime + * provider the registry is never consulted, because it is never written. + */ +export async function listWorkspaceSandboxes(workspaceId: string): Promise { + const rows = await db + .select(SANDBOX_COLUMNS) + .from(workspaceSandbox) + .where(eq(workspaceSandbox.workspaceId, workspaceId)) + .orderBy(workspaceSandbox.name) + + return attachBuildStatus(rows) +} + +/** + * Reads one sandbox back, scoped to its workspace. Fetches the single row rather + * than filtering a full list — this runs after every create and update. + */ +export async function readWorkspaceSandbox( + workspaceId: string, + sandboxId: string +): Promise { + const rows = await db + .select(SANDBOX_COLUMNS) + .from(workspaceSandbox) + .where(and(eq(workspaceSandbox.id, sandboxId), eq(workspaceSandbox.workspaceId, workspaceId))) + .limit(1) + + const [sandbox] = await attachBuildStatus(rows) + return sandbox ?? null +} + +/** + * Enqueues a build for a spec, if the active provider prebuilds. Invalidating + * the resolution cache first means an execution started right after a save never + * reads the previous image for the edited sandbox. + */ +export async function scheduleSandboxBuild(spec: SandboxSpecUpdate): Promise { + invalidateSandboxResolution() + await ensureSandboxImage( + { language: spec.language, dependencies: spec.dependencies }, + spec.specHash + ) +} + +/** True when a name is already taken in the workspace by a different sandbox. */ +export async function isSandboxNameTaken( + workspaceId: string, + name: string, + excludeId?: string +): Promise { + const [existing] = await db + .select({ id: workspaceSandbox.id }) + .from(workspaceSandbox) + .where(and(eq(workspaceSandbox.workspaceId, workspaceId), eq(workspaceSandbox.name, name))) + .limit(1) + return Boolean(existing && existing.id !== excludeId) +} diff --git a/apps/sim/lib/workflows/autolayout/utils.ts b/apps/sim/lib/workflows/autolayout/utils.ts index 9d340d0e9d5..ff18c4129b7 100644 --- a/apps/sim/lib/workflows/autolayout/utils.ts +++ b/apps/sim/lib/workflows/autolayout/utils.ts @@ -20,6 +20,7 @@ import { isSubBlockFeatureEnabled, isSubBlockHidden, isSubBlockVisibleForMode, + isToolInputOnlySubBlock, isTriggerModeSubBlock, } from '@/lib/workflows/subblocks/visibility' import { getBlock } from '@/blocks' @@ -172,6 +173,10 @@ function getVisiblePreviewSubBlockCount(block: BlockState): number { return blockConfig.subBlocks.filter((subBlock) => { if (subBlock.hidden || subBlock.hideFromPreview) return false if (!isSubBlockFeatureEnabled(subBlock)) return false + // Never rendered on the canvas, so it must not contribute to node height — + // this filter has to agree with `workflow-block.tsx`'s or blocks lay out + // with a phantom row of space. + if (isToolInputOnlySubBlock(subBlock)) return false if (isSubBlockHidden(subBlock)) return false if (effectiveTrigger) { diff --git a/apps/sim/lib/workflows/subblocks/display.test.ts b/apps/sim/lib/workflows/subblocks/display.test.ts index 310af5d3701..5be68fcca06 100644 --- a/apps/sim/lib/workflows/subblocks/display.test.ts +++ b/apps/sim/lib/workflows/subblocks/display.test.ts @@ -15,6 +15,7 @@ import { getDisplayValue, resolveDropdownLabel, resolveFilterFieldLabel, + resolveSandboxLabel, resolveSkillsLabel, resolveToolsLabel, resolveVariablesLabel, @@ -33,6 +34,7 @@ const workflowMulti = { const variablesInput = { id: 'variables', type: 'variables-input' } as SubBlockConfig const toolInput = { id: 'tools', type: 'tool-input' } as SubBlockConfig const skillInput = { id: 'skills', type: 'skill-input' } as SubBlockConfig +const sandboxPicker = { id: 'sandboxId', type: 'combobox' } as SubBlockConfig describe('summarizeNames', () => { it('formats 0, 1, 2, and 2+N name lists', () => { @@ -184,6 +186,28 @@ describe('resolveSkillsLabel', () => { }) }) +describe('resolveSandboxLabel', () => { + const sandboxes = [{ id: '443f4934-26ab-44ab-8000-000000000000', name: 'Test' }] + + it('resolves the stored id to the name so the card never shows a uuid', () => { + expect(resolveSandboxLabel(sandboxPicker, sandboxes[0].id, sandboxes)).toBe('Test') + }) + + it('returns null for an id the workspace no longer has', () => { + expect(resolveSandboxLabel(sandboxPicker, 'sbx-deleted', sandboxes)).toBeNull() + }) + + it('returns null before the list loads, and for an empty selection', () => { + expect(resolveSandboxLabel(sandboxPicker, sandboxes[0].id, [])).toBeNull() + expect(resolveSandboxLabel(sandboxPicker, '', sandboxes)).toBeNull() + }) + + it('ignores other comboboxes so it cannot relabel an unrelated field', () => { + const other = { id: 'model', type: 'combobox' } as SubBlockConfig + expect(resolveSandboxLabel(other, sandboxes[0].id, sandboxes)).toBeNull() + }) +}) + describe('resolveDropdownLabel', () => { const dropdown = { id: 'mode', diff --git a/apps/sim/lib/workflows/subblocks/display.ts b/apps/sim/lib/workflows/subblocks/display.ts index 26088c25fff..17689f68818 100644 --- a/apps/sim/lib/workflows/subblocks/display.ts +++ b/apps/sim/lib/workflows/subblocks/display.ts @@ -544,3 +544,26 @@ export function resolveSkillsLabel( return summarizeNames(names) } + +/** + * Resolves the Function block's stored sandbox id to the sandbox name. + * + * Unlike its siblings there is no dedicated subblock type to match on: the picker + * is a plain `combobox` whose options load asynchronously, so its static + * `options` array is empty and {@link resolveDropdownLabel} finds nothing — + * leaving the block card printing a raw UUID. Matching the field id is what + * narrows this to the one picker that needs it. + * + * An id with no matching sandbox returns `null` rather than a guess, so a deleted + * sandbox falls through to the caller's own placeholder. + */ +export function resolveSandboxLabel( + subBlock: SubBlockConfig | undefined, + rawValue: unknown, + sandboxes: Array<{ id: string; name: string }> +): string | null { + if (subBlock?.id !== 'sandboxId' || subBlock.type !== 'combobox') return null + if (typeof rawValue !== 'string' || !rawValue) return null + + return sandboxes.find((sandbox) => sandbox.id === rawValue)?.name ?? null +} diff --git a/apps/sim/lib/workflows/subblocks/options.ts b/apps/sim/lib/workflows/subblocks/options.ts index 526dcb0d826..9c6dc78216a 100644 --- a/apps/sim/lib/workflows/subblocks/options.ts +++ b/apps/sim/lib/workflows/subblocks/options.ts @@ -1,6 +1,14 @@ +import { fetchPersonalEnvironment, fetchWorkspaceEnvironment } from '@/lib/environment/api' import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { + environmentKeys, + PERSONAL_ENVIRONMENT_STALE_TIME, + WORKSPACE_ENVIRONMENT_STALE_TIME, +} from '@/hooks/queries/environment' +import { getSandboxListQueryOptions, type SandboxListResponse } from '@/hooks/queries/sandboxes' import { getWorkflowListQueryOptions } from '@/hooks/queries/utils/workflow-list-query' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' +import { useSubBlockStore } from '@/stores/workflows/subblock/store' interface SubBlockOption { label: string @@ -30,3 +38,113 @@ export async function fetchWorkspaceWorkflowOptions(options?: { ) .map((workflow) => ({ id: workflow.id, label: workflow.name })) } + +/** + * Loads the active workspace's secret NAMES for the Function block's secret-scope + * picker. Names only — values stay server-side and are injected at execution, the + * same discipline the copilot's workspace context uses. + */ +export async function fetchWorkspaceSecretNameOptions(): Promise { + const workspaceId = useWorkflowRegistry.getState().hydration.workspaceId + if (!workspaceId) return [] + + const [workspace, personal] = await Promise.all([ + getQueryClient().fetchQuery({ + queryKey: environmentKeys.workspace(workspaceId), + queryFn: ({ signal }: { signal?: AbortSignal }) => + fetchWorkspaceEnvironment(workspaceId, signal), + staleTime: WORKSPACE_ENVIRONMENT_STALE_TIME, + }), + getQueryClient().fetchQuery({ + queryKey: environmentKeys.personal(), + queryFn: ({ signal }: { signal?: AbortSignal }) => fetchPersonalEnvironment(signal), + staleTime: PERSONAL_ENVIRONMENT_STALE_TIME, + }), + ]) + + // Personal variables shadow workspace ones at execution, so both are offered + // under one de-duplicated list of names. + const names = new Set([ + ...Object.keys(workspace?.workspace?.variables ?? {}), + ...Object.keys(personal ?? {}), + ]) + return [...names].sort().map((name) => ({ id: name, label: name })) +} + +/** + * Labels a sandbox for the picker. The name is what identifies it, so that is all + * the label carries by default — the block's own list is already scoped to one + * language, where repeating it on every row is noise. + * + * `showLanguage` serves the one caller that cannot filter: agent tool-input + * renders this field under a synthetic id where the sibling `language` value is + * unreachable, so its list spans both languages and the name alone is ambiguous. + * + * A failed build is always marked. It is the difference between a selection that + * runs and one that does not, so it is not decoration. + */ +function toSandboxOption( + sandbox: { + id: string + name: string + language: string + buildStatus: string | null + }, + options?: { showLanguage?: boolean } +): SubBlockOption { + const parts = [sandbox.name] + if (options?.showLanguage) { + parts.push(sandbox.language === 'python' ? 'Python' : 'JavaScript') + } + if (sandbox.buildStatus === 'failed') parts.push('build failed') + return { id: sandbox.id, label: parts.join(' · ') } +} + +async function loadWorkspaceSandboxes(): Promise { + const workspaceId = useWorkflowRegistry.getState().hydration.workspaceId + if (!workspaceId) return [] + const data = await getQueryClient().fetchQuery(getSandboxListQueryOptions(workspaceId)) + return data.sandboxes +} + +/** + * Loads the sandboxes a Function block can run in, scoped to the language its + * sibling `language` subblock selects — a Python block must never be offered an + * npm sandbox. The block re-fetches when `language` changes (`dependsOn`). + */ +export async function fetchWorkspaceSandboxOptions(blockId: string): Promise { + const language = useSubBlockStore.getState().getValue(blockId, 'language') + const sandboxes = await loadWorkspaceSandboxes() + // The missing `language` that makes filtering impossible is exactly what makes + // the language worth showing, so the two stay in lockstep. + const showLanguage = !language + return sandboxes + .filter((sandbox) => !language || sandbox.language === language) + .map((sandbox) => toSandboxOption(sandbox, { showLanguage })) +} + +/** + * Hydrates a stored sandbox id to its label before the option list loads. + * + * A selection left over from before a language switch is still shown, flagged + * rather than hidden. Returning `null` here would drop the field back to its + * "Default image" placeholder while the value stayed stored + * and stayed fatal at execution — the field would read as cleared and the run + * would still fail, with nothing to point at. Labelling it is what lets the + * author see what to fix. + */ +export async function fetchWorkspaceSandboxOption( + blockId: string, + optionId: string +): Promise { + const language = useSubBlockStore.getState().getValue(blockId, 'language') + const sandboxes = await loadWorkspaceSandboxes() + const sandbox = sandboxes.find((candidate) => candidate.id === optionId) + if (!sandbox) return null + + const option = toSandboxOption(sandbox, { showLanguage: !language }) + if (language && sandbox.language !== language) { + return { ...option, label: `${option.label} · wrong language for this block` } + } + return option +} diff --git a/apps/sim/lib/workflows/subblocks/visibility.ts b/apps/sim/lib/workflows/subblocks/visibility.ts index b64fc025a2b..ef588530a03 100644 --- a/apps/sim/lib/workflows/subblocks/visibility.ts +++ b/apps/sim/lib/workflows/subblocks/visibility.ts @@ -518,12 +518,37 @@ export function resolveDependencyValue( return values[dependencyKey] } +/** + * Whether a subblock only applies when the block is used as an agent tool. + * + * `paramVisibility` filters what appears *inside* tool-input but cannot hide a + * subblock from the canvas, so this is a separate axis rather than another + * visibility level. + */ +export function isToolInputOnlySubBlock(subBlock: Pick): boolean { + return subBlock.context === 'tool-input' +} + +/** + * Whether any env var named by an env-gate spec is truthy. + * + * A gate may name several vars, comma-separated, meaning "any of these" — that + * is what lets a renamed flag ship without every existing deployment losing the + * field until it sets the new var. Shared by both gates so the two cannot + * interpret their value differently. + */ +function anyEnvSet(spec: string): boolean { + return spec.split(',').some((name) => isTruthy(getEnv(name.trim()))) +} + /** * Check if a subblock is gated by a feature flag. */ -export function isSubBlockFeatureEnabled(subBlock: SubBlockConfig): boolean { +export function isSubBlockFeatureEnabled( + subBlock: Pick +): boolean { if (!subBlock.showWhenEnvSet) return true - return isTruthy(getEnv(subBlock.showWhenEnvSet)) + return anyEnvSet(subBlock.showWhenEnvSet) } /** @@ -539,6 +564,6 @@ export function isSubBlockHidden( ): boolean { const hosted = options?.hosted ?? isHosted if (subBlock.hideWhenHosted && hosted) return true - if (subBlock.hideWhenEnvSet && isTruthy(getEnv(subBlock.hideWhenEnvSet))) return true + if (subBlock.hideWhenEnvSet && anyEnvSet(subBlock.hideWhenEnvSet)) return true return false } diff --git a/apps/sim/lib/workspaces/admin-move.ts b/apps/sim/lib/workspaces/admin-move.ts index fa6243fdaad..ef6fcf160c3 100644 --- a/apps/sim/lib/workspaces/admin-move.ts +++ b/apps/sim/lib/workspaces/admin-move.ts @@ -16,7 +16,12 @@ import { generateId } from '@sim/utils/id' import { normalizeEmail } from '@sim/utils/string' import { and, asc, count, eq, ilike, inArray, isNull, ne, or, sql } from 'drizzle-orm' import { acquireOrganizationMutationLock } from '@/lib/billing/organizations/membership' +import { isEnterprise } from '@/lib/billing/plan-helpers' import { changeWorkspaceStoragePayerInTx } from '@/lib/billing/storage/payer-transfer' +import { + ENTITLED_SUBSCRIPTION_STATUSES, + hasPaidSubscriptionStatus, +} from '@/lib/billing/subscriptions/utils' import { enqueueOutboxEvent, type OutboxHandler } from '@/lib/core/outbox/service' import type { DbOrTx } from '@/lib/db/types' import { getInvitationById } from '@/lib/invitations/core' @@ -31,7 +36,6 @@ import { import { WORKSPACE_MODE } from '@/lib/workspaces/policy' const logger = createLogger('AdminWorkspaceMove') -const ENTITLED_STATUSES = ['active', 'past_due'] as const export class WorkspaceMoveError extends Error { constructor( @@ -203,7 +207,7 @@ export async function getWorkspaceMovePreflight( .where( and( eq(subscription.referenceId, destinationOrganizationId), - inArray(subscription.status, [...ENTITLED_STATUSES]) + inArray(subscription.status, ENTITLED_SUBSCRIPTION_STATUSES) ) ) .limit(1), @@ -546,7 +550,7 @@ function getEnterpriseSeatCapacity(row?: { status: string | null metadata: unknown }): number | null { - if (!row || row.plan !== 'enterprise' || !ENTITLED_STATUSES.includes(row.status as 'active')) { + if (!row || !isEnterprise(row.plan) || !hasPaidSubscriptionStatus(row.status)) { return null } if (!row.metadata || typeof row.metadata !== 'object') return null diff --git a/apps/sim/lib/workspaces/policy.test.ts b/apps/sim/lib/workspaces/policy.test.ts index 806ede152f8..82918d27bd2 100644 --- a/apps/sim/lib/workspaces/policy.test.ts +++ b/apps/sim/lib/workspaces/policy.test.ts @@ -143,6 +143,76 @@ describe('getWorkspaceCreationPolicy', () => { expect(result.currentWorkspaceCount).toBe(5) }) + // The Max cap previously read `isMax`, which required `isPro` and so excluded + // both `team_25000` and `enterprise`. Those tiers fell to the `isPro ? 3 : 1` + // branch and got ONE personal workspace — fewer than a plain Pro's three. + it('gives the team plan at the Max credit tier the same ten personal workspaces as Max', async () => { + mockGetUserOrganization.mockResolvedValue({ + organizationId: 'org-1', + role: 'owner', + memberId: 'member-1', + }) + // A past_due org subscription is not `hasUsableSubscriptionStatus`, so the + // organization branch does not apply and the personal cap decides. + mockGetOrganizationSubscription.mockResolvedValue({ + id: 'sub-1', + plan: 'team_25000', + status: 'past_due', + }) + mockGetHighestPrioritySubscription.mockResolvedValueOnce({ + id: 'sub-1', + plan: 'team_25000', + status: 'past_due', + }) + queueTableRows(workspace, [{ value: 5 }]) + + const result = await getWorkspaceCreationPolicy({ userId: 'user-1' }) + + expect(result.canCreate).toBe(true) + expect(result.workspaceMode).toBe(WORKSPACE_MODE.PERSONAL) + expect(result.maxWorkspaces).toBe(10) + }) + + it('gives an enterprise payer ten personal workspaces despite carrying no credit suffix', async () => { + mockGetHighestPrioritySubscription.mockResolvedValueOnce({ + id: 'sub-1', + plan: 'enterprise', + status: 'active', + }) + queueTableRows(workspace, [{ value: 5 }]) + + const result = await getWorkspaceCreationPolicy({ userId: 'user-1' }) + + expect(result.canCreate).toBe(true) + expect(result.workspaceMode).toBe(WORKSPACE_MODE.PERSONAL) + expect(result.maxWorkspaces).toBe(10) + }) + + // The personal cap is only a fallback: an enterprise org admin is routed to + // organization mode and is uncapped, which is why the bug above stayed hidden. + it('leaves enterprise organization workspaces uncapped for org admins', async () => { + mockGetUserOrganization.mockResolvedValueOnce({ + organizationId: 'org-1', + role: 'owner', + memberId: 'member-1', + }) + mockGetOrganizationSubscription.mockResolvedValueOnce({ + id: 'sub-1', + plan: 'enterprise', + status: 'active', + }) + queueTableRows(member, [{ userId: 'owner-1' }]) + + const result = await getWorkspaceCreationPolicy({ + userId: 'user-1', + activeOrganizationId: 'org-1', + }) + + expect(result.canCreate).toBe(true) + expect(result.workspaceMode).toBe(WORKSPACE_MODE.ORGANIZATION) + expect(result.maxWorkspaces).toBeNull() + }) + it('blocks max users once they already own ten personal workspaces', async () => { mockGetHighestPrioritySubscription.mockResolvedValueOnce({ id: 'sub-1', diff --git a/apps/sim/lib/workspaces/policy.ts b/apps/sim/lib/workspaces/policy.ts index 70d3cdb06d2..6fc9218487f 100644 --- a/apps/sim/lib/workspaces/policy.ts +++ b/apps/sim/lib/workspaces/policy.ts @@ -7,7 +7,7 @@ import { getOrganizationSubscription } from '@/lib/billing/core/billing' import { getHighestPrioritySubscription } from '@/lib/billing/core/plan' import { getUserOrganization } from '@/lib/billing/organizations/membership' import type { PlanCategory } from '@/lib/billing/plan-helpers' -import { getPlanType, isEnterprise, isMax, isPro, isTeam } from '@/lib/billing/plan-helpers' +import { getPlanType, isEnterprise, isMaxTier, isPro, isTeam } from '@/lib/billing/plan-helpers' import { hasUsableSubscriptionStatus } from '@/lib/billing/subscriptions/utils' import { isBillingEnabled } from '@/lib/core/config/env-flags' import type { DbOrTx } from '@/lib/db/types' @@ -411,7 +411,17 @@ export async function getWorkspaceCreationPolicy({ const highestPrioritySubscription = await getHighestPrioritySubscription(userId) const plan = highestPrioritySubscription?.plan - const maxWorkspaces = isMax(plan) ? 10 : isPro(plan) ? 3 : 1 + /** + * Personal (non-organization) workspace cap. Organization workspaces are + * uncapped and returned above, so this is only reached when the org branch does + * not apply — including when a Team/Enterprise org's subscription is `past_due` + * and therefore not `hasUsableSubscriptionStatus`. + * + * Deliberately tier-only: `getHighestPrioritySubscription` already admits + * `past_due`, and delinquency is enforced by the billing-blocked gates rather + * than by shrinking the cap, which would only obstruct recovery. + */ + const maxWorkspaces = isMaxTier(plan) ? 10 : isPro(plan) ? 3 : 1 const currentWorkspaceCount = await countNonOrganizationOwnedWorkspaces(userId) if (currentWorkspaceCount >= maxWorkspaces) { diff --git a/apps/sim/providers/types.ts b/apps/sim/providers/types.ts index 2bb1a9d31c0..1538846e58b 100644 --- a/apps/sim/providers/types.ts +++ b/apps/sim/providers/types.ts @@ -129,6 +129,13 @@ export interface ProviderToolConfig { required: string[] } usageControl?: ToolUsageControl + /** + * Params the model may never supply, because the tool declares them + * `user-only` or `hidden`. Stripped from the model's arguments before they + * merge with the user's — omitting them from {@link ProviderToolConfig.parameters} + * alone does not stop a model from emitting one anyway. + */ + modelBlockedParams?: string[] /** Block-level params transformer — converts SubBlock values to tool-ready params */ paramsTransform?: (params: Record) => Record } diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index 05d60e37a12..5b2f0febaf7 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -581,6 +581,45 @@ function buildCustomBlockInputMappingSchema( * @param options Additional options including dependencies and selected operation * @returns The provider tool config or null if transform fails */ +/** + * Drops model-supplied arguments for params the tool declares off-limits to the + * model (`user-only` / `hidden`). + * + * Deliberately keyed on the declared visibility rather than on "absent from + * `parameters.properties`": an MCP or custom tool may legitimately accept keys + * beyond its advertised properties (`additionalProperties`), and silently + * dropping those would truncate its arguments. Only a param the tool itself + * marked as not-for-the-model is removed. + */ +function stripModelBlockedParams( + blockedParams: string[] | undefined, + llmArgs: Record +): Record { + if (!blockedParams?.length) return llmArgs + const blocked = new Set(blockedParams) + const filtered: Record = {} + for (const [key, value] of Object.entries(llmArgs)) { + if (!blocked.has(key)) filtered[key] = value + } + return filtered +} + +/** Reads a multi-select value that may still be JSON-encoded from `StoredTool.params`. */ +function readMountedSecretNames(raw: unknown): string[] { + let value = raw + if (typeof value === 'string') { + try { + value = JSON.parse(value) + } catch { + // A bare name rather than a list — treat it as a single entry. + return value ? [value as string] : [] + } + } + return Array.isArray(value) + ? value.filter((name): name is string => typeof name === 'string' && name.length > 0) + : [] +} + export async function transformBlockTool( block: any, options: { @@ -715,10 +754,11 @@ export async function transformBlockTool( const userProvidedParams = block.params || {} - const { schema: llmSchema, enrichedDescription } = await createLLMToolSchema( - toolConfig, - userProvidedParams - ) + const { + schema: llmSchema, + enrichedDescription, + modelBlockedParams, + } = await createLLMToolSchema(toolConfig, userProvidedParams) const canonicalGroups: CanonicalGroup[] = blockDef?.subBlocks ? Object.values(buildCanonicalIndex(blockDef.subBlocks).groupsById).filter(isCanonicalPair) @@ -747,6 +787,16 @@ export async function transformBlockTool( toolDescription = workflowMetadata.description } } + } else if (toolId === 'function_execute' && resolvedResourceParams.secretScope === 'selected') { + // Scoping alone would leave the model guessing: the secrets are injected + // server-side and nothing else advertises them. Names only — values never + // enter the provider request, matching the copilot's workspace-context rule. + // `StoredTool.params` holds strings, so a multi-select arrives JSON-encoded; + // the executor's paramsTransform parses it later, but this runs before that. + const mounted = readMountedSecretNames(resolvedResourceParams.mountedSecrets) + toolDescription = mounted.length + ? `${toolDescription}\n\nWorkspace secrets available to this code: ${mounted.join(', ')}. Reference one as {{NAME}} or environmentVariables['NAME']. No other secrets are readable.` + : `${toolDescription}\n\nThis code has no access to workspace secrets.` } else if (toolId.startsWith('knowledge_') && resolvedResourceParams.knowledgeBaseId) { uniqueToolId = `${toolConfig.id}_${resolvedResourceParams.knowledgeBaseId}` } else if (toolId.startsWith('table_') && resolvedResourceParams.tableId) { @@ -812,6 +862,7 @@ export async function transformBlockTool( description: toolDescription, params: userProvidedParams, parameters: llmSchema, + modelBlockedParams, paramsTransform, } } @@ -1408,6 +1459,7 @@ export function prepareToolExecution( tool: { params?: Record parameters?: Record + modelBlockedParams?: string[] paramsTransform?: (params: Record) => Record }, llmArgs: Record, @@ -1428,7 +1480,16 @@ export function prepareToolExecution( toolParams: Record executionParams: Record } { - let toolParams = mergeToolParameters(tool.params || {}, llmArgs) as Record + // Providers are supposed to emit only declared arguments, but nothing enforces + // it on the parsed tool call — and `mergeToolParameters` seeds its result from + // the model's args, so an undeclared key survives whenever the user's value is + // empty. That is a privilege escalation for `user-only` params: a Function tool + // scoped to "Selected secrets" with an empty list is an explicit deny, and a + // model emitting `mountedSecrets: ['STRIPE_KEY']` would otherwise mount it. + let toolParams = mergeToolParameters( + tool.params || {}, + stripModelBlockedParams(tool.modelBlockedParams, llmArgs) + ) as Record if (tool.paramsTransform) { try { diff --git a/apps/sim/scripts/verify-sandbox-parity.ts b/apps/sim/scripts/verify-sandbox-parity.ts index 1820a291b8f..2732a2fff23 100644 --- a/apps/sim/scripts/verify-sandbox-parity.ts +++ b/apps/sim/scripts/verify-sandbox-parity.ts @@ -20,6 +20,11 @@ * DAYTONA_DOC_SNAPSHOT_ID=mothership-docs: \ * bun run apps/sim/scripts/verify-sandbox-parity.ts * + * # Include the dependency-set cases (needs real workspace_sandbox rows): + * SANDBOX_PARITY_WORKSPACE_ID=... \ + * SANDBOX_PARITY_PYTHON_SANDBOX_ID=... # a Python sandbox declaring `pandas` + * SANDBOX_PARITY_JS_SANDBOX_ID=... # a JavaScript sandbox declaring `axios` + * * Exits non-zero if any case fails, so it can be wired to a schedule later. */ @@ -33,9 +38,20 @@ interface Case { name: string /** Skipped unless the doc image is configured for the active provider. */ needsDoc?: boolean + /** + * Skipped unless `SANDBOX_PARITY_PYTHON_SANDBOX_ID` / + * `SANDBOX_PARITY_JS_SANDBOX_ID` name a real workspace sandbox — resolving one + * needs a DB row, so it cannot be synthesized here. + */ + needsSandbox?: 'python' | 'javascript' run: () => Promise<{ ok: boolean; detail: string }> } +/** A workspace + sandbox to resolve the dependency-set cases against. */ +const PARITY_WORKSPACE_ID = process.env.SANDBOX_PARITY_WORKSPACE_ID +const PARITY_PYTHON_SANDBOX_ID = process.env.SANDBOX_PARITY_PYTHON_SANDBOX_ID +const PARITY_JS_SANDBOX_ID = process.env.SANDBOX_PARITY_JS_SANDBOX_ID + const CASES: Case[] = [ { name: 'python: result marker round-trip', @@ -104,6 +120,38 @@ const CASES: Case[] = [ return { ok: platform === 'linux', detail: res.error ?? `platform=${platform}` } }, }, + { + // Prebuilt on E2B, runtime install on Daytona — the same selection must make + // the same import succeed on both. + name: 'python: a selected sandbox makes its packages importable', + needsSandbox: 'python', + run: async () => { + const res = await executeInSandbox({ + code: `import json, pandas\nprint("${SIM_RESULT_PREFIX}" + json.dumps({"v": pandas.__version__}))`, + language: CodeLanguage.Python, + timeoutMs: 300_000, + workspaceId: PARITY_WORKSPACE_ID, + sandboxId: PARITY_PYTHON_SANDBOX_ID, + }) + const version = (res.result as { v?: string } | null)?.v + return { ok: Boolean(version), detail: res.error ?? `pandas=${version}` } + }, + }, + { + name: 'javascript: a selected sandbox resolves its packages', + needsSandbox: 'javascript', + run: async () => { + const res = await executeInSandbox({ + code: `const axios = require('axios')\nconsole.log('${SIM_RESULT_PREFIX}' + JSON.stringify({ ok: typeof axios.get === 'function' }))`, + language: CodeLanguage.JavaScript, + timeoutMs: 300_000, + workspaceId: PARITY_WORKSPACE_ID, + sandboxId: PARITY_JS_SANDBOX_ID, + }) + const ok = (res.result as { ok?: boolean } | null)?.ok + return { ok: ok === true, detail: res.error ?? `resolved=${ok}` } + }, + }, { name: 'shell: env vars + user-authored marker', run: async () => { @@ -169,6 +217,13 @@ async function main() { skipped++ continue } + const sandboxId = + testCase.needsSandbox === 'python' ? PARITY_PYTHON_SANDBOX_ID : PARITY_JS_SANDBOX_ID + if (testCase.needsSandbox && !(PARITY_WORKSPACE_ID && sandboxId)) { + console.log(`SKIP ${testCase.name} (no ${testCase.needsSandbox} sandbox configured)`) + skipped++ + continue + } const started = Date.now() try { const { ok, detail } = await testCase.run() diff --git a/apps/sim/serializer/index.ts b/apps/sim/serializer/index.ts index f018f509ced..73c2b9f145b 100644 --- a/apps/sim/serializer/index.ts +++ b/apps/sim/serializer/index.ts @@ -12,6 +12,7 @@ import { isNonEmptyValue, isSubBlockFeatureEnabled, isSubBlockHidden, + isToolInputOnlySubBlock, resolveCanonicalMode, } from '@/lib/workflows/subblocks/visibility' import { getBlock } from '@/blocks' @@ -52,6 +53,11 @@ function shouldSerializeSubBlock( canonicalModeOverrides?: CanonicalModeOverrides ): boolean { if (!isSubBlockFeatureEnabled(subBlockConfig)) return false + // Only meaningful when the block is invoked as an agent tool, where the + // value lives on the tool entry rather than the block. Serializing it here + // would let a non-UI writer (copilot, YAML import) set an invisible secret + // scope that the executor's env inlining does not honor. + if (isToolInputOnlySubBlock(subBlockConfig)) return false if (isSubBlockHidden(subBlockConfig)) return false if (subBlockConfig.mode === 'trigger') { diff --git a/apps/sim/tools/function/execute.ts b/apps/sim/tools/function/execute.ts index 857799f65ef..b9e19b891a9 100644 --- a/apps/sim/tools/function/execute.ts +++ b/apps/sim/tools/function/execute.ts @@ -85,6 +85,24 @@ export const functionExecuteTool: ToolConfig } + /** Workspace sandbox whose dependency set this execution runs against. */ + sandboxId?: string + /** + * Which workspace secrets the code may read. Unset and `'all'` both mean every + * secret, resolved at execution so ones added later are included. + */ + secretScope?: 'all' | 'selected' + /** Secret names visible to the code when {@link secretScope} is `'selected'`. */ + mountedSecrets?: string[] envVars?: Record workflowVariables?: Record blockData?: Record diff --git a/apps/sim/tools/params.ts b/apps/sim/tools/params.ts index 1ffc78914a4..9ea7d81b1bf 100644 --- a/apps/sim/tools/params.ts +++ b/apps/sim/tools/params.ts @@ -6,6 +6,7 @@ import { type CanonicalModeOverrides, evaluateSubBlockCondition, isCanonicalPair, + isSubBlockFeatureEnabled, isSubBlockHidden, isTriggerModeSubBlock, resolveCanonicalMode, @@ -152,6 +153,14 @@ export interface UserToolSchemaOptions { export interface LLMToolSchemaResult { schema: ToolSchema enrichedDescription?: string + /** + * Params the model is never allowed to supply, because the tool declares them + * `user-only` or `hidden`. Omitting them from {@link schema} is not enough on + * its own — nothing stops a model from emitting an undeclared key, and the + * merge downstream seeds from the model's args — so the names travel with the + * schema for `prepareToolExecution` to strip. + */ + modelBlockedParams?: string[] } export interface ValidationResult { @@ -637,6 +646,13 @@ export async function createLLMToolSchema( required: [], } + // Derived from the declarations rather than from which branch below skipped a + // param: the loop's `continue`s also skip params the user simply filled in, + // and those are not off-limits to the model. + const modelBlockedParams = Object.entries(toolConfig.params) + .filter(([, param]) => param.visibility === 'user-only' || param.visibility === 'hidden') + .map(([paramId]) => paramId) + for (const [paramId, param] of Object.entries(toolConfig.params)) { const enrichmentConfig = toolConfig.schemaEnrichment?.[paramId] @@ -705,12 +721,13 @@ export async function createLLMToolSchema( return { schema: enriched.parameters as ToolSchema, enrichedDescription: enriched.description, + modelBlockedParams, } } } } - return { schema } + return { schema, modelBlockedParams } } /** @@ -1142,6 +1159,11 @@ export function getSubBlocksForToolInput( // Hide tool API key fields when running on hosted Sim or when env var is set if (isSubBlockHidden(sb)) continue + // A field the deployment has switched off is not offerable here either — + // the canvas already hides it, and offering it in tool-input lets an author + // pick a value the executor will refuse (e.g. Python with no sandbox provider). + if (!isSubBlockFeatureEnabled(sb)) continue + // Determine the effective param ID (canonical or subblock id) const effectiveParamId = sb.canonicalParamId || sb.id diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index 14df76348b2..d167b0cad4e 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.3.0 +version: 1.4.0 appVersion: "v0.7.44" kubeVersion: ">=1.25.0-0" home: https://sim.ai diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 5fb76fd9319..72f8cf52551 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -1374,6 +1374,18 @@ cronjobs: successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 1 + # Deletes prebuilt sandbox images that no workspace sandbox references and that + # have gone unused past the retention window, from the provider and locally. + # A no-op on deployments whose sandbox provider installs at run time. + cleanupSandboxImages: + enabled: true + name: cleanup-sandbox-images + schedule: "30 4 * * *" + path: "/api/cron/cleanup-sandbox-images" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + # Processes the transactional outbox (deployment side-effects retry / dead-letter) # and reaps stale workspace-fork background-work rows. Both are safety nets behind # the immediate post-commit processing, so frequent runs keep retries timely. diff --git a/packages/db/migrations/0277_workspace_sandboxes.sql b/packages/db/migrations/0277_workspace_sandboxes.sql new file mode 100644 index 00000000000..ef603ac2067 --- /dev/null +++ b/packages/db/migrations/0277_workspace_sandboxes.sql @@ -0,0 +1,39 @@ +CREATE TYPE "public"."sandbox_image_status" AS ENUM('pending', 'building', 'ready', 'failed');--> statement-breakpoint +CREATE TYPE "public"."sandbox_language" AS ENUM('javascript', 'python');--> statement-breakpoint +CREATE TABLE "sandbox_image" ( + "id" text PRIMARY KEY NOT NULL, + "provider" text NOT NULL, + "spec_hash" text NOT NULL, + "spec" jsonb NOT NULL, + "status" "sandbox_image_status" DEFAULT 'pending' NOT NULL, + "image_ref" text, + "provider_image_id" text, + "build_id" text, + "error_code" text, + "error_message" text, + "error_detail" text, + "last_used_at" timestamp, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "workspace_sandbox" ( + "id" text PRIMARY KEY NOT NULL, + "workspace_id" text NOT NULL, + "name" text NOT NULL, + "language" "sandbox_language" NOT NULL, + "dependencies" jsonb DEFAULT '[]'::jsonb NOT NULL, + "spec_hash" text NOT NULL, + "created_by" text, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "workspace_sandbox" ADD CONSTRAINT "workspace_sandbox_workspace_id_workspace_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspace"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "workspace_sandbox" ADD CONSTRAINT "workspace_sandbox_created_by_user_id_fk" FOREIGN KEY ("created_by") REFERENCES "public"."user"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "sandbox_image_provider_spec_unique" ON "sandbox_image" USING btree ("provider","spec_hash");--> statement-breakpoint +CREATE INDEX "sandbox_image_status_idx" ON "sandbox_image" USING btree ("status");--> statement-breakpoint +CREATE INDEX "sandbox_image_last_used_idx" ON "sandbox_image" USING btree ("last_used_at");--> statement-breakpoint +CREATE UNIQUE INDEX "workspace_sandbox_workspace_name_unique" ON "workspace_sandbox" USING btree ("workspace_id","name");--> statement-breakpoint +CREATE INDEX "workspace_sandbox_workspace_idx" ON "workspace_sandbox" USING btree ("workspace_id");--> statement-breakpoint +CREATE INDEX "workspace_sandbox_spec_hash_idx" ON "workspace_sandbox" USING btree ("spec_hash"); \ No newline at end of file diff --git a/packages/db/migrations/meta/0277_snapshot.json b/packages/db/migrations/meta/0277_snapshot.json new file mode 100644 index 00000000000..c6924f1fb91 --- /dev/null +++ b/packages/db/migrations/meta/0277_snapshot.json @@ -0,0 +1,18285 @@ +{ + "id": "3944d26f-f7b1-48e8-87a5-fd9bc577e10a", + "prevId": "100afbc7-1a7f-4f05-9950-b2dbd6996c15", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.academy_certificate": { + "name": "academy_certificate", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "course_id": { + "name": "course_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "academy_cert_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "issued_at": { + "name": "issued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "certificate_number": { + "name": "certificate_number", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "academy_certificate_user_id_idx": { + "name": "academy_certificate_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_course_id_idx": { + "name": "academy_certificate_course_id_idx", + "columns": [ + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_user_course_unique": { + "name": "academy_certificate_user_course_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "course_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_number_idx": { + "name": "academy_certificate_number_idx", + "columns": [ + { + "expression": "certificate_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "academy_certificate_status_idx": { + "name": "academy_certificate_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "academy_certificate_user_id_user_id_fk": { + "name": "academy_certificate_user_id_user_id_fk", + "tableFrom": "academy_certificate", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "academy_certificate_certificate_number_unique": { + "name": "academy_certificate_certificate_number_unique", + "nullsNotDistinct": false, + "columns": ["certificate_number"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_account_on_account_id_provider_id": { + "name": "idx_account_on_account_id_provider_id", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_key": { + "name": "api_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'personal'" + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_key_workspace_type_idx": { + "name": "api_key_workspace_type_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_user_type_idx": { + "name": "api_key_user_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_key_key_hash_idx": { + "name": "api_key_key_hash_idx", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_key_user_id_user_id_fk": { + "name": "api_key_user_id_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_workspace_id_workspace_id_fk": { + "name": "api_key_workspace_id_workspace_id_fk", + "tableFrom": "api_key", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_key_created_by_user_id_fk": { + "name": "api_key_created_by_user_id_fk", + "tableFrom": "api_key", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_key_key_unique": { + "name": "api_key_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": { + "workspace_type_check": { + "name": "workspace_type_check", + "value": "(type = 'workspace' AND workspace_id IS NOT NULL) OR (type = 'personal' AND workspace_id IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.async_jobs": { + "name": "async_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "run_at": { + "name": "run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "async_jobs_status_started_at_idx": { + "name": "async_jobs_status_started_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_status_completed_at_idx": { + "name": "async_jobs_status_completed_at_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "completed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_pending_run_at_idx": { + "name": "async_jobs_schedule_pending_run_at_idx", + "columns": [ + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "async_jobs_schedule_processing_started_at_idx": { + "name": "async_jobs_schedule_processing_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"async_jobs\".\"type\" = 'schedule-execution' AND \"async_jobs\".\"status\" = 'processing'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_name": { + "name": "actor_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_email": { + "name": "actor_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_name": { + "name": "resource_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_log_workspace_created_idx": { + "name": "audit_log_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_workspace_created_at_id_idx": { + "name": "audit_log_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_actor_created_idx": { + "name": "audit_log_actor_created_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_resource_idx": { + "name": "audit_log_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_action_idx": { + "name": "audit_log_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "audit_log_workspace_id_workspace_id_fk": { + "name": "audit_log_workspace_id_workspace_id_fk", + "tableFrom": "audit_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "audit_log_actor_id_user_id_fk": { + "name": "audit_log_actor_id_user_id_fk", + "tableFrom": "audit_log", + "tableTo": "user", + "columnsFrom": ["actor_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.background_work_status": { + "name": "background_work_status", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "background_work_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "background_work_status_value", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "background_work_status_workspace_status_idx": { + "name": "background_work_status_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_workflow_status_idx": { + "name": "background_work_status_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_child_ws_idx": { + "name": "background_work_status_meta_child_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'childWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "background_work_status_meta_other_ws_idx": { + "name": "background_work_status_meta_other_ws_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'otherWorkspaceId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "background_work_status_workspace_id_workspace_id_fk": { + "name": "background_work_status_workspace_id_workspace_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "background_work_status_workflow_id_workflow_id_fk": { + "name": "background_work_status_workflow_id_workflow_id_fk", + "tableFrom": "background_work_status", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat": { + "name": "chat", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "customizations": { + "name": "customizations", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "output_configs": { + "name": "output_configs", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "include_thinking": { + "name": "include_thinking", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "include_tool_calls": { + "name": "include_tool_calls", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "identifier_idx": { + "name": "identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"chat\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_archived_at_partial_idx": { + "name": "chat_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"chat\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_chat_on_workflow_id_archived_at": { + "name": "idx_chat_on_workflow_id_archived_at", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_workflow_id_workflow_id_fk": { + "name": "chat_workflow_id_workflow_id_fk", + "tableFrom": "chat", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_user_id_user_id_fk": { + "name": "chat_user_id_user_id_fk", + "tableFrom": "chat", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_async_tool_calls": { + "name": "copilot_async_tool_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "checkpoint_id": { + "name": "checkpoint_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "args": { + "name": "args", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "status": { + "name": "status", + "type": "copilot_async_tool_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "permission_decision": { + "name": "permission_decision", + "type": "copilot_tool_permission_decision", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "permission_decided_at": { + "name": "permission_decided_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_async_tool_calls_run_id_idx": { + "name": "copilot_async_tool_calls_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_checkpoint_id_idx": { + "name": "copilot_async_tool_calls_checkpoint_id_idx", + "columns": [ + { + "expression": "checkpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_idx": { + "name": "copilot_async_tool_calls_tool_call_id_idx", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_status_idx": { + "name": "copilot_async_tool_calls_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_run_status_idx": { + "name": "copilot_async_tool_calls_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_async_tool_calls_tool_call_id_unique": { + "name": "copilot_async_tool_calls_tool_call_id_unique", + "columns": [ + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_async_tool_calls_run_id_copilot_runs_id_fk": { + "name": "copilot_async_tool_calls_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk": { + "name": "copilot_async_tool_calls_checkpoint_id_copilot_run_checkpoints_id_fk", + "tableFrom": "copilot_async_tool_calls", + "tableTo": "copilot_run_checkpoints", + "columnsFrom": ["checkpoint_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_chats": { + "name": "copilot_chats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "chat_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'copilot'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'claude-3-7-sonnet-latest'" + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preview_yaml": { + "name": "preview_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan_artifact": { + "name": "plan_artifact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "auto_allowed_tools": { + "name": "auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "pinned": { + "name": "pinned", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_chats_user_id_idx": { + "name": "copilot_chats_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workflow_id_idx": { + "name": "copilot_chats_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workflow_idx": { + "name": "copilot_chats_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_idx": { + "name": "copilot_chats_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_created_at_idx": { + "name": "copilot_chats_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_updated_at_idx": { + "name": "copilot_chats_updated_at_idx", + "columns": [ + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_workspace_created_at_id_idx": { + "name": "copilot_chats_workspace_created_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"created_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_chats_user_workspace_deleted_partial_idx": { + "name": "copilot_chats_user_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_chats\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_chats_user_id_user_id_fk": { + "name": "copilot_chats_user_id_user_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workflow_id_workflow_id_fk": { + "name": "copilot_chats_workflow_id_workflow_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_chats_workspace_id_workspace_id_fk": { + "name": "copilot_chats_workspace_id_workspace_id_fk", + "tableFrom": "copilot_chats", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_feedback": { + "name": "copilot_feedback", + "schema": "", + "columns": { + "feedback_id": { + "name": "feedback_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_query": { + "name": "user_query", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_response": { + "name": "agent_response", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_positive": { + "name": "is_positive", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "feedback": { + "name": "feedback", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_yaml": { + "name": "workflow_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_feedback_user_id_idx": { + "name": "copilot_feedback_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_chat_id_idx": { + "name": "copilot_feedback_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_user_chat_idx": { + "name": "copilot_feedback_user_chat_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_is_positive_idx": { + "name": "copilot_feedback_is_positive_idx", + "columns": [ + { + "expression": "is_positive", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_feedback_created_at_idx": { + "name": "copilot_feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_feedback_user_id_user_id_fk": { + "name": "copilot_feedback_user_id_user_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_feedback_chat_id_copilot_chats_id_fk": { + "name": "copilot_feedback_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_feedback", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_messages": { + "name": "copilot_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_message_id": { + "name": "parent_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_messages_chat_message_unique": { + "name": "copilot_messages_chat_message_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_created_at_idx": { + "name": "copilot_messages_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_seq_idx": { + "name": "copilot_messages_chat_seq_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_chat_stream_idx": { + "name": "copilot_messages_chat_stream_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"stream_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_messages_user_created_at_idx": { + "name": "copilot_messages_user_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"copilot_messages\".\"role\" = 'user' AND \"copilot_messages\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_messages_chat_id_copilot_chats_id_fk": { + "name": "copilot_messages_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_messages", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_run_checkpoints": { + "name": "copilot_run_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "pending_tool_call_id": { + "name": "pending_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conversation_snapshot": { + "name": "conversation_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "agent_state": { + "name": "agent_state", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "provider_request": { + "name": "provider_request", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_run_checkpoints_run_id_idx": { + "name": "copilot_run_checkpoints_run_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_pending_tool_call_id_idx": { + "name": "copilot_run_checkpoints_pending_tool_call_id_idx", + "columns": [ + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_run_checkpoints_run_pending_tool_unique": { + "name": "copilot_run_checkpoints_run_pending_tool_unique", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pending_tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_run_checkpoints_run_id_copilot_runs_id_fk": { + "name": "copilot_run_checkpoints_run_id_copilot_runs_id_fk", + "tableFrom": "copilot_run_checkpoints", + "tableTo": "copilot_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_runs": { + "name": "copilot_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_run_id": { + "name": "parent_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stream_id": { + "name": "stream_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent": { + "name": "agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "copilot_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "request_context": { + "name": "request_context", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "copilot_runs_execution_id_idx": { + "name": "copilot_runs_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_parent_run_id_idx": { + "name": "copilot_runs_parent_run_id_idx", + "columns": [ + { + "expression": "parent_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_id_idx": { + "name": "copilot_runs_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_user_id_idx": { + "name": "copilot_runs_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workflow_id_idx": { + "name": "copilot_runs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_id_idx": { + "name": "copilot_runs_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_status_idx": { + "name": "copilot_runs_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_chat_execution_idx": { + "name": "copilot_runs_chat_execution_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_execution_started_at_idx": { + "name": "copilot_runs_execution_started_at_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_workspace_completed_at_id_idx": { + "name": "copilot_runs_workspace_completed_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"completed_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_runs_stream_id_unique": { + "name": "copilot_runs_stream_id_unique", + "columns": [ + { + "expression": "stream_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_runs_chat_id_copilot_chats_id_fk": { + "name": "copilot_runs_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_user_id_user_id_fk": { + "name": "copilot_runs_user_id_user_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workflow_id_workflow_id_fk": { + "name": "copilot_runs_workflow_id_workflow_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_runs_workspace_id_workspace_id_fk": { + "name": "copilot_runs_workspace_id_workspace_id_fk", + "tableFrom": "copilot_runs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.copilot_workflow_read_hashes": { + "name": "copilot_workflow_read_hashes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "copilot_workflow_read_hashes_chat_id_idx": { + "name": "copilot_workflow_read_hashes_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_workflow_id_idx": { + "name": "copilot_workflow_read_hashes_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "copilot_workflow_read_hashes_chat_workflow_unique": { + "name": "copilot_workflow_read_hashes_chat_workflow_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk": { + "name": "copilot_workflow_read_hashes_chat_id_copilot_chats_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "copilot_workflow_read_hashes_workflow_id_workflow_id_fk": { + "name": "copilot_workflow_read_hashes_workflow_id_workflow_id_fk", + "tableFrom": "copilot_workflow_read_hashes", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "credential_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_key": { + "name": "env_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "env_owner_user_id": { + "name": "env_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_service_account_key": { + "name": "encrypted_service_account_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_workspace_id_idx": { + "name": "credential_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_type_idx": { + "name": "credential_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_provider_id_idx": { + "name": "credential_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_account_id_idx": { + "name": "credential_account_id_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_env_owner_user_id_idx": { + "name": "credential_env_owner_user_id_idx", + "columns": [ + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_account_unique": { + "name": "credential_workspace_account_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "account_id IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_env_unique": { + "name": "credential_workspace_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_workspace'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_workspace_personal_env_unique": { + "name": "credential_workspace_personal_env_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "env_owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "type = 'env_personal'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_workspace_id_workspace_id_fk": { + "name": "credential_workspace_id_workspace_id_fk", + "tableFrom": "credential", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_account_id_account_id_fk": { + "name": "credential_account_id_account_id_fk", + "tableFrom": "credential", + "tableTo": "account", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_env_owner_user_id_user_id_fk": { + "name": "credential_env_owner_user_id_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["env_owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_created_by_user_id_fk": { + "name": "credential_created_by_user_id_fk", + "tableFrom": "credential", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "credential_oauth_source_check": { + "name": "credential_oauth_source_check", + "value": "(type <> 'oauth') OR (account_id IS NOT NULL AND provider_id IS NOT NULL)" + }, + "credential_workspace_env_source_check": { + "name": "credential_workspace_env_source_check", + "value": "(type <> 'env_workspace') OR (env_key IS NOT NULL AND env_owner_user_id IS NULL)" + }, + "credential_personal_env_source_check": { + "name": "credential_personal_env_source_check", + "value": "(type <> 'env_personal') OR (env_key IS NOT NULL AND env_owner_user_id IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.credential_member": { + "name": "credential_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "credential_member_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "credential_member_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "joined_at": { + "name": "joined_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credential_member_user_id_idx": { + "name": "credential_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_role_idx": { + "name": "credential_member_role_idx", + "columns": [ + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_status_idx": { + "name": "credential_member_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "credential_member_unique": { + "name": "credential_member_unique", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "credential_member_credential_id_credential_id_fk": { + "name": "credential_member_credential_id_credential_id_fk", + "tableFrom": "credential_member", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_user_id_user_id_fk": { + "name": "credential_member_user_id_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_member_invited_by_user_id_fk": { + "name": "credential_member_invited_by_user_id_fk", + "tableFrom": "credential_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_block": { + "name": "custom_block", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inputs": { + "name": "inputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "outputs": { + "name": "outputs", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_block_organization_id_idx": { + "name": "custom_block_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_workflow_id_idx": { + "name": "custom_block_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_block_organization_type_unique": { + "name": "custom_block_organization_type_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_block_organization_id_organization_id_fk": { + "name": "custom_block_organization_id_organization_id_fk", + "tableFrom": "custom_block", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_workflow_id_workflow_id_fk": { + "name": "custom_block_workflow_id_workflow_id_fk", + "tableFrom": "custom_block", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_block_created_by_user_id_fk": { + "name": "custom_block_created_by_user_id_fk", + "tableFrom": "custom_block", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custom_tools": { + "name": "custom_tools", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "custom_tools_workspace_id_idx": { + "name": "custom_tools_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "custom_tools_workspace_title_unique": { + "name": "custom_tools_workspace_title_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "title", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "custom_tools_workspace_id_workspace_id_fk": { + "name": "custom_tools_workspace_id_workspace_id_fk", + "tableFrom": "custom_tools", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "custom_tools_user_id_user_id_fk": { + "name": "custom_tools_user_id_user_id_fk", + "tableFrom": "custom_tools", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drain_runs": { + "name": "data_drain_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "drain_id": { + "name": "drain_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "data_drain_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "data_drain_run_trigger", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rows_exported": { + "name": "rows_exported", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bytes_written": { + "name": "bytes_written", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cursor_before": { + "name": "cursor_before", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cursor_after": { + "name": "cursor_after", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locators": { + "name": "locators", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": { + "data_drain_runs_drain_started_idx": { + "name": "data_drain_runs_drain_started_idx", + "columns": [ + { + "expression": "drain_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drain_runs_drain_id_data_drains_id_fk": { + "name": "data_drain_runs_drain_id_data_drains_id_fk", + "tableFrom": "data_drain_runs", + "tableTo": "data_drains", + "columnsFrom": ["drain_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_drains": { + "name": "data_drains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "data_drain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_type": { + "name": "destination_type", + "type": "data_drain_destination", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "destination_config": { + "name": "destination_config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "destination_credentials": { + "name": "destination_credentials", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schedule_cadence": { + "name": "schedule_cadence", + "type": "data_drain_cadence", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_drains_org_idx": { + "name": "data_drains_org_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_due_idx": { + "name": "data_drains_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_drains_org_name_unique": { + "name": "data_drains_org_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_drains_organization_id_organization_id_fk": { + "name": "data_drains_organization_id_organization_id_fk", + "tableFrom": "data_drains", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_drains_created_by_user_id_fk": { + "name": "data_drains_created_by_user_id_fk", + "tableFrom": "data_drains", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.docs_embeddings": { + "name": "docs_embeddings", + "schema": "", + "columns": { + "chunk_id": { + "name": "chunk_id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "chunk_text": { + "name": "chunk_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_document": { + "name": "source_document", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_link": { + "name": "source_link", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_text": { + "name": "header_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "header_level": { + "name": "header_level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": true + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "chunk_text_tsv": { + "name": "chunk_text_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"docs_embeddings\".\"chunk_text\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "docs_emb_source_document_idx": { + "name": "docs_emb_source_document_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_header_level_idx": { + "name": "docs_emb_header_level_idx", + "columns": [ + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_source_header_idx": { + "name": "docs_emb_source_header_idx", + "columns": [ + { + "expression": "source_document", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "header_level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_model_idx": { + "name": "docs_emb_model_idx", + "columns": [ + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_emb_created_at_idx": { + "name": "docs_emb_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "docs_embedding_vector_hnsw_idx": { + "name": "docs_embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "docs_emb_metadata_gin_idx": { + "name": "docs_emb_metadata_gin_idx", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "docs_emb_chunk_text_fts_idx": { + "name": "docs_emb_chunk_text_fts_idx", + "columns": [ + { + "expression": "chunk_text_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "docs_embedding_not_null_check": { + "name": "docs_embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + }, + "docs_header_level_check": { + "name": "docs_header_level_check", + "value": "\"header_level\" >= 1 AND \"header_level\" <= 6" + } + }, + "isRLSEnabled": false + }, + "public.document": { + "name": "document", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_url": { + "name": "file_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "processing_status": { + "name": "processing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_completed_at": { + "name": "processing_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_excluded": { + "name": "user_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "doc_kb_id_idx": { + "name": "doc_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_filename_idx": { + "name": "doc_filename_idx", + "columns": [ + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_processing_status_idx": { + "name": "doc_processing_status_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "processing_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_external_id_idx": { + "name": "doc_connector_external_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"document\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_connector_id_idx": { + "name": "doc_connector_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_storage_key_idx": { + "name": "doc_storage_key_idx", + "columns": [ + { + "expression": "storage_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"storage_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_archived_at_partial_idx": { + "name": "doc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_deleted_at_partial_idx": { + "name": "doc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"document\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag1_idx": { + "name": "doc_tag1_idx", + "columns": [ + { + "expression": "tag1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag2_idx": { + "name": "doc_tag2_idx", + "columns": [ + { + "expression": "tag2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag3_idx": { + "name": "doc_tag3_idx", + "columns": [ + { + "expression": "tag3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag4_idx": { + "name": "doc_tag4_idx", + "columns": [ + { + "expression": "tag4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag5_idx": { + "name": "doc_tag5_idx", + "columns": [ + { + "expression": "tag5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag6_idx": { + "name": "doc_tag6_idx", + "columns": [ + { + "expression": "tag6", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_tag7_idx": { + "name": "doc_tag7_idx", + "columns": [ + { + "expression": "tag7", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number1_idx": { + "name": "doc_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number2_idx": { + "name": "doc_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number3_idx": { + "name": "doc_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number4_idx": { + "name": "doc_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_number5_idx": { + "name": "doc_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date1_idx": { + "name": "doc_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_date2_idx": { + "name": "doc_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean1_idx": { + "name": "doc_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean2_idx": { + "name": "doc_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "doc_boolean3_idx": { + "name": "doc_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_knowledge_base_id_knowledge_base_id_fk": { + "name": "document_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "document_connector_id_knowledge_connector_id_fk": { + "name": "document_connector_id_knowledge_connector_id_fk", + "tableFrom": "document", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "document_uploaded_by_user_id_fk": { + "name": "document_uploaded_by_user_id_fk", + "tableFrom": "document", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.embedding": { + "name": "embedding", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chunk_index": { + "name": "chunk_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "chunk_hash": { + "name": "chunk_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_length": { + "name": "content_length", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "vector(1536)", + "primaryKey": false, + "notNull": false + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "start_offset": { + "name": "start_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "end_offset": { + "name": "end_offset", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tag1": { + "name": "tag1", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag2": { + "name": "tag2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag3": { + "name": "tag3", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag4": { + "name": "tag4", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag5": { + "name": "tag5", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag6": { + "name": "tag6", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag7": { + "name": "tag7", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "number1": { + "name": "number1", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number2": { + "name": "number2", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number3": { + "name": "number3", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number4": { + "name": "number4", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "number5": { + "name": "number5", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "date1": { + "name": "date1", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "date2": { + "name": "date2", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "boolean1": { + "name": "boolean1", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean2": { + "name": "boolean2", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "boolean3": { + "name": "boolean3", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "content_tsv": { + "name": "content_tsv", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "to_tsvector('english', \"embedding\".\"content\")", + "type": "stored" + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "emb_kb_id_idx": { + "name": "emb_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_id_idx": { + "name": "emb_doc_id_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_chunk_idx": { + "name": "emb_doc_chunk_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chunk_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_model_idx": { + "name": "emb_kb_model_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_kb_enabled_idx": { + "name": "emb_kb_enabled_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_doc_enabled_idx": { + "name": "emb_doc_enabled_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "embedding_vector_hnsw_idx": { + "name": "embedding_vector_hnsw_idx", + "columns": [ + { + "expression": "embedding", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "vector_cosine_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hnsw", + "with": { + "m": 16, + "ef_construction": 64 + } + }, + "emb_tag1_idx": { + "name": "emb_tag1_idx", + "columns": [ + { + "expression": "tag1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag2_idx": { + "name": "emb_tag2_idx", + "columns": [ + { + "expression": "tag2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag3_idx": { + "name": "emb_tag3_idx", + "columns": [ + { + "expression": "tag3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag4_idx": { + "name": "emb_tag4_idx", + "columns": [ + { + "expression": "tag4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag5_idx": { + "name": "emb_tag5_idx", + "columns": [ + { + "expression": "tag5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag6_idx": { + "name": "emb_tag6_idx", + "columns": [ + { + "expression": "tag6", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_tag7_idx": { + "name": "emb_tag7_idx", + "columns": [ + { + "expression": "tag7", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number1_idx": { + "name": "emb_number1_idx", + "columns": [ + { + "expression": "number1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number2_idx": { + "name": "emb_number2_idx", + "columns": [ + { + "expression": "number2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number3_idx": { + "name": "emb_number3_idx", + "columns": [ + { + "expression": "number3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number4_idx": { + "name": "emb_number4_idx", + "columns": [ + { + "expression": "number4", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_number5_idx": { + "name": "emb_number5_idx", + "columns": [ + { + "expression": "number5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date1_idx": { + "name": "emb_date1_idx", + "columns": [ + { + "expression": "date1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_date2_idx": { + "name": "emb_date2_idx", + "columns": [ + { + "expression": "date2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean1_idx": { + "name": "emb_boolean1_idx", + "columns": [ + { + "expression": "boolean1", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean2_idx": { + "name": "emb_boolean2_idx", + "columns": [ + { + "expression": "boolean2", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_boolean3_idx": { + "name": "emb_boolean3_idx", + "columns": [ + { + "expression": "boolean3", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "emb_content_fts_idx": { + "name": "emb_content_fts_idx", + "columns": [ + { + "expression": "content_tsv", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "embedding_knowledge_base_id_knowledge_base_id_fk": { + "name": "embedding_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "embedding", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embedding_document_id_document_id_fk": { + "name": "embedding_document_id_document_id_fk", + "tableFrom": "embedding", + "tableTo": "document", + "columnsFrom": ["document_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "embedding_not_null_check": { + "name": "embedding_not_null_check", + "value": "\"embedding\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.environment": { + "name": "environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "environment_user_id_user_id_fk": { + "name": "environment_user_id_user_id_fk", + "tableFrom": "environment", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "environment_user_id_unique": { + "name": "environment_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_dependencies": { + "name": "execution_large_value_dependencies", + "schema": "", + "columns": { + "parent_key": { + "name": "parent_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_key": { + "name": "child_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_dependencies_workspace_parent_key_idx": { + "name": "execution_large_value_dependencies_workspace_parent_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_value_dependencies_workspace_child_key_idx": { + "name": "execution_large_value_dependencies_workspace_child_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_dependencies_workspace_id_workspace_id_fk": { + "name": "execution_large_value_dependencies_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_dependencies", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_dependencies_parent_key_child_key_pk": { + "name": "execution_large_value_dependencies_parent_key_child_key_pk", + "columns": ["parent_key", "child_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_value_references": { + "name": "execution_large_value_references", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "execution_large_value_reference_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_large_value_references_workspace_execution_source_idx": { + "name": "execution_large_value_references_workspace_execution_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_value_references_workspace_id_workspace_id_fk": { + "name": "execution_large_value_references_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_value_references_workflow_id_workflow_id_fk": { + "name": "execution_large_value_references_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_value_references", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_large_value_references_key_execution_id_source_pk": { + "name": "execution_large_value_references_key_execution_id_source_pk", + "columns": ["key", "execution_id", "source"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_large_values": { + "name": "execution_large_values", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_execution_id": { + "name": "owner_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "execution_large_values_owner_execution_id_idx": { + "name": "execution_large_values_owner_execution_id_idx", + "columns": [ + { + "expression": "owner_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_cleanup_idx": { + "name": "execution_large_values_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_large_values_tombstone_cleanup_idx": { + "name": "execution_large_values_tombstone_cleanup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_large_values\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_large_values_workspace_id_workspace_id_fk": { + "name": "execution_large_values_workspace_id_workspace_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_large_values_workflow_id_workflow_id_fk": { + "name": "execution_large_values_workflow_id_workflow_id_fk", + "tableFrom": "execution_large_values", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.folder": { + "name": "folder", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "folder_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "folder_user_idx": { + "name": "folder_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_idx": { + "name": "folder_workspace_resource_parent_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_parent_sort_idx": { + "name": "folder_parent_sort_idx", + "columns": [ + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_deleted_at_idx": { + "name": "folder_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_deleted_partial_idx": { + "name": "folder_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"folder\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "folder_workspace_resource_parent_name_active_unique": { + "name": "folder_workspace_resource_parent_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"parent_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"folder\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "folder_user_id_user_id_fk": { + "name": "folder_user_id_user_id_fk", + "tableFrom": "folder", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_workspace_id_workspace_id_fk": { + "name": "folder_workspace_id_workspace_id_fk", + "tableFrom": "folder", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "folder_parent_id_folder_id_fk": { + "name": "folder_parent_id_folder_id_fk", + "tableFrom": "folder", + "tableTo": "folder", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.idempotency_key": { + "name": "idempotency_key", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idempotency_key_created_at_idx": { + "name": "idempotency_key_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "invitation_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'organization'" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "membership_intent": { + "name": "membership_intent", + "type": "invitation_membership_intent", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'internal'" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "invitation_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_status_idx": { + "name": "invitation_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_pending_email_org_unique": { + "name": "invitation_pending_email_org_unique", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"invitation\".\"status\" = 'pending' AND \"invitation\".\"organization_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invitation_token_unique": { + "name": "invitation_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation_workspace_grant": { + "name": "invitation_workspace_grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "invitation_id": { + "name": "invitation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission": { + "name": "permission", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "invitation_workspace_grant_unique": { + "name": "invitation_workspace_grant_unique", + "columns": [ + { + "expression": "invitation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_workspace_grant_workspace_id_idx": { + "name": "invitation_workspace_grant_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_workspace_grant_invitation_id_invitation_id_fk": { + "name": "invitation_workspace_grant_invitation_id_invitation_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "invitation", + "columnsFrom": ["invitation_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_workspace_grant_workspace_id_workspace_id_fk": { + "name": "invitation_workspace_grant_workspace_id_workspace_id_fk", + "tableFrom": "invitation_workspace_grant", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.job_execution_logs": { + "name": "job_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schedule_id": { + "name": "schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "job_execution_logs_schedule_id_idx": { + "name": "job_execution_logs_schedule_id_idx", + "columns": [ + { + "expression": "schedule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_started_at_idx": { + "name": "job_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_workspace_ended_at_id_idx": { + "name": "job_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_execution_id_unique": { + "name": "job_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "job_execution_logs_trigger_idx": { + "name": "job_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_execution_logs_schedule_id_workflow_schedule_id_fk": { + "name": "job_execution_logs_schedule_id_workflow_schedule_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workflow_schedule", + "columnsFrom": ["schedule_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "job_execution_logs_workspace_id_workspace_id_fk": { + "name": "job_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "job_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base": { + "name": "knowledge_base", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text-embedding-3-small'" + }, + "embedding_dimension": { + "name": "embedding_dimension", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1536 + }, + "chunking_config": { + "name": "chunking_config", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{\"maxSize\": 1024, \"minSize\": 1, \"overlap\": 200}'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_user_id_idx": { + "name": "kb_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_id_idx": { + "name": "kb_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_user_workspace_idx": { + "name": "kb_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_folder_id_idx": { + "name": "kb_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_deleted_at_idx": { + "name": "kb_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_deleted_partial_idx": { + "name": "kb_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_base\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_workspace_name_active_unique": { + "name": "kb_workspace_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"knowledge_base\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_user_id_user_id_fk": { + "name": "knowledge_base_user_id_user_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_workspace_id_workspace_id_fk": { + "name": "knowledge_base_workspace_id_workspace_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_base_folder_id_folder_id_fk": { + "name": "knowledge_base_folder_id_folder_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_base_tag_definitions": { + "name": "knowledge_base_tag_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_slot": { + "name": "tag_slot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "field_type": { + "name": "field_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'text'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "kb_tag_definitions_kb_slot_idx": { + "name": "kb_tag_definitions_kb_slot_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_slot", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_display_name_idx": { + "name": "kb_tag_definitions_kb_display_name_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kb_tag_definitions_kb_id_idx": { + "name": "kb_tag_definitions_kb_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_base_tag_definitions_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_base_tag_definitions", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector": { + "name": "knowledge_connector", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connector_type": { + "name": "connector_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_config": { + "name": "source_config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "sync_mode": { + "name": "sync_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'full'" + }, + "sync_interval_minutes": { + "name": "sync_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1440 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_sync_doc_count": { + "name": "last_sync_doc_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "next_sync_at": { + "name": "next_sync_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kc_knowledge_base_id_idx": { + "name": "kc_knowledge_base_id_idx", + "columns": [ + { + "expression": "knowledge_base_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_status_next_sync_idx": { + "name": "kc_status_next_sync_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_sync_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_archived_at_partial_idx": { + "name": "kc_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "kc_deleted_at_partial_idx": { + "name": "kc_deleted_at_partial_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"knowledge_connector\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_knowledge_base_id_knowledge_base_id_fk": { + "name": "knowledge_connector_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_connector", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.knowledge_connector_sync_log": { + "name": "knowledge_connector_sync_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connector_id": { + "name": "connector_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "docs_added": { + "name": "docs_added", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_updated": { + "name": "docs_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_deleted": { + "name": "docs_deleted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_unchanged": { + "name": "docs_unchanged", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "docs_failed": { + "name": "docs_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "kcsl_connector_id_idx": { + "name": "kcsl_connector_id_idx", + "columns": [ + { + "expression": "connector_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk": { + "name": "knowledge_connector_sync_log_connector_id_knowledge_connector_id_fk", + "tableFrom": "knowledge_connector_sync_log", + "tableTo": "knowledge_connector", + "columnsFrom": ["connector_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_server_oauth": { + "name": "mcp_server_oauth", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_information": { + "name": "client_information", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_created_at": { + "name": "state_created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_server_oauth_server_unique": { + "name": "mcp_server_oauth_server_unique", + "columns": [ + { + "expression": "mcp_server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_server_oauth_state_idx": { + "name": "mcp_server_oauth_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk": { + "name": "mcp_server_oauth_mcp_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "mcp_servers", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_server_oauth_user_id_user_id_fk": { + "name": "mcp_server_oauth_user_id_user_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "mcp_server_oauth_workspace_id_workspace_id_fk": { + "name": "mcp_server_oauth_workspace_id_workspace_id_fk", + "tableFrom": "mcp_server_oauth", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transport": { + "name": "transport", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'headers'" + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_secret": { + "name": "oauth_client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "headers": { + "name": "headers", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "timeout": { + "name": "timeout", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30000 + }, + "retries": { + "name": "retries", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 3 + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_connected": { + "name": "last_connected", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "connection_status": { + "name": "connection_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'disconnected'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status_config": { + "name": "status_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_tools_refresh": { + "name": "last_tools_refresh", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_requests": { + "name": "total_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_used": { + "name": "last_used", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_servers_workspace_enabled_idx": { + "name": "mcp_servers_workspace_enabled_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_servers_workspace_deleted_partial_idx": { + "name": "mcp_servers_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"mcp_servers\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_servers_workspace_id_workspace_id_fk": { + "name": "mcp_servers_workspace_id_workspace_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_servers_created_by_user_id_fk": { + "name": "mcp_servers_created_by_user_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_user_id_unique": { + "name": "member_user_id_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory": { + "name": "memory", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "memory_key_idx": { + "name": "memory_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_idx": { + "name": "memory_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_key_idx": { + "name": "memory_workspace_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memory_workspace_deleted_partial_idx": { + "name": "memory_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"memory\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_workspace_id_workspace_id_fk": { + "name": "memory_workspace_id_workspace_id_fk", + "tableFrom": "memory", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_allowed_sender": { + "name": "mothership_inbox_allowed_sender", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_sender_ws_email_idx": { + "name": "inbox_sender_ws_email_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_allowed_sender_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_allowed_sender_added_by_user_id_fk": { + "name": "mothership_inbox_allowed_sender_added_by_user_id_fk", + "tableFrom": "mothership_inbox_allowed_sender", + "tableTo": "user", + "columnsFrom": ["added_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_task": { + "name": "mothership_inbox_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_email": { + "name": "from_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_name": { + "name": "from_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body_preview": { + "name": "body_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_text": { + "name": "body_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_html": { + "name": "body_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_message_id": { + "name": "email_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "in_reply_to": { + "name": "in_reply_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_message_id": { + "name": "response_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agentmail_message_id": { + "name": "agentmail_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "trigger_job_id": { + "name": "trigger_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result_summary": { + "name": "result_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rejection_reason": { + "name": "rejection_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cc_recipients": { + "name": "cc_recipients", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inbox_task_ws_created_at_idx": { + "name": "inbox_task_ws_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_ws_status_idx": { + "name": "inbox_task_ws_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_response_msg_id_idx": { + "name": "inbox_task_response_msg_id_idx", + "columns": [ + { + "expression": "response_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_email_msg_id_idx": { + "name": "inbox_task_email_msg_id_idx", + "columns": [ + { + "expression": "email_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_inbox_task_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_task_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mothership_inbox_task_chat_id_copilot_chats_id_fk": { + "name": "mothership_inbox_task_chat_id_copilot_chats_id_fk", + "tableFrom": "mothership_inbox_task", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_inbox_webhook": { + "name": "mothership_inbox_webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mothership_inbox_webhook_workspace_id_workspace_id_fk": { + "name": "mothership_inbox_webhook_workspace_id_workspace_id_fk", + "tableFrom": "mothership_inbox_webhook", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mothership_inbox_webhook_workspace_id_unique": { + "name": "mothership_inbox_webhook_workspace_id_unique", + "nullsNotDistinct": false, + "columns": ["workspace_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mothership_settings": { + "name": "mothership_settings", + "schema": "", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mcp_tool_refs": { + "name": "mcp_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "custom_tool_refs": { + "name": "custom_tool_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "skill_refs": { + "name": "skill_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mothership_settings_workspace_id_idx": { + "name": "mothership_settings_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mothership_settings_workspace_id_workspace_id_fk": { + "name": "mothership_settings_workspace_id_workspace_id_fk", + "tableFrom": "mothership_settings", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "session_policy_settings": { + "name": "session_policy_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "security_policy_version": { + "name": "security_policy_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "whitelabel_settings": { + "name": "whitelabel_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "data_retention_settings": { + "name": "data_retention_settings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "org_usage_limit": { + "name": "org_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "departed_member_usage": { + "name": "departed_member_usage", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_member_usage_limit": { + "name": "organization_member_usage_limit", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "usage_limit": { + "name": "usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "set_by": { + "name": "set_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_member_usage_limit_org_user_unique": { + "name": "org_member_usage_limit_org_user_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_member_usage_limit_organization_id_idx": { + "name": "org_member_usage_limit_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_member_usage_limit_organization_id_organization_id_fk": { + "name": "organization_member_usage_limit_organization_id_organization_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_user_id_user_id_fk": { + "name": "organization_member_usage_limit_user_id_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_member_usage_limit_set_by_user_id_fk": { + "name": "organization_member_usage_limit_set_by_user_id_fk", + "tableFrom": "organization_member_usage_limit", + "tableTo": "user", + "columnsFrom": ["set_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outbox_event": { + "name": "outbox_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10 + }, + "available_at": { + "name": "available_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "outbox_event_status_available_idx": { + "name": "outbox_event_status_available_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_locked_at_idx": { + "name": "outbox_event_locked_at_idx", + "columns": [ + { + "expression": "locked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "outbox_event_type_created_idx": { + "name": "outbox_event_type_created_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.paused_executions": { + "name": "paused_executions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_snapshot": { + "name": "execution_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "pause_points": { + "name": "pause_points", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "total_pause_count": { + "name": "total_pause_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "resumed_count": { + "name": "resumed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "automatic_resume_retry_count": { + "name": "automatic_resume_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paused'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "paused_at": { + "name": "paused_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "next_resume_at": { + "name": "next_resume_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "paused_executions_workflow_id_idx": { + "name": "paused_executions_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_status_idx": { + "name": "paused_executions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_execution_id_unique": { + "name": "paused_executions_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "paused_executions_next_resume_at_idx": { + "name": "paused_executions_next_resume_at_idx", + "columns": [ + { + "expression": "next_resume_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'paused' AND next_resume_at IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "paused_executions_workflow_id_workflow_id_fk": { + "name": "paused_executions_workflow_id_workflow_id_fk", + "tableFrom": "paused_executions", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pending_credential_draft": { + "name": "pending_credential_draft", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pending_draft_user_provider_ws": { + "name": "pending_draft_user_provider_ws", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pending_credential_draft_user_id_user_id_fk": { + "name": "pending_credential_draft_user_id_user_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_workspace_id_workspace_id_fk": { + "name": "pending_credential_draft_workspace_id_workspace_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pending_credential_draft_credential_id_credential_id_fk": { + "name": "pending_credential_draft_credential_id_credential_id_fk", + "tableFrom": "pending_credential_draft", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group": { + "name": "permission_group", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "permission_group_created_by_idx": { + "name": "permission_group_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_name_unique": { + "name": "permission_group_organization_name_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_organization_default_unique": { + "name": "permission_group_organization_default_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_organization_id_organization_id_fk": { + "name": "permission_group_organization_id_organization_id_fk", + "tableFrom": "permission_group", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_created_by_user_id_fk": { + "name": "permission_group_created_by_user_id_fk", + "tableFrom": "permission_group", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_member": { + "name": "permission_group_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "assigned_by": { + "name": "assigned_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_member_group_id_idx": { + "name": "permission_group_member_group_id_idx", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_group_user_unique": { + "name": "permission_group_member_group_user_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_member_organization_user_idx": { + "name": "permission_group_member_organization_user_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_member_permission_group_id_permission_group_id_fk": { + "name": "permission_group_member_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_organization_id_organization_id_fk": { + "name": "permission_group_member_organization_id_organization_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_user_id_user_id_fk": { + "name": "permission_group_member_user_id_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_member_assigned_by_user_id_fk": { + "name": "permission_group_member_assigned_by_user_id_fk", + "tableFrom": "permission_group_member", + "tableTo": "user", + "columnsFrom": ["assigned_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permission_group_workspace": { + "name": "permission_group_workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "permission_group_id": { + "name": "permission_group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permission_group_workspace_workspace_id_idx": { + "name": "permission_group_workspace_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permission_group_workspace_group_workspace_unique": { + "name": "permission_group_workspace_group_workspace_unique", + "columns": [ + { + "expression": "permission_group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permission_group_workspace_permission_group_id_permission_group_id_fk": { + "name": "permission_group_workspace_permission_group_id_permission_group_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "permission_group", + "columnsFrom": ["permission_group_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_workspace_id_workspace_id_fk": { + "name": "permission_group_workspace_workspace_id_workspace_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "permission_group_workspace_organization_id_organization_id_fk": { + "name": "permission_group_workspace_organization_id_organization_id_fk", + "tableFrom": "permission_group_workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.permissions": { + "name": "permissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "permission_type": { + "name": "permission_type", + "type": "permission_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "permissions_user_id_idx": { + "name": "permissions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_entity_idx": { + "name": "permissions_entity_idx", + "columns": [ + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_type_idx": { + "name": "permissions_user_entity_type_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_permission_idx": { + "name": "permissions_user_entity_permission_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "permission_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_user_entity_idx": { + "name": "permissions_user_entity_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "permissions_unique_constraint": { + "name": "permissions_unique_constraint", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "permissions_user_id_user_id_fk": { + "name": "permissions_user_id_user_id_fk", + "tableFrom": "permissions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pinned_item": { + "name": "pinned_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pinned_item_user_workspace_idx": { + "name": "pinned_item_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_resource_idx": { + "name": "pinned_item_resource_idx", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pinned_item_user_resource_unique": { + "name": "pinned_item_user_resource_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pinned_item_user_id_user_id_fk": { + "name": "pinned_item_user_id_user_id_fk", + "tableFrom": "pinned_item", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pinned_item_workspace_id_workspace_id_fk": { + "name": "pinned_item_workspace_id_workspace_id_fk", + "tableFrom": "pinned_item", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.public_share": { + "name": "public_share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public'" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_emails": { + "name": "allowed_emails", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "public_share_token_unique": { + "name": "public_share_token_unique", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_unique": { + "name": "public_share_resource_unique", + "columns": [ + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_resource_id_idx": { + "name": "public_share_resource_id_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "public_share_workspace_id_idx": { + "name": "public_share_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "public_share_workspace_id_workspace_id_fk": { + "name": "public_share_workspace_id_workspace_id_fk", + "tableFrom": "public_share", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "public_share_created_by_user_id_fk": { + "name": "public_share_created_by_user_id_fk", + "tableFrom": "public_share", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_bucket": { + "name": "rate_limit_bucket", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tokens": { + "name": "tokens", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.resume_queue": { + "name": "resume_queue", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "paused_execution_id": { + "name": "paused_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_execution_id": { + "name": "parent_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "new_execution_id": { + "name": "new_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "context_id": { + "name": "context_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resume_input": { + "name": "resume_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "resume_queue_parent_status_idx": { + "name": "resume_queue_parent_status_idx", + "columns": [ + { + "expression": "parent_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resume_queue_new_execution_idx": { + "name": "resume_queue_new_execution_idx", + "columns": [ + { + "expression": "new_execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "resume_queue_paused_execution_id_paused_executions_id_fk": { + "name": "resume_queue_paused_execution_id_paused_executions_id_fk", + "tableFrom": "resume_queue", + "tableTo": "paused_executions", + "columnsFrom": ["paused_execution_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandbox_image": { + "name": "sandbox_image", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "spec_hash": { + "name": "spec_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "spec": { + "name": "spec", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "sandbox_image_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_image_id": { + "name": "provider_image_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "build_id": { + "name": "build_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_detail": { + "name": "error_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sandbox_image_provider_spec_unique": { + "name": "sandbox_image_provider_spec_unique", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "spec_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_image_status_idx": { + "name": "sandbox_image_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_image_last_used_idx": { + "name": "sandbox_image_last_used_idx", + "columns": [ + { + "expression": "last_used_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_token_idx": { + "name": "session_token_idx", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_active_organization_id_organization_id_fk": { + "name": "session_active_organization_id_organization_id_fk", + "tableFrom": "session", + "tableTo": "organization", + "columnsFrom": ["active_organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "theme": { + "name": "theme", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'system'" + }, + "auto_connect": { + "name": "auto_connect", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "telemetry_enabled": { + "name": "telemetry_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "email_preferences": { + "name": "email_preferences", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "billing_usage_notifications_enabled": { + "name": "billing_usage_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "show_training_controls": { + "name": "show_training_controls", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "super_user_mode_enabled": { + "name": "super_user_mode_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "mothership_environment": { + "name": "mothership_environment", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "error_notifications_enabled": { + "name": "error_notifications_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "snap_to_grid_size": { + "name": "snap_to_grid_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "show_action_bar": { + "name": "show_action_bar", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "copilot_enabled_models": { + "name": "copilot_enabled_models", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "copilot_auto_allowed_tools": { + "name": "copilot_auto_allowed_tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_active_workspace_id": { + "name": "last_active_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "settings_user_id_user_id_fk": { + "name": "settings_user_id_user_id_fk", + "tableFrom": "settings", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_user_id_unique": { + "name": "settings_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sim_trigger_state": { + "name": "sim_trigger_state", + "schema": "", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "last_fired_at": { + "name": "last_fired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sim_trigger_state_workflow_id_workflow_id_fk": { + "name": "sim_trigger_state_workflow_id_workflow_id_fk", + "tableFrom": "sim_trigger_state", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "sim_trigger_state_workflow_id_block_id_scope_key_pk": { + "name": "sim_trigger_state_workflow_id_block_id_scope_key_pk", + "columns": ["workflow_id", "block_id", "scope_key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill": { + "name": "skill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_workspace_name_unique": { + "name": "skill_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_workspace_id_workspace_id_fk": { + "name": "skill_workspace_id_workspace_id_fk", + "tableFrom": "skill", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_user_id_user_id_fk": { + "name": "skill_user_id_user_id_fk", + "tableFrom": "skill", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_member": { + "name": "skill_member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_member_user_id_idx": { + "name": "skill_member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skill_member_unique": { + "name": "skill_member_unique", + "columns": [ + { + "expression": "skill_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_member_skill_id_skill_id_fk": { + "name": "skill_member_skill_id_skill_id_fk", + "tableFrom": "skill_member", + "tableTo": "skill", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_user_id_user_id_fk": { + "name": "skill_member_user_id_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "skill_member_invited_by_user_id_fk": { + "name": "skill_member_invited_by_user_id_fk", + "tableFrom": "skill_member", + "tableTo": "user", + "columnsFrom": ["invited_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_domain": { + "name": "sso_domain", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "verification_token": { + "name": "verification_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified_at": { + "name": "verified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sso_domain_organization_id_idx": { + "name": "sso_domain_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_domain_idx": { + "name": "sso_domain_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_org_domain_unique": { + "name": "sso_domain_org_domain_unique", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_domain_verified_unique": { + "name": "sso_domain_verified_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "status = 'verified'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_domain_organization_id_organization_id_fk": { + "name": "sso_domain_organization_id_organization_id_fk", + "tableFrom": "sso_domain", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_domain_created_by_user_id_fk": { + "name": "sso_domain_created_by_user_id_fk", + "tableFrom": "sso_domain", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_provider": { + "name": "sso_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sso_provider_provider_id_idx": { + "name": "sso_provider_provider_id_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_domain_idx": { + "name": "sso_provider_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_user_id_idx": { + "name": "sso_provider_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sso_provider_organization_id_idx": { + "name": "sso_provider_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sso_provider_user_id_user_id_fk": { + "name": "sso_provider_user_id_user_id_fk", + "tableFrom": "sso_provider", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sso_provider_organization_id_organization_id_fk": { + "name": "sso_provider_organization_id_organization_id_fk", + "tableFrom": "sso_provider", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subscription": { + "name": "subscription", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_subscription_id": { + "name": "stripe_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "period_start": { + "name": "period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "period_end": { + "name": "period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancel_at_period_end": { + "name": "cancel_at_period_end", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cancel_at": { + "name": "cancel_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "canceled_at": { + "name": "canceled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "seats": { + "name": "seats", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "trial_start": { + "name": "trial_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trial_end": { + "name": "trial_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_interval": { + "name": "billing_interval", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_schedule_id": { + "name": "stripe_schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "subscription_reference_status_idx": { + "name": "subscription_reference_status_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "check_enterprise_metadata": { + "name": "check_enterprise_metadata", + "value": "plan != 'enterprise' OR metadata IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.table_jobs": { + "name": "table_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rows_processed": { + "name": "rows_processed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_jobs_one_active_per_table": { + "name": "table_jobs_one_active_per_table", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"table_jobs\".\"status\" = 'running' AND \"table_jobs\".\"type\" <> 'export'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_watchdog_idx": { + "name": "table_jobs_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_jobs_table_started_idx": { + "name": "table_jobs_table_started_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_jobs_table_id_user_table_definitions_id_fk": { + "name": "table_jobs_table_id_user_table_definitions_id_fk", + "tableFrom": "table_jobs", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_jobs_workspace_id_workspace_id_fk": { + "name": "table_jobs_workspace_id_workspace_id_fk", + "tableFrom": "table_jobs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_row_executions": { + "name": "table_row_executions", + "schema": "", + "columns": { + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "running_block_ids": { + "name": "running_block_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "block_errors": { + "name": "block_errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "enrichment_details": { + "name": "enrichment_details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_row_executions_table_status_idx": { + "name": "table_row_executions_table_status_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"status\" IN ('queued', 'running', 'pending')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_execution_id_idx": { + "name": "table_row_executions_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"table_row_executions\".\"execution_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_row_executions_table_group_idx": { + "name": "table_row_executions_table_group_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_row_executions_table_id_user_table_definitions_id_fk": { + "name": "table_row_executions_table_id_user_table_definitions_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_row_executions_row_id_user_table_rows_id_fk": { + "name": "table_row_executions_row_id_user_table_rows_id_fk", + "tableFrom": "table_row_executions", + "tableTo": "user_table_rows", + "columnsFrom": ["row_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "table_row_executions_row_id_group_id_pk": { + "name": "table_row_executions_row_id_group_id_pk", + "columns": ["row_id", "group_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_run_dispatches": { + "name": "table_run_dispatches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "cursor": { + "name": "cursor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "limit": { + "name": "limit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "processed_count": { + "name": "processed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_manual_run": { + "name": "is_manual_run", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "table_run_dispatches_active_idx": { + "name": "table_run_dispatches_active_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_run_dispatches_watchdog_idx": { + "name": "table_run_dispatches_watchdog_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_run_dispatches_table_id_user_table_definitions_id_fk": { + "name": "table_run_dispatches_table_id_user_table_definitions_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_workspace_id_workspace_id_fk": { + "name": "table_run_dispatches_workspace_id_workspace_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_run_dispatches_triggered_by_user_id_user_id_fk": { + "name": "table_run_dispatches_triggered_by_user_id_user_id_fk", + "tableFrom": "table_run_dispatches", + "tableTo": "user", + "columnsFrom": ["triggered_by_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.table_views": { + "name": "table_views", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "table_views_table_created_idx": { + "name": "table_views_table_created_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "table_views_table_default_unique": { + "name": "table_views_table_default_unique", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default = true", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "table_views_table_id_user_table_definitions_id_fk": { + "name": "table_views_table_id_user_table_definitions_id_fk", + "tableFrom": "table_views", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_workspace_id_workspace_id_fk": { + "name": "table_views_workspace_id_workspace_id_fk", + "tableFrom": "table_views", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "table_views_created_by_user_id_fk": { + "name": "table_views_created_by_user_id_fk", + "tableFrom": "table_views", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_log": { + "name": "usage_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "usage_log_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "usage_log_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost": { + "name": "cost", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_entity_type": { + "name": "billing_entity_type", + "type": "billing_entity_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "billing_entity_id": { + "name": "billing_entity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_period_start": { + "name": "billing_period_start", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "billing_period_end": { + "name": "billing_period_end", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "usage_log_user_created_at_idx": { + "name": "usage_log_user_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_source_idx": { + "name": "usage_log_source_idx", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_id_idx": { + "name": "usage_log_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workflow_id_idx": { + "name": "usage_log_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_event_key_unique": { + "name": "usage_log_event_key_unique", + "columns": [ + { + "expression": "event_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"usage_log\".\"event_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_entity_period_idx": { + "name": "usage_log_billing_entity_period_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_billing_period_cost_idx": { + "name": "usage_log_billing_period_cost_idx", + "columns": [ + { + "expression": "billing_entity_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_entity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_start", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "billing_period_end", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_log\".\"billing_entity_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_workspace_created_at_idx": { + "name": "usage_log_workspace_created_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "usage_log_execution_id_idx": { + "name": "usage_log_execution_id_idx", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "usage_log_user_id_user_id_fk": { + "name": "usage_log_user_id_user_id_fk", + "tableFrom": "usage_log", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "usage_log_workspace_id_workspace_id_fk": { + "name": "usage_log_workspace_id_workspace_id_fk", + "tableFrom": "usage_log", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "usage_log_workflow_id_workflow_id_fk": { + "name": "usage_log_workflow_id_workflow_id_fk", + "tableFrom": "usage_log", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_log_billing_scope_all_or_none": { + "name": "usage_log_billing_scope_all_or_none", + "value": "(\n (\"usage_log\".\"billing_entity_type\" IS NULL AND \"usage_log\".\"billing_entity_id\" IS NULL AND \"usage_log\".\"billing_period_start\" IS NULL AND \"usage_log\".\"billing_period_end\" IS NULL)\n OR\n (\"usage_log\".\"billing_entity_type\" IS NOT NULL AND \"usage_log\".\"billing_entity_id\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" IS NOT NULL AND \"usage_log\".\"billing_period_end\" IS NOT NULL AND \"usage_log\".\"billing_period_start\" < \"usage_log\".\"billing_period_end\")\n )" + } + }, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": ["normalized_email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_stats": { + "name": "user_stats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "total_manual_executions": { + "name": "total_manual_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_api_calls": { + "name": "total_api_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_webhook_triggers": { + "name": "total_webhook_triggers", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_scheduled_executions": { + "name": "total_scheduled_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_chat_executions": { + "name": "total_chat_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_executions": { + "name": "total_mcp_executions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_tokens_used": { + "name": "total_tokens_used", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_cost": { + "name": "total_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_usage_limit": { + "name": "current_usage_limit", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'5'" + }, + "usage_limit_updated_at": { + "name": "usage_limit_updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "current_period_cost": { + "name": "current_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_cost": { + "name": "last_period_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "billed_overage_this_period": { + "name": "billed_overage_this_period", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "pro_period_cost_snapshot": { + "name": "pro_period_cost_snapshot", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "pro_period_cost_snapshot_at": { + "name": "pro_period_cost_snapshot_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "credit_balance": { + "name": "credit_balance", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_copilot_cost": { + "name": "total_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_copilot_cost": { + "name": "current_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_period_copilot_cost": { + "name": "last_period_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "total_copilot_tokens": { + "name": "total_copilot_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_copilot_calls": { + "name": "total_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_calls": { + "name": "total_mcp_copilot_calls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_mcp_copilot_cost": { + "name": "total_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "current_period_mcp_copilot_cost": { + "name": "current_period_mcp_copilot_cost", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_active": { + "name": "last_active", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "billing_blocked": { + "name": "billing_blocked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "billing_blocked_reason": { + "name": "billing_blocked_reason", + "type": "billing_blocked_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "limit_notifications": { + "name": "limit_notifications", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": { + "user_stats_user_id_user_id_fk": { + "name": "user_stats_user_id_user_id_fk", + "tableFrom": "user_stats", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_stats_user_id_unique": { + "name": "user_stats_user_id_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_definitions": { + "name": "user_table_definitions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "max_rows": { + "name": "max_rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "row_count": { + "name": "row_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "rows_version": { + "name": "rows_version", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "schema_locked": { + "name": "schema_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "insert_locked": { + "name": "insert_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "update_locked": { + "name": "update_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "delete_locked": { + "name": "delete_locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "user_table_def_workspace_id_idx": { + "name": "user_table_def_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_folder_id_idx": { + "name": "user_table_def_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_name_unique": { + "name": "user_table_def_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"user_table_definitions\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_archived_at_idx": { + "name": "user_table_def_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_def_workspace_archived_partial_idx": { + "name": "user_table_def_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"user_table_definitions\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_definitions_workspace_id_workspace_id_fk": { + "name": "user_table_definitions_workspace_id_workspace_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_definitions_folder_id_folder_id_fk": { + "name": "user_table_definitions_folder_id_folder_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "user_table_definitions_created_by_user_id_fk": { + "name": "user_table_definitions_created_by_user_id_fk", + "tableFrom": "user_table_definitions", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_table_rows": { + "name": "user_table_rows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "table_id": { + "name": "table_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "user_table_rows_tenant_data_gin_idx": { + "name": "user_table_rows_tenant_data_gin_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"data\" jsonb_path_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "user_table_rows_workspace_table_idx": { + "name": "user_table_rows_workspace_table_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_position_idx": { + "name": "user_table_rows_table_position_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "position", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_order_key_idx": { + "name": "user_table_rows_table_order_key_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "order_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_table_rows_table_id_id_idx": { + "name": "user_table_rows_table_id_id_idx", + "columns": [ + { + "expression": "table_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "user_table_rows_table_id_user_table_definitions_id_fk": { + "name": "user_table_rows_table_id_user_table_definitions_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user_table_definitions", + "columnsFrom": ["table_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_workspace_id_workspace_id_fk": { + "name": "user_table_rows_workspace_id_workspace_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_table_rows_created_by_user_id_fk": { + "name": "user_table_rows_created_by_user_id_fk", + "tableFrom": "user_table_rows", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "verification_expires_at_idx": { + "name": "verification_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.waitlist": { + "name": "waitlist", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "waitlist_email_unique": { + "name": "waitlist_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook": { + "name": "webhook", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_status": { + "name": "registration_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "registration_generation": { + "name": "registration_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "config_fingerprint": { + "name": "config_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prepared_at": { + "name": "prepared_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "routing_key": { + "name": "routing_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_config": { + "name": "provider_config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "path_deployment_unique": { + "name": "path_deployment_unique", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_workflow_deployment_idx": { + "name": "webhook_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_routing_key_active_idx": { + "name": "webhook_routing_key_active_idx", + "columns": [ + { + "expression": "routing_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NULL AND \"webhook\".\"routing_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_archived_at_partial_idx": { + "name": "webhook_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468": { + "name": "idx_webhook_on_provider_is_active_workflow_id_deploym_bdeed5468", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_tiktok_credential_id_idx": { + "name": "webhook_tiktok_credential_id_idx", + "columns": [ + { + "expression": "((\"provider_config\")::jsonb ->> 'credentialId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook\".\"provider\" = 'tiktok' AND \"webhook\".\"is_active\" = true AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_webhook_on_workflow_id_block_id_updated_at_desc": { + "name": "idx_webhook_on_workflow_id_block_id_updated_at_desc", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_active_registration_unique": { + "name": "webhook_active_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'active' AND \"webhook\".\"block_id\" IS NOT NULL AND \"webhook\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_candidate_registration_unique": { + "name": "webhook_candidate_registration_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"webhook\".\"registration_status\" = 'candidate' AND \"webhook\".\"block_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_registration_status_generation_idx": { + "name": "webhook_registration_status_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_workflow_id_workflow_id_fk": { + "name": "webhook_workflow_id_workflow_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "webhook_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "webhook", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_registration_status_check": { + "name": "webhook_registration_status_check", + "value": "\"webhook\".\"registration_status\" IS NULL OR \"webhook\".\"registration_status\" IN ('active', 'candidate', 'retired', 'orphaned')" + }, + "webhook_registration_generation_check": { + "name": "webhook_registration_generation_check", + "value": "\"webhook\".\"registration_generation\" IS NULL OR \"webhook\".\"registration_generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.webhook_path_claim": { + "name": "webhook_path_claim", + "schema": "", + "columns": { + "path": { + "name": "path", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "webhook_path_claim_workflow_idx": { + "name": "webhook_path_claim_workflow_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_path_claim_workflow_id_workflow_id_fk": { + "name": "webhook_path_claim_workflow_id_workflow_id_fk", + "tableFrom": "webhook_path_claim", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_path_claim_generation_check": { + "name": "webhook_path_claim_generation_check", + "value": "\"webhook_path_claim\".\"generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow": { + "name": "workflow", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_synced": { + "name": "last_synced", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "is_deployed": { + "name": "is_deployed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deployed_at": { + "name": "deployed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "is_public_api": { + "name": "is_public_api", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "fork_sync_excluded": { + "name": "fork_sync_excluded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_user_id_idx": { + "name": "workflow_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_id_idx": { + "name": "workflow_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_user_workspace_idx": { + "name": "workflow_user_workspace_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_folder_name_active_unique": { + "name": "workflow_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_folder_sort_idx": { + "name": "workflow_folder_sort_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_archived_at_idx": { + "name": "workflow_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_workspace_archived_partial_idx": { + "name": "workflow_workspace_archived_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_user_id_user_id_fk": { + "name": "workflow_user_id_user_id_fk", + "tableFrom": "workflow", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_workspace_id_workspace_id_fk": { + "name": "workflow_workspace_id_workspace_id_fk", + "tableFrom": "workflow", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_folder_id_folder_id_fk": { + "name": "workflow_folder_id_folder_id_fk", + "tableFrom": "workflow", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_blocks": { + "name": "workflow_blocks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position_x": { + "name": "position_x", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "position_y": { + "name": "position_y", + "type": "numeric", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "horizontal_handles": { + "name": "horizontal_handles", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_wide": { + "name": "is_wide", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "advanced_mode": { + "name": "advanced_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "trigger_mode": { + "name": "trigger_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "locked": { + "name": "locked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "height": { + "name": "height", + "type": "numeric", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "sub_blocks": { + "name": "sub_blocks", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "outputs": { + "name": "outputs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_blocks_workflow_id_idx": { + "name": "workflow_blocks_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_blocks_type_idx": { + "name": "workflow_blocks_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_blocks_workflow_id_workflow_id_fk": { + "name": "workflow_blocks_workflow_id_workflow_id_fk", + "tableFrom": "workflow_blocks", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_checkpoints": { + "name": "workflow_checkpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_state": { + "name": "workflow_state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_checkpoints_user_id_idx": { + "name": "workflow_checkpoints_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_id_idx": { + "name": "workflow_checkpoints_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_id_idx": { + "name": "workflow_checkpoints_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_message_id_idx": { + "name": "workflow_checkpoints_message_id_idx", + "columns": [ + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_user_workflow_idx": { + "name": "workflow_checkpoints_user_workflow_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_workflow_chat_idx": { + "name": "workflow_checkpoints_workflow_chat_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_created_at_idx": { + "name": "workflow_checkpoints_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_checkpoints_chat_created_at_idx": { + "name": "workflow_checkpoints_chat_created_at_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_checkpoints_user_id_user_id_fk": { + "name": "workflow_checkpoints_user_id_user_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_workflow_id_workflow_id_fk": { + "name": "workflow_checkpoints_workflow_id_workflow_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_checkpoints_chat_id_copilot_chats_id_fk": { + "name": "workflow_checkpoints_chat_id_copilot_chats_id_fk", + "tableFrom": "workflow_checkpoints", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_deployment_operation": { + "name": "workflow_deployment_operation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_active_version_id": { + "name": "previous_active_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'preparing'" + }, + "component_readiness": { + "name": "component_readiness", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_deployment_operation_workflow_generation_unique": { + "name": "workflow_deployment_operation_workflow_generation_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_idempotency_unique": { + "name": "workflow_deployment_operation_workflow_idempotency_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_in_flight_unique": { + "name": "workflow_deployment_operation_workflow_in_flight_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_status_idx": { + "name": "workflow_deployment_operation_workflow_status_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_deployment_version_idx": { + "name": "workflow_deployment_operation_deployment_version_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_operation_workflow_version_generation_idx": { + "name": "workflow_deployment_operation_workflow_version_generation_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_operation_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_operation_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_deployment_operation_previous_active_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_deployment_operation", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["previous_active_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_deployment_operation_action_check": { + "name": "workflow_deployment_operation_action_check", + "value": "\"workflow_deployment_operation\".\"action\" IN ('deploy', 'activate')" + }, + "workflow_deployment_operation_status_check": { + "name": "workflow_deployment_operation_status_check", + "value": "\"workflow_deployment_operation\".\"status\" IN ('preparing', 'activating', 'active', 'failed', 'superseded')" + }, + "workflow_deployment_operation_generation_check": { + "name": "workflow_deployment_operation_generation_check", + "value": "\"workflow_deployment_operation\".\"generation\" > 0" + }, + "workflow_deployment_operation_protocol_version_check": { + "name": "workflow_deployment_operation_protocol_version_check", + "value": "\"workflow_deployment_operation\".\"protocol_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow_deployment_version": { + "name": "workflow_deployment_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_deployment_version_workflow_version_unique": { + "name": "workflow_deployment_version_workflow_version_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_workflow_active_idx": { + "name": "workflow_deployment_version_workflow_active_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_active", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_deployment_version_created_at_idx": { + "name": "workflow_deployment_version_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_deployment_version_workflow_id_workflow_id_fk": { + "name": "workflow_deployment_version_workflow_id_workflow_id_fk", + "tableFrom": "workflow_deployment_version", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_edges": { + "name": "workflow_edges", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_block_id": { + "name": "source_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_handle": { + "name": "source_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_handle": { + "name": "target_handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_edges_workflow_id_idx": { + "name": "workflow_edges_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_source_idx": { + "name": "workflow_edges_workflow_source_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_edges_workflow_target_idx": { + "name": "workflow_edges_workflow_target_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_edges_workflow_id_workflow_id_fk": { + "name": "workflow_edges_workflow_id_workflow_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_source_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_source_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["source_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_edges_target_block_id_workflow_blocks_id_fk": { + "name": "workflow_edges_target_block_id_workflow_blocks_id_fk", + "tableFrom": "workflow_edges", + "tableTo": "workflow_blocks", + "columnsFrom": ["target_block_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_logs": { + "name": "workflow_execution_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_snapshot_id": { + "name": "state_snapshot_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "total_duration_ms": { + "name": "total_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "execution_data": { + "name": "execution_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "cost": { + "name": "cost", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "cost_total": { + "name": "cost_total", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "models_used": { + "name": "models_used", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "files": { + "name": "files", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_execution_logs_workflow_id_idx": { + "name": "workflow_execution_logs_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_state_snapshot_id_idx": { + "name": "workflow_execution_logs_state_snapshot_id_idx", + "columns": [ + { + "expression": "state_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_deployment_version_id_idx": { + "name": "workflow_execution_logs_deployment_version_id_idx", + "columns": [ + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_trigger_idx": { + "name": "workflow_execution_logs_trigger_idx", + "columns": [ + { + "expression": "trigger", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_level_idx": { + "name": "workflow_execution_logs_level_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_started_at_idx": { + "name": "workflow_execution_logs_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_execution_id_unique": { + "name": "workflow_execution_logs_execution_id_unique", + "columns": [ + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workflow_started_at_idx": { + "name": "workflow_execution_logs_workflow_started_at_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_idx": { + "name": "workflow_execution_logs_workspace_started_at_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_started_at_id_desc_idx": { + "name": "workflow_execution_logs_workspace_started_at_id_desc_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"started_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"id\" DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_workspace_cost_total_idx": { + "name": "workflow_execution_logs_workspace_cost_total_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_total", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_models_used_idx": { + "name": "workflow_execution_logs_models_used_idx", + "columns": [ + { + "expression": "models_used", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "workflow_execution_logs_workspace_ended_at_id_idx": { + "name": "workflow_execution_logs_workspace_ended_at_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "date_trunc('milliseconds', \"ended_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_running_started_at_idx": { + "name": "workflow_execution_logs_running_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "status = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_execution_logs_completed_ended_at_idx": { + "name": "workflow_execution_logs_completed_ended_at_idx", + "columns": [ + { + "expression": "ended_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_execution_logs\".\"status\" = 'completed' AND \"workflow_execution_logs\".\"level\" = 'info' AND \"workflow_execution_logs\".\"ended_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_logs_workflow_id_workflow_id_fk": { + "name": "workflow_execution_logs_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_execution_logs_workspace_id_workspace_id_fk": { + "name": "workflow_execution_logs_workspace_id_workspace_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk": { + "name": "workflow_execution_logs_state_snapshot_id_workflow_execution_snapshots_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_execution_snapshots", + "columnsFrom": ["state_snapshot_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_execution_logs_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_execution_logs", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_execution_snapshots": { + "name": "workflow_execution_snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state_hash": { + "name": "state_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state_data": { + "name": "state_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_snapshots_workflow_id_idx": { + "name": "workflow_snapshots_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_hash_idx": { + "name": "workflow_snapshots_hash_idx", + "columns": [ + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_workflow_hash_idx": { + "name": "workflow_snapshots_workflow_hash_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_snapshots_created_at_idx": { + "name": "workflow_snapshots_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_execution_snapshots_workflow_id_workflow_id_fk": { + "name": "workflow_execution_snapshots_workflow_id_workflow_id_fk", + "tableFrom": "workflow_execution_snapshots", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_server": { + "name": "workflow_mcp_server", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_server_workspace_id_idx": { + "name": "workflow_mcp_server_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_created_by_idx": { + "name": "workflow_mcp_server_created_by_idx", + "columns": [ + { + "expression": "created_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_deleted_at_idx": { + "name": "workflow_mcp_server_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_server_workspace_deleted_partial_idx": { + "name": "workflow_mcp_server_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_server\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_server_workspace_id_workspace_id_fk": { + "name": "workflow_mcp_server_workspace_id_workspace_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_server_created_by_user_id_fk": { + "name": "workflow_mcp_server_created_by_user_id_fk", + "tableFrom": "workflow_mcp_server", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_mcp_tool": { + "name": "workflow_mcp_tool", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_description": { + "name": "tool_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parameter_schema": { + "name": "parameter_schema", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "parameter_description_overrides": { + "name": "parameter_description_overrides", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'::json" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_mcp_tool_server_id_idx": { + "name": "workflow_mcp_tool_server_id_idx", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_workflow_id_idx": { + "name": "workflow_mcp_tool_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_server_workflow_unique": { + "name": "workflow_mcp_tool_server_workflow_unique", + "columns": [ + { + "expression": "server_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_mcp_tool_archived_at_partial_idx": { + "name": "workflow_mcp_tool_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_mcp_tool\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk": { + "name": "workflow_mcp_tool_server_id_workflow_mcp_server_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow_mcp_server", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_mcp_tool_workflow_id_workflow_id_fk": { + "name": "workflow_mcp_tool_workflow_id_workflow_id_fk", + "tableFrom": "workflow_mcp_tool", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_schedule": { + "name": "workflow_schedule", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_version_id": { + "name": "deployment_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deployment_operation_id": { + "name": "deployment_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cron_expression": { + "name": "cron_expression", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_ran_at": { + "name": "last_ran_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_queued_at": { + "name": "last_queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "trigger_type": { + "name": "trigger_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "failed_count": { + "name": "failed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "infra_retry_count": { + "name": "infra_retry_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'workflow'" + }, + "job_title": { + "name": "job_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle": { + "name": "lifecycle", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'persistent'" + }, + "success_condition": { + "name": "success_condition", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "max_runs": { + "name": "max_runs", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "run_count": { + "name": "run_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "source_chat_id": { + "name": "source_chat_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_task_name": { + "name": "source_task_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_user_id": { + "name": "source_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "job_history": { + "name": "job_history", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "contexts": { + "name": "contexts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "excluded_dates": { + "name": "excluded_dates", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ends_at": { + "name": "ends_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_schedule_workflow_block_deployment_unique": { + "name": "workflow_schedule_workflow_block_deployment_unique", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_workflow_deployment_idx": { + "name": "workflow_schedule_workflow_deployment_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_archived_at_partial_idx": { + "name": "workflow_schedule_archived_at_partial_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6": { + "name": "idx_workflow_schedule_on_source_workspace_id_source_t_c07f3bba6", + "columns": [ + { + "expression": "source_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_workflow_idx": { + "name": "workflow_schedule_due_workflow_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deployment_version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND (\"workflow_schedule\".\"source_type\" = 'workflow' OR \"workflow_schedule\".\"source_type\" IS NULL)", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_schedule_due_job_idx": { + "name": "workflow_schedule_due_job_idx", + "columns": [ + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_queued_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_schedule\".\"archived_at\" IS NULL AND \"workflow_schedule\".\"status\" NOT IN ('disabled', 'completed') AND \"workflow_schedule\".\"source_type\" = 'job'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_schedule_workflow_id_workflow_id_fk": { + "name": "workflow_schedule_workflow_id_workflow_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk": { + "name": "workflow_schedule_deployment_version_id_workflow_deployment_version_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_version", + "columnsFrom": ["deployment_version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk": { + "name": "workflow_schedule_deployment_operation_id_workflow_deployment_operation_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workflow_deployment_operation", + "columnsFrom": ["deployment_operation_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_schedule_source_user_id_user_id_fk": { + "name": "workflow_schedule_source_user_id_user_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "user", + "columnsFrom": ["source_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_schedule_source_workspace_id_workspace_id_fk": { + "name": "workflow_schedule_source_workspace_id_workspace_id_fk", + "tableFrom": "workflow_schedule", + "tableTo": "workspace", + "columnsFrom": ["source_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_subflows": { + "name": "workflow_subflows", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_subflows_workflow_id_idx": { + "name": "workflow_subflows_workflow_id_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_subflows_workflow_type_idx": { + "name": "workflow_subflows_workflow_type_idx", + "columns": [ + { + "expression": "workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_subflows_workflow_id_workflow_id_fk": { + "name": "workflow_subflows_workflow_id_workflow_id_fk", + "tableFrom": "workflow_subflows", + "tableTo": "workflow", + "columnsFrom": ["workflow_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace": { + "name": "workspace", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'#33C482'" + }, + "logo_url": { + "name": "logo_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workspace_mode": { + "name": "workspace_mode", + "type": "workspace_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'grandfathered_shared'" + }, + "billed_account_user_id": { + "name": "billed_account_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_used_bytes": { + "name": "storage_used_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "allow_personal_api_keys": { + "name": "allow_personal_api_keys", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "inbox_enabled": { + "name": "inbox_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "inbox_address": { + "name": "inbox_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inbox_provider_id": { + "name": "inbox_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "organization_assigned_at": { + "name": "organization_assigned_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "forked_from_workspace_id": { + "name": "forked_from_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_owner_id_idx": { + "name": "workspace_owner_id_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_organization_id_idx": { + "name": "workspace_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_mode_idx": { + "name": "workspace_mode_idx", + "columns": [ + { + "expression": "workspace_mode", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_forked_from_workspace_id_idx": { + "name": "workspace_forked_from_workspace_id_idx", + "columns": [ + { + "expression": "forked_from_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_owner_id_user_id_fk": { + "name": "workspace_owner_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["owner_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_organization_id_organization_id_fk": { + "name": "workspace_organization_id_organization_id_fk", + "tableFrom": "workspace", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_billed_account_user_id_user_id_fk": { + "name": "workspace_billed_account_user_id_user_id_fk", + "tableFrom": "workspace", + "tableTo": "user", + "columnsFrom": ["billed_account_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workspace_forked_from_workspace_id_workspace_id_fk": { + "name": "workspace_forked_from_workspace_id_workspace_id_fk", + "tableFrom": "workspace", + "tableTo": "workspace", + "columnsFrom": ["forked_from_workspace_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workspace_storage_used_bytes_non_negative": { + "name": "workspace_storage_used_bytes_non_negative", + "value": "\"workspace\".\"storage_used_bytes\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workspace_byok_keys": { + "name": "workspace_byok_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_byok_workspace_provider_idx": { + "name": "workspace_byok_workspace_provider_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_byok_keys_workspace_id_workspace_id_fk": { + "name": "workspace_byok_keys_workspace_id_workspace_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_byok_keys_created_by_user_id_fk": { + "name": "workspace_byok_keys_created_by_user_id_fk", + "tableFrom": "workspace_byok_keys", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_environment": { + "name": "workspace_environment", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "variables": { + "name": "variables", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_environment_workspace_unique": { + "name": "workspace_environment_workspace_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_environment_workspace_id_workspace_id_fk": { + "name": "workspace_environment_workspace_id_workspace_id_fk", + "tableFrom": "workspace_environment", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_file": { + "name": "workspace_file", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uploaded_by": { + "name": "uploaded_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_file_workspace_id_idx": { + "name": "workspace_file_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_key_idx": { + "name": "workspace_file_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_deleted_at_idx": { + "name": "workspace_file_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_file_workspace_deleted_partial_idx": { + "name": "workspace_file_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_file\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_file_workspace_id_workspace_id_fk": { + "name": "workspace_file_workspace_id_workspace_id_fk", + "tableFrom": "workspace_file", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_file_uploaded_by_user_id_fk": { + "name": "workspace_file_uploaded_by_user_id_fk", + "tableFrom": "workspace_file", + "tableTo": "user", + "columnsFrom": ["uploaded_by"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_file_key_unique": { + "name": "workspace_file_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_files": { + "name": "workspace_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context": { + "name": "context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chat_id": { + "name": "chat_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_files_key_active_unique": { + "name": "workspace_files_key_active_unique", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_folder_name_active_unique": { + "name": "workspace_files_workspace_folder_name_active_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"folder_id\", '')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "original_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"deleted_at\" IS NULL AND \"workspace_files\".\"context\" = 'workspace' AND \"workspace_files\".\"workspace_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_display_name_unique": { + "name": "workspace_files_chat_display_name_unique", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workspace_files\".\"context\" = 'mothership' AND \"workspace_files\".\"chat_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_key_idx": { + "name": "workspace_files_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_user_id_idx": { + "name": "workspace_files_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_id_idx": { + "name": "workspace_files_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_folder_id_idx": { + "name": "workspace_files_folder_id_idx", + "columns": [ + { + "expression": "folder_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_context_idx": { + "name": "workspace_files_context_idx", + "columns": [ + { + "expression": "context", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_chat_id_idx": { + "name": "workspace_files_chat_id_idx", + "columns": [ + { + "expression": "chat_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_deleted_at_idx": { + "name": "workspace_files_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_files_workspace_deleted_partial_idx": { + "name": "workspace_files_workspace_deleted_partial_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workspace_files\".\"deleted_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_files_user_id_user_id_fk": { + "name": "workspace_files_user_id_user_id_fk", + "tableFrom": "workspace_files", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_workspace_id_workspace_id_fk": { + "name": "workspace_files_workspace_id_workspace_id_fk", + "tableFrom": "workspace_files", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_files_folder_id_folder_id_fk": { + "name": "workspace_files_folder_id_folder_id_fk", + "tableFrom": "workspace_files", + "tableTo": "folder", + "columnsFrom": ["folder_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workspace_files_chat_id_copilot_chats_id_fk": { + "name": "workspace_files_chat_id_copilot_chats_id_fk", + "tableFrom": "workspace_files", + "tableTo": "copilot_chats", + "columnsFrom": ["chat_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_block_map": { + "name": "workspace_fork_block_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_workflow_id": { + "name": "parent_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_block_id": { + "name": "parent_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_workflow_id": { + "name": "child_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_block_id": { + "name": "child_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_block_map_child_ws_parent_unique": { + "name": "workspace_fork_block_map_child_ws_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_unique": { + "name": "workspace_fork_block_map_child_ws_child_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_parent_wf_idx": { + "name": "workspace_fork_block_map_child_ws_parent_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_block_map_child_ws_child_wf_idx": { + "name": "workspace_fork_block_map_child_ws_child_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "child_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_block_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_block_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_block_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_dependent_value": { + "name": "workspace_fork_dependent_value", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workflow_id": { + "name": "target_workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_block_id": { + "name": "target_block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_block_key": { + "name": "sub_block_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_dependent_value_child_ws_wf_idx": { + "name": "workspace_fork_dependent_value_child_ws_wf_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_dependent_value_field_unique": { + "name": "workspace_fork_dependent_value_field_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workflow_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_block_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sub_block_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_dependent_value_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_dependent_value", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_promote_run": { + "name": "workspace_fork_promote_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_workspace_id": { + "name": "source_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_workspace_id": { + "name": "target_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "workspace_fork_promote_direction", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_promote_run_child_ws_target_unique": { + "name": "workspace_fork_promote_run_child_ws_target_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_promote_run_target_ws_idx": { + "name": "workspace_fork_promote_run_target_ws_idx", + "columns": [ + { + "expression": "target_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_promote_run_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_promote_run_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_promote_run_created_by_user_id_fk": { + "name": "workspace_fork_promote_run_created_by_user_id_fk", + "tableFrom": "workspace_fork_promote_run", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_fork_resource_map": { + "name": "workspace_fork_resource_map", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "child_workspace_id": { + "name": "child_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_type": { + "name": "resource_type", + "type": "workspace_fork_resource_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_resource_id": { + "name": "parent_resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "child_resource_id": { + "name": "child_resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_fork_resource_map_child_ws_idx": { + "name": "workspace_fork_resource_map_child_ws_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_ws_type_idx": { + "name": "workspace_fork_resource_map_child_ws_type_idx", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_fork_resource_map_child_type_parent_unique": { + "name": "workspace_fork_resource_map_child_type_parent_unique", + "columns": [ + { + "expression": "child_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_fork_resource_map_child_workspace_id_workspace_id_fk": { + "name": "workspace_fork_resource_map_child_workspace_id_workspace_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "workspace", + "columnsFrom": ["child_workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_fork_resource_map_created_by_user_id_fk": { + "name": "workspace_fork_resource_map_created_by_user_id_fk", + "tableFrom": "workspace_fork_resource_map", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_sandbox": { + "name": "workspace_sandbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "sandbox_language", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "dependencies": { + "name": "dependencies", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "spec_hash": { + "name": "spec_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_sandbox_workspace_name_unique": { + "name": "workspace_sandbox_workspace_name_unique", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_sandbox_workspace_idx": { + "name": "workspace_sandbox_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workspace_sandbox_spec_hash_idx": { + "name": "workspace_sandbox_spec_hash_idx", + "columns": [ + { + "expression": "spec_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workspace_sandbox_workspace_id_workspace_id_fk": { + "name": "workspace_sandbox_workspace_id_workspace_id_fk", + "tableFrom": "workspace_sandbox", + "tableTo": "workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspace_sandbox_created_by_user_id_fk": { + "name": "workspace_sandbox_created_by_user_id_fk", + "tableFrom": "workspace_sandbox", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.academy_cert_status": { + "name": "academy_cert_status", + "schema": "public", + "values": ["active", "revoked", "expired"] + }, + "public.background_work_kind": { + "name": "background_work_kind", + "schema": "public", + "values": ["deployment_side_effects", "fork_content_copy", "fork_sync", "fork_rollback"] + }, + "public.background_work_status_value": { + "name": "background_work_status_value", + "schema": "public", + "values": ["pending", "processing", "completed", "completed_with_warnings", "failed"] + }, + "public.billing_blocked_reason": { + "name": "billing_blocked_reason", + "schema": "public", + "values": ["payment_failed", "dispute"] + }, + "public.billing_entity_type": { + "name": "billing_entity_type", + "schema": "public", + "values": ["user", "organization"] + }, + "public.chat_type": { + "name": "chat_type", + "schema": "public", + "values": ["mothership", "copilot"] + }, + "public.copilot_async_tool_status": { + "name": "copilot_async_tool_status", + "schema": "public", + "values": ["pending", "running", "completed", "failed", "cancelled", "delivered"] + }, + "public.copilot_run_status": { + "name": "copilot_run_status", + "schema": "public", + "values": ["active", "paused_waiting_for_tool", "resuming", "complete", "error", "cancelled"] + }, + "public.copilot_tool_permission_decision": { + "name": "copilot_tool_permission_decision", + "schema": "public", + "values": ["allow", "allow_chat", "always_allow", "skip"] + }, + "public.credential_member_role": { + "name": "credential_member_role", + "schema": "public", + "values": ["admin", "member"] + }, + "public.credential_member_status": { + "name": "credential_member_status", + "schema": "public", + "values": ["active", "pending", "revoked"] + }, + "public.credential_type": { + "name": "credential_type", + "schema": "public", + "values": ["oauth", "env_workspace", "env_personal", "service_account"] + }, + "public.data_drain_cadence": { + "name": "data_drain_cadence", + "schema": "public", + "values": ["hourly", "daily"] + }, + "public.data_drain_destination": { + "name": "data_drain_destination", + "schema": "public", + "values": ["s3", "gcs", "azure_blob", "datadog", "bigquery", "snowflake", "webhook"] + }, + "public.data_drain_run_status": { + "name": "data_drain_run_status", + "schema": "public", + "values": ["running", "success", "failed"] + }, + "public.data_drain_run_trigger": { + "name": "data_drain_run_trigger", + "schema": "public", + "values": ["cron", "manual"] + }, + "public.data_drain_source": { + "name": "data_drain_source", + "schema": "public", + "values": ["workflow_logs", "job_logs", "audit_logs", "copilot_chats", "copilot_runs"] + }, + "public.execution_large_value_reference_source": { + "name": "execution_large_value_reference_source", + "schema": "public", + "values": ["execution_log", "paused_snapshot"] + }, + "public.folder_resource_type": { + "name": "folder_resource_type", + "schema": "public", + "values": ["workflow", "file", "knowledge_base", "table"] + }, + "public.invitation_kind": { + "name": "invitation_kind", + "schema": "public", + "values": ["organization", "workspace"] + }, + "public.invitation_membership_intent": { + "name": "invitation_membership_intent", + "schema": "public", + "values": ["internal", "external"] + }, + "public.invitation_status": { + "name": "invitation_status", + "schema": "public", + "values": ["pending", "accepted", "rejected", "cancelled", "expired"] + }, + "public.permission_type": { + "name": "permission_type", + "schema": "public", + "values": ["admin", "write", "read"] + }, + "public.sandbox_image_status": { + "name": "sandbox_image_status", + "schema": "public", + "values": ["pending", "building", "ready", "failed"] + }, + "public.sandbox_language": { + "name": "sandbox_language", + "schema": "public", + "values": ["javascript", "python"] + }, + "public.usage_log_category": { + "name": "usage_log_category", + "schema": "public", + "values": ["model", "fixed", "tool"] + }, + "public.usage_log_source": { + "name": "usage_log_source", + "schema": "public", + "values": [ + "workflow", + "wand", + "copilot", + "workspace-chat", + "mcp_copilot", + "mothership_block", + "knowledge-base", + "voice-input", + "enrichment" + ] + }, + "public.workspace_fork_promote_direction": { + "name": "workspace_fork_promote_direction", + "schema": "public", + "values": ["push", "pull"] + }, + "public.workspace_fork_resource_type": { + "name": "workspace_fork_resource_type", + "schema": "public", + "values": [ + "workflow", + "oauth_credential", + "service_account_credential", + "env_var", + "table", + "knowledge_base", + "knowledge_document", + "file", + "mcp_server", + "workflow_mcp_server", + "custom_tool", + "skill" + ] + }, + "public.workspace_mode": { + "name": "workspace_mode", + "schema": "public", + "values": ["personal", "organization", "grandfathered_shared"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index 43d813aafee..d9cc85fe28e 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -1933,6 +1933,13 @@ "when": 1785352177983, "tag": "0276_drop_legacy_folder_tables", "breakpoints": true + }, + { + "idx": 277, + "version": "7", + "when": 1785358812851, + "tag": "0277_workspace_sandboxes", + "breakpoints": true } ] } diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 778f1aa8420..6f872b2d3a2 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -4272,3 +4272,86 @@ export const dataDrainRuns = pgTable( drainStartedIdx: index('data_drain_runs_drain_started_idx').on(table.drainId, table.startedAt), }) ) + +export const sandboxLanguageEnum = pgEnum('sandbox_language', ['javascript', 'python']) + +export type SandboxLanguageValue = (typeof sandboxLanguageEnum.enumValues)[number] + +export const sandboxImageStatusEnum = pgEnum('sandbox_image_status', [ + 'pending', + 'building', + 'ready', + 'failed', +]) + +export type SandboxImageStatusValue = (typeof sandboxImageStatusEnum.enumValues)[number] + +/** + * A workspace's named library of dependency sets. Provider-agnostic: the same + * row drives a prebuilt E2B template and a Daytona runtime install, and only the + * materialization step differs. `specHash` is the content address shared with + * `sandboxImage`, so editing dependencies points the sandbox at a new build + * while the old one stays valid for in-flight executions. + */ +export const workspaceSandbox = pgTable( + 'workspace_sandbox', + { + id: text('id').primaryKey(), + workspaceId: text('workspace_id') + .notNull() + .references(() => workspace.id, { onDelete: 'cascade' }), + name: text('name').notNull(), + language: sandboxLanguageEnum('language').notNull(), + dependencies: jsonb('dependencies').$type().notNull().default(sql`'[]'::jsonb`), + specHash: text('spec_hash').notNull(), + createdBy: text('created_by').references(() => user.id, { onDelete: 'set null' }), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), + }, + (table) => ({ + workspaceNameUnique: uniqueIndex('workspace_sandbox_workspace_name_unique').on( + table.workspaceId, + table.name + ), + workspaceIdx: index('workspace_sandbox_workspace_idx').on(table.workspaceId), + specHashIdx: index('workspace_sandbox_spec_hash_idx').on(table.specHash), + }) +) + +/** + * Build registry for the prebuilt strategy, keyed by content address so two + * workspaces declaring the same dependency set share one build. Never written + * under a runtime-strategy provider. + */ +export const sandboxImage = pgTable( + 'sandbox_image', + { + id: text('id').primaryKey(), + provider: text('provider').notNull(), + specHash: text('spec_hash').notNull(), + spec: jsonb('spec').notNull(), + status: sandboxImageStatusEnum('status').notNull().default('pending'), + /** Passed to the provider at create time once `status` is `ready`. */ + imageRef: text('image_ref'), + /** Provider-side image identifier, when it differs from `imageRef`. */ + providerImageId: text('provider_image_id'), + buildId: text('build_id'), + /** Classified taxonomy code; see lib/execution/remote-sandbox/build-errors.ts. */ + errorCode: text('error_code'), + /** User-facing copy rendered from the code at classification time. */ + errorMessage: text('error_message'), + /** Installer log tail, shown behind a disclosure. */ + errorDetail: text('error_detail'), + lastUsedAt: timestamp('last_used_at'), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), + }, + (table) => ({ + providerSpecUnique: uniqueIndex('sandbox_image_provider_spec_unique').on( + table.provider, + table.specHash + ), + statusIdx: index('sandbox_image_status_idx').on(table.status), + lastUsedIdx: index('sandbox_image_last_used_idx').on(table.lastUsedAt), + }) +) diff --git a/packages/testing/src/mocks/env-flags.mock.ts b/packages/testing/src/mocks/env-flags.mock.ts index d8e3ecd699a..d62fd102717 100644 --- a/packages/testing/src/mocks/env-flags.mock.ts +++ b/packages/testing/src/mocks/env-flags.mock.ts @@ -29,6 +29,7 @@ export interface EnvFlagsMockState { isAccessControlEnabled: boolean isOrganizationsEnabled: boolean isInboxEnabled: boolean + isSandboxesEnabled: boolean isWhitelabelingEnabled: boolean isAuditLogsEnabled: boolean isDataRetentionEnabled: boolean @@ -76,6 +77,7 @@ const defaultEnvFlagsState: EnvFlagsMockState = { // `true` so upgrades do not remove a feature. See // ENTERPRISE_FEATURE_LEGACY_DEFAULTS. isInboxEnabled: true, + isSandboxesEnabled: true, isWhitelabelingEnabled: true, isSessionPoliciesEnabled: true, isAuditLogsEnabled: false, diff --git a/packages/testing/src/mocks/schema.mock.ts b/packages/testing/src/mocks/schema.mock.ts index 5cccc2c8af0..e0803f29e05 100644 --- a/packages/testing/src/mocks/schema.mock.ts +++ b/packages/testing/src/mocks/schema.mock.ts @@ -242,6 +242,33 @@ export const schemaMock = { createdAt: 'createdAt', updatedAt: 'updatedAt', }, + workspaceSandbox: { + id: 'id', + workspaceId: 'workspaceId', + name: 'name', + language: 'language', + dependencies: 'dependencies', + specHash: 'specHash', + createdBy: 'createdBy', + createdAt: 'createdAt', + updatedAt: 'updatedAt', + }, + sandboxImage: { + id: 'id', + provider: 'provider', + specHash: 'specHash', + spec: 'spec', + status: 'status', + imageRef: 'imageRef', + providerImageId: 'providerImageId', + buildId: 'buildId', + errorCode: 'errorCode', + errorMessage: 'errorMessage', + errorDetail: 'errorDetail', + lastUsedAt: 'lastUsedAt', + createdAt: 'createdAt', + updatedAt: 'updatedAt', + }, workspaceBYOKKeys: { id: 'id', workspaceId: 'workspaceId', diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 8df1ad2a511..6c84d43eca0 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 997, - zodRoutes: 997, + totalRoutes: 1000, + zodRoutes: 1000, nonZodRoutes: 0, } as const @@ -68,6 +68,7 @@ const INDIRECT_ZOD_ROUTES = new Set([ 'apps/sim/app/api/cron/cleanup-tasks/route.ts', 'apps/sim/app/api/cron/cleanup-soft-deletes/route.ts', 'apps/sim/app/api/cron/cleanup-stale-executions/route.ts', + 'apps/sim/app/api/cron/cleanup-sandbox-images/route.ts', 'apps/sim/app/api/cron/renew-subscriptions/route.ts', 'apps/sim/app/api/cron/reconcile-billing-seats/route.ts', 'apps/sim/app/api/cron/reconcile-inbox-entitlement/route.ts', diff --git a/scripts/setup/checks.ts b/scripts/setup/checks.ts index d304234926e..1d1a196b598 100644 --- a/scripts/setup/checks.ts +++ b/scripts/setup/checks.ts @@ -385,6 +385,26 @@ function checkCoherence(ctx: CheckContext): Finding[] { }) } + // NEXT_PUBLIC_SANDBOX_ENABLED is not a 1:1 twin: remote execution is available + // under E2B_ENABLED or, when SANDBOX_PROVIDER=daytona, DAYTONA_API_KEY. Without + // it the Function block hides its language dropdown and sandbox selector even + // though the server would happily run Python. + const sandboxProvider = (sim.vars.get('SANDBOX_PROVIDER') || 'e2b').toLowerCase() + const remoteSandboxAvailable = + sandboxProvider === 'daytona' + ? Boolean(sim.vars.get('DAYTONA_API_KEY')) + : isTruthy(sim.vars.get('E2B_ENABLED')) + if (remoteSandboxAvailable && !isTruthy(sim.vars.get('NEXT_PUBLIC_SANDBOX_ENABLED'))) { + findings.push({ + group: 'coherence', + status: 'fail', + message: + 'remote sandboxes are configured but NEXT_PUBLIC_SANDBOX_ENABLED is unset — the Function block will hide its language and sandbox controls', + fix: 'doctor --fix sets NEXT_PUBLIC_SANDBOX_ENABLED=true', + autofix: () => writeEnvValues(sim.target, { NEXT_PUBLIC_SANDBOX_ENABLED: 'true' }), + }) + } + const disableAuth = sim.vars.get('DISABLE_AUTH') if ( isTruthy(disableAuth) && From ee0157df4bd5d90f7543ba2d6f8ecd5f19cb5abf Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 30 Jul 2026 17:21:17 -0700 Subject: [PATCH 10/21] feat(tables): add currency column type on a new column-type registry (#6106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(tables): add currency column type on a new column-type registry Adds a `currency` column type, and consolidates the per-type knowledge it would otherwise have been scattered across. **Currency.** Stores a plain number and carries an ISO 4217 `currencyCode` as display metadata. That split is what keeps it cheap: filtering, sorting, uniqueness and CSV export all reuse the numeric paths unchanged, changing a column's currency rewrites no rows, and the public row output stays a number rather than a locale-formatted string consumers would have to reparse. Input accepts the shapes an amount actually arrives in — `$1,234.56`, `1 234,56 €`, `(12.00)` — so pastes, CSV imports and tool writes land as numbers instead of being nulled. **The registry.** Adding this type initially required edits in ~40 places: 32 switch arms under `lib/table`, ~26 UI branches, two hand-maintained icon maps, and a coercion implementation duplicated four times. Every one of those failed silently when missed — a missing `jsonbCastForType` arm compares numbers as text; a missing compatibility arm blocks all conversions. `lib/table/column-types/` now holds one file per type carrying its label, icon, badge colour, storage cast, filter operators, coercion, validation, compatibility and formatting. `Record` on both registries is the completeness gate: adding a type to the union is a compile error naming exactly the two files to fill in, and the interface then requires every field. The 32 switch arms are down to 3. Two duplicates collapse as a consequence: - The client no longer mirrors the server's select id-resolution. Those helpers lived in `validation.ts`, which imports drizzle, so anything reaching them became server-only and the grid hand-rolled its own copy. Extracting them to `select-options.ts` lets both sides share one implementation, so the optimistic cache can no longer disagree with what gets persisted. - The two icon maps become one registry read. It also fixes a live inconsistency it surfaced: currency got a numeric keypad in the grid's inline editor but a plain text field in the row modal. Behaviour-neutral by construction: all 1046 tests in the touched areas pass unchanged, with no test edits. * test(tables): guard the column-type registry's invariants Property tests for the registry itself rather than any one type: entries key by their own id, COLUMN_TYPES stays derived, an unknown type degrades to string instead of throwing, only opaque-id types restrict filter operators, only configuration-free types are CSV-inferable, and every type that can reject a draft has a message to show. Plus the metadata-ownership matrix, which pins the generic ownership check to the same answers the hardcoded per-type rules gave. These target the registry's silent-failure class — a wrong jsonbCast or a stray operator whitelist used to be invisible until a filter failed in SQL. Both are verified to fail under mutation. * fix(tables): read exponent-form amounts and reject bad currency PATCHes up front Two P1s from review. Scientific notation lost magnitude. `String()` emits exponent form past 1e21, so a stored amount round-trips through the editor as `1e+21` — and the sanitizer treated the `e` as decoration to strip, reading it back as 121. An untouched cell silently lost 19 orders of magnitude on its next edit. Exponent form is now taken at face value, but only when the string is wholly a numeric literal once symbols are removed, so `12 EUR` (whose `E` survives the strip) still parses through the separator path. A failed currency PATCH left a partial rename. `renameColumn` commits in its own transaction before the currency write, so a `currencyCode` the service would reject — an unsupported code, or any code on a non-currency column — errored only after the rename had stuck. Both are now caught before the first write, matching the guard the route already applies to unique-on-select for exactly this reason. * refactor(tables): finish the registry migration and drop the dead config Audit pass over every consumer, closing the gaps the first cut left. Functional gap: the copilot agent had no currency support at all — it could create a currency column with no code and could never re-denominate one. `add_column` and `update_column` now accept `currencyCode`, with the same up-front validation and the same code-only routing as the HTTP routes. Config that consumers were still restating, now read from the registry: - `supportsUnique` replaces the unique-on-select guard stated in three places (service, both column routes, the copilot tool). - `editor === 'toggle'` replaces seven `type === 'boolean'` checks in the grid and expanded popover, all of which meant the same thing. - `defaultMetadata` replaces the per-type stamping in `addTableColumn` and `updateColumnType`. - `sampleValue` replaces the per-type example values in the LLM prompt scaffolding. - `storesOpaqueIds` replaces the select filter in the find-row matcher. Dead config removed: `getTypeBadgeVariant` had zero callers (already dead on staging), and it was the only reader of `badgeVariant` — so the field, its union, and all seven values went with it. `inferFromCsv` was read by nothing but a comment; CSV inference is an ordered heuristic a boolean cannot express, so it is gone too and `InferredCsvColumnType` is no longer exported. Fixes a latent crash found on the way: unique-constraint checking normalized a cell keyed on its RUNTIME type but reconstructed it keyed on the column's DECLARED type, so a unique `date` column stored a bare `2024-01-01` and then threw `SyntaxError` parsing it back. Both directions now go through JSON unconditionally. Pre-existing, unrelated to currency. Adds the `/add-column-type` skill and a Tables section in CLAUDE.md/AGENTS.md pointing at it, so the next type is one file plus two registry entries. * fix(tables): run the column PATCH guards ahead of the rename, not after it Greptile was right and my previous reply was wrong. The guards were added in the right shape but the wrong place — below `renameColumn`, which is the first write and commits in its own transaction. A PATCH combining a rename with an invalid currency therefore still committed the rename and then returned 400, exactly the counterexample reported. Moved the column lookup and all three pre-flight guards above every write. This also closes the same latent hole for the pre-existing unique-on-select guard, which sat in the same position. Adds route tests that assert `renameColumn` was never called on each rejection path, and that a valid combined rename + currency change still targets the new name. Verified to fail against the previous ordering. * fix(tables): make the retype gate and the write path share one parser A simplify pass over the registry found two real defects and several places the abstraction was being worked around. Silent data loss on conversion. `isCompatibleWith` was hand-written per type and had already drifted from `coerce`, despite the interface promising they could not: `boolean` accepted '1'/'0'/0/1 in the gate but only 'true'/'false' in the write path, so converting a column holding "1" reported zero incompatible rows and then nulled every one of them. `date` drifted the other way. `isCompatibleWith` is now optional and defaults to `coerce(...).ok`, so the two are the same code; only `select` overrides, because its rules are about the column (cleared-vs-required, cardinality) not the value. `isColumnType` used `in`, which matches inherited keys — `isColumnType('toString')` was true and `columnTypeById('toString')` returned `Function.prototype.toString`, which the validator would then call `.validateDefinition()` on. Now `Object.hasOwn`. `defaultMetadata` only ran on the currency arm of a retype, so a future type would get its defaults on create but silently not on conversion. It now runs for every non-select target, carrying forward only metadata the TARGET type declares it owns — a currency→text conversion no longer strands a currencyCode. The index doc claimed the registry is kept out of the `@/lib/table` barrel so 44 server modules don't pull `@sim/emcn/icons`. That was false: `constants.ts` re-exported `COLUMN_TYPES` from the icon-carrying `registry.ts`, and the barrel re-exports `constants`. `COLUMN_TYPES` now lives in the icon-free `types.ts`; verified with an import tracer that both are icon-free again. Also: 5 no-op `validateDefinition`s and 4 duplicated formatters collapsed into registry defaults; `CURRENCY_OPTIONS` was an eager module-load IIFE costing ~8ms of ICU work on every table API route for a list only the config sidebar reads, now built on first call; and the skill's validation grep claimed 'should return nothing' when it returns 8 legitimate hits — it now explains how to tell a leak from a genuine special case. * fix(tables): reject a non-leading sign so dates don't parse as amounts Found by Cursor Bugbot. `parseCurrencyInput` dropped every `-` as decoration, so an ISO date's hyphens vanished and its digit groups joined: `2024-01-01` read as 20240101. With the gate now sharing the write path's parser, a date → currency conversion reported zero incompatible rows and silently turned every cell into a huge number. A sign is only meaningful at the front; an interior one means the string is not a single amount. Leading signs, accounting parentheses, symbols, ISO codes, grouping separators, and exponent form all still parse — covered by the existing cases plus new ones, verified to fail without the fix. * fix(tables): use getErrorMessage in the columns route test mock `check:utils` bans the inline `e instanceof Error ? e.message : fallback` form; the mock for `rootErrorMessage` used it. * fix(tables): rename the column last so a failed write leaves it untouched Greptile's remaining concern: the pre-flight guards read a schema snapshot, so a column-type change landing concurrently can still make a later write fail — and with the rename running first, that failure returned an error with the rename already committed. Guards cannot close that window; each write is its own locked transaction and only the write itself sees the authoritative state. Ordering can. The rename is the one write that is purely cosmetic, so it now runs last: a failed typed write leaves the column entirely untouched, and a failed rename leaves the typed change applied under the old name — the recoverable half. The typed writes target the column's current name, since no rename has happened yet. Tests cover both directions: a typed write rejected mid-flight must not rename, and a successful one must rename strictly after. Verified to fail under the previous ordering. * fix(tables): write back coerced values on every conversion Round 4 findings, all real. A conversion is allowed exactly when the target type's `coerce` accepts the value — and `coerce` frequently TRANSFORMS it. Only `select` and `currency` wrote the transformed value back, so a conversion to any other transforming type left the cell holding its old bytes under the new type. Converting a number column to `date` accepted epoch values, stored them unchanged, and then `(data->>'col')::timestamptz` failed on EVERY query against that column. I opened this myself by defaulting `isCompatibleWith` to `coerce(...).ok`. Fixed at the class rather than the instance: the compatibility scan now records whatever `coerce` produced whenever it differs from what is stored, and one generic write-back applies it. That subsumes the currency-specific migration entirely, so it and its helpers are gone. `select` keeps its own id↔name migrations, which are not coerce-expressible in the outbound direction. The post-conversion column definition is built once, before the scan, so the coercion reads the same metadata the stored value is later validated against. Exponent parsing was ambiguous when followed by text: `1e5 EUR` read as 15. An `e` with a digit on both sides is an exponent marker, so if the string is not a clean numeric literal it is refused rather than guessed — the digit on both sides is what keeps the `E` inside `12 EUR` parsing normally. A failed rename could still leave a typed change committed. The one rename failure a caller can cause — a name already taken — is now rejected up front, leaving only the concurrent-collision race, which no pre-flight check can close without spanning all writes in one transaction. * fix(tables): stop a blank cell blocking an optional type conversion Found by Cursor Bugbot. `''` is incompatible with every numeric type, and the compatibility scan counted it as a hard blocker regardless of whether the target was optional — so a text column with a single empty cell could not be converted to a number at all, and the error said 'to a required ...' either way. An unreadable-but-empty cell is not a conversion failure. The write path already turns an unreadable value into null on an optional column, so the conversion now does the same and records null for it. A required target still reports it, which the existing guard above already does with the message that actually fits. Also pins the two intentional divergences from the pre-registry behavior. A differential run of the registry against the pre-refactor implementations (55 values x 7 column shapes) found ZERO coercion differences and exactly two compatibility differences, both deliberate: boolean now rejects the '1'/'0' conversions the old gate accepted and then nulled, and date now accepts the epoch numbers its write path always accepted. Tests pin both so neither can be silently reverted or widened. * fix(tables): refuse conversions that would invent or destroy values Final adversarial scan found two data-corrupting conversions, both opened by defaulting the retype gate to the write path's parser. number → date destroyed every value. `date.coerce` reads a number as epoch milliseconds, which is right for one deliberate write and catastrophic applied to a whole column: 1, 5, 42 became three timestamps in January 1970, and a Unix-seconds column landed in 1970 rather than the year it meant. Irreversible. `date` now overrides the gate to reject numbers, restoring the pre-refactor behavior, and the contract states the rule the override obeys: a gate may be STRICTER than `coerce`, never looser. Stricter refuses a bulk conversion while single writes still work; looser is the direction that corrupts. string → currency invented values. The parser stripped every non-digit and joined what was left, so `01/02/2024` read as 1022024, `Room 101` as 101, and `0.1.2` as 12 — a column of SKUs or phone numbers converted with zero reported incompatibilities. What remains after removing symbols, spacing and an ISO code must now be only digits and separators, and grouping must be well-formed (a first group of 1-3 digits, the rest exactly 3). Every legitimate form still parses, including all the locale variants. Also generifies the last three metadata leaks: `buildConvertedColumn` strips and carries back by iterating the key list rather than naming keys (naming them meant a future type's metadata rode onto a target that rejects it, failing that column's validation on every later write), `normalizeColumn` forwards metadata through a shared `typeMetadataOf`, and `filterOperatorsFor` moved onto the definition — it was a per-type branch inside the registry's own accessor, the one thing the registry exists to forbid. Skill corrected: it claimed COLUMN_TYPES derives from the registry (backwards), promised exactly two compile errors (four once a type owns metadata), used a grep that missed half the real branches, and never mentioned `import.ts`'s second coercion path, whose silent default arm is the costliest miss available. Differential re-run vs the pre-refactor implementations: 0 coercion differences, 1 intentional compatibility difference (boolean no longer accepts the 0/1 conversions the old gate accepted and then nulled). * fix(tables): let the row modal accept formatted amounts again Found by Cursor Bugbot. I unified the row modal's input type with the grid's `inputMode` last round, but in the wrong direction: mapping `inputMode: 'decimal'` to `` made the modal reject $1,234.56, 1.234,56 and (12.00) — the exact formats `parseCurrencyInput` exists to accept, and which the grid's inline editor takes fine. A native number input and a numeric keypad are different things. Types whose parser accepts formatted text now say so, and get a text field with `inputMode='decimal'` — the shape the grid already uses. A plain number keeps the native input, its spinner, and its validation. * fix(tables): fold a rename into the write it accompanies Closes the last partial-update window, properly rather than by pre-checking around it. A rename is metadata-only — `renameColumn`'s own comment says so: rows, metadata, and workflow-group refs all key on the stable column id, so it is a pure schema write. Nothing forced it to be its own transaction. Running it separately is what created the window: whichever half committed first survived a failure in the other, and no pre-flight guard can close a concurrent collision because only the write itself sees authoritative state. The four column writes now accept an optional `newName` and apply it through one shared `applyPendingRename`, which validates the name shape and checks the collision against the very schema snapshot that write is landing in. A combined request rides the rename on whichever write runs last, so both halves commit together or neither does — a concurrent claim on the name now aborts the whole transaction instead of leaving the other change applied. The routes also address every write by the column's stable id rather than its name, so folding a rename into one write cannot break the next one's lookup. A rename with nothing to ride on still runs standalone. What remains partial is a type write followed by a failing constraints write — two independently locked transactions, pre-existing, and untouched by this PR. * fix(tables): migrate scalar cells when converting a column to select Found by Cursor Bugbot. `resolveSelectOptionId` stringifies a number or boolean before matching, so a `number` column whose values equal option NAMES passes the compatibility gate — but `migrateCellsToSelectIds` only rewrote JSONB `string` and `array` cells. Those cells stayed raw numbers inside a select column, where they render as nothing and fail option membership on the next write. `data->>key` yields the text form for every scalar, so the existing lookup already worked; the predicate was simply too narrow. Widened to cover `number` and `boolean`. The outbound migration is unchanged — cells leaving a select column are option ids, always strings or arrays. Pre-existing on staging (both the resolver's scalar handling and the migration SQL predate this branch), but it lives in a file this PR creates. Tests pin the resolver behavior the predicate depends on, so narrowing either one without the other now fails. * fix(tables): validate a retype's unique against the values it writes Validated the last partial-update seam with a focused investigation rather than assuming. The answer was split. `required` is already safe: `updateColumnType` runs the same `countEmptyCells` against the constraint the request is about to set, which is why that check exists. `unique` was not, and the reachable case commits the unrecoverable half. A text column holding "5" and "5.0", PATCHed with {type: number, unique: true}: the conversion succeeds and coerces both to 5, then the separate constraint write finds duplicates and 400s — with the column already numeric and "5.0" irreversibly rewritten. A pre-scan of the raw text finds nothing; the conversion is what manufactures the duplicate. The retype now carries `unique` and checks it after the write-back, against the values it just wrote. Constraint changes on a workflow-output column were the same shape — rejected by the constraint write, after a type change had committed. Now rejected in the route's pre-flight block, before any write. The duplicate scan is extracted and shared between both paths for the same reason `countEmptyCells` is: two copies of one rule is the drift that produced the original required-check bug. Deliberately NOT merging `updateColumnType` and `updateColumnConstraints`. They assert different lock levels (destructive vs schema-only) and only the retype needs the full row scan, so merging would either force a constraints-only toggle to materialize every row or reintroduce the branching it was meant to remove. With both reachable failures pre-validated, what remains at the seam is concurrent races no in-process check can close. * fix(tables): don't drop a rename when the write it rides on no-ops Found by Cursor Bugbot — a bug I introduced folding the rename in. `updateColumnCurrency` returns early when the code is unchanged, and that return sat ahead of the rename, so PATCH {name, currencyCode} with the column's current code answered 200 with the rename silently discarded. Both early returns now treat a pending rename as work: the currency path only no-ops when the code is unchanged AND no rename is riding along, and the retype path applies a rename-only write when the type is unchanged. `applyPendingRename` signals "nothing to do" by returning the same reference, which is what lets both detect it cleanly. Also extracts `persistColumns` — five sites were repeating the same schema-write-and-return. * fix(tables): make a combined column PATCH a single transaction Finishes the fold-in rather than pre-validating around the seam. A retype now APPLIES the constraints it already validates against — it checks empty cells for `required` and post-conversion duplicates for `unique`, so it was doing the work without persisting the result — and the route skips the separate constraint write when the type changed. A request combining a rename, a retype and constraint changes is now one locked transaction: no half of it can commit while another fails. The separate constraint write remains for requests that do not change type, which is the only case that still needs it. Deliberately still NOT merging the two service functions. They assert different lock levels (destructive vs schema-only) and only the retype needs the full row scan into memory, so a merged function would force a constraints-only toggle to materialize every row or reintroduce the branching it was meant to remove. Folding the payload in gets atomicity without either cost. * fix(tables): reject a flattened list as an amount; fold constraints into every typed write Two findings from round 11. Multi-select converted to nonsense amounts. `selectValueForConversion` flattens a multi cell to its comma-joined option names, and the parser read that as a formatted number: options 12 and 34 became 12.34, and 100 and 200 became 100200. No real amount puts whitespace after a separator, but a delimited list does — so a separator followed by whitespace is now refused. Every legitimate form still parses, including space-grouped locales. Combined options-or-currency + constraints could still commit partially. Those two writes now carry constraints the same way the retype does, through one shared `applyConstraints` that validates (workflow-output, empty cells for required, supportsUnique and duplicates for unique) and applies them. The separate constraint write now runs only when no typed write does. Three copies of those rules is the drift that produced the original required-check bug, so they live in one place. * fix(tables): validate constraints after the migrations that rewrite cells Self-caught while reviewing my own previous commit, which introduced both. `updateColumnOptions` ran the shared `applyConstraints` BEFORE its cell migrations. Those migrations rewrite stored values — a single<->multi toggle changes the shape, removing an option clears cells — so a `unique` scan read the pre-migration values, passed, and the rewrite could then produce the duplicates the scan was meant to prevent. Moved to after the migrations, which is where `updateColumnType` already had it. The same commit also left the options path running `required`'s empty-cell check twice: once in the shared helper and once in its original inline block, whose comment still described a separate constraint write that no longer runs. Removed the duplicate — one query, one rule, which is the whole point of the shared helper. Also routes the options path through `persistColumns` like the others. * fix(tables): stop inventing amounts from identifiers; fix the copilot retype Adversarial pass over the final state, seven real findings. Two destroyed data. The copilot `update_column` still used the two-transaction pattern the HTTP routes were fixed for: `unique` was never forwarded to the typed write, so a retype+unique committed the conversion and then failed the constraint — the same irrecoverable half. It now rides the typed write, and the separate constraint write only runs when no typed write did. And the parser's three-letter strip removed ANY three letters, not an ISO code: `SKU400` parsed as 400, `ABC1234` as 1234. Converting a column of part numbers to currency rewrote every cell with an invented value — while the comment two lines above claimed a SKU was exactly what it prevented. The rule is now that a letter touching a digit means identifier, not amount; a currency marker is always separated by a space or a symbol. That same change fixed a class the review surfaced: the pinned currencies could not parse their own conventional notation. `R$ 1.234,56`, `1 234,56 kr`, `1234,56 zł`, `CHF 1’234.56` and Indian lakh grouping (`₹12,34,567.89`) all work now — these are what Intl emits, so a paste from a spreadsheet was being rejected. `updateColumnConstraints` was a fourth copy of the constraint rules the shared helper exists to unify, and had already drifted: it hardcoded `type === 'select'` where the helper asks the registry, so a future type declaring `supportsUnique: false` would have been ignored on that path. It now uses the helper. `updateColumnType`'s unchanged-type early return silently discarded every field except the rename. Callers gate on the type changing, but from a read taken before the lock — so a concurrent change could land there with real work pending and answer success. It now throws. Also: `UpdateColumnCurrencyData` was missing `required`, which only compiled because the routes pass it through a spread; a missing column returns 404 instead of a 400 reading "of type undefined"; and the comments describing the old two-transaction architecture are gone. Verified NOT a bug: CSV export of a currency column writes the raw number, so export/import round-trips losslessly. * fix(tables): read the negative and RTL forms Intl actually emits An Intl sweep across 24 locales found two forms the parser rejected, both from an ordinary spreadsheet paste. `Intl` emits U+2212 MINUS SIGN rather than the ASCII hyphen for negatives in several locales, so `−12,50 kr` read as null instead of -12.5. And it wraps RTL-locale output in invisible bidi control marks, so `‏1,234.56 ‏₪` carried characters that are not part of the amount. Both are now normalized away. 24 locales x 6 amounts now round-trip, up from 99/100 when the sweep started — and the test generates them from `Intl` rather than listing them by hand, so a parser change cannot quietly regress a locale nobody remembered to write down. Locales that format with their own numeral systems (Arabic-Indic) are still rejected, and now say so in the docstring. That is a safe failure — null rather than a wrong value — and supporting them is a wider decision than this type, since it would also touch `number`, display, and sorting. --- .agents/skills/add-column-type/SKILL.md | 160 +++++ .claude/commands/add-column-type.md | 159 ++++ .claude/skills/add-column-type | 1 + .cursor/commands/add-column-type.md | 154 ++++ AGENTS.md | 6 + CLAUDE.md | 6 + .../content/docs/en/integrations/table.mdx | 2 +- apps/docs/content/docs/en/tables/index.mdx | 4 + .../api/table/[tableId]/columns/route.test.ts | 320 +++++++++ .../app/api/table/[tableId]/columns/route.ts | 140 +++- apps/sim/app/api/table/utils.ts | 6 +- .../api/v1/tables/[tableId]/columns/route.ts | 140 +++- .../column-config-sidebar.tsx | 45 ++ .../column-config-sidebar/column-types.ts | 27 +- .../components/row-modal/row-modal.tsx | 27 +- .../table-grid/cells/cell-render.tsx | 8 + .../cells/expanded-cell-popover.tsx | 22 +- .../table-grid/cells/inline-editors.tsx | 31 +- .../table-grid/headers/column-type-icon.tsx | 28 +- .../components/table-grid/headers/index.ts | 2 +- .../components/table-grid/table-grid.tsx | 69 +- .../workflow-sidebar/workflow-sidebar.tsx | 12 +- .../[workspaceId]/tables/[tableId]/table.tsx | 4 +- .../[workspaceId]/tables/[tableId]/utils.ts | 107 +-- apps/sim/hooks/use-table-undo.ts | 1 + apps/sim/lib/api/contracts/tables.ts | 34 +- .../copilot/tools/server/table/user-table.ts | 61 +- .../table/__tests__/column-conversion.test.ts | 145 +++- .../__tests__/column-type-registry.test.ts | 205 ++++++ apps/sim/lib/table/__tests__/currency.test.ts | 279 +++++++ .../lib/table/__tests__/validation.test.ts | 66 ++ apps/sim/lib/table/cell-format.ts | 3 +- apps/sim/lib/table/column-naming.ts | 16 +- apps/sim/lib/table/column-types/boolean.ts | 40 ++ apps/sim/lib/table/column-types/currency.ts | 68 ++ apps/sim/lib/table/column-types/date.ts | 63 ++ apps/sim/lib/table/column-types/index.ts | 21 + apps/sim/lib/table/column-types/json.ts | 38 + apps/sim/lib/table/column-types/number.ts | 45 ++ .../lib/table/column-types/registry.server.ts | 238 ++++++ apps/sim/lib/table/column-types/registry.ts | 117 +++ apps/sim/lib/table/column-types/select.ts | 143 ++++ apps/sim/lib/table/column-types/string.ts | 39 + .../lib/table/column-types/types.server.ts | 47 ++ apps/sim/lib/table/column-types/types.ts | 199 +++++ apps/sim/lib/table/columns/service.ts | 680 +++++++++++------- apps/sim/lib/table/constants.ts | 12 +- apps/sim/lib/table/currency.ts | 326 +++++++++ apps/sim/lib/table/export-format.ts | 9 +- apps/sim/lib/table/import.test.ts | 9 + apps/sim/lib/table/import.ts | 33 +- apps/sim/lib/table/index.ts | 1 + apps/sim/lib/table/llm/enrichment.ts | 5 +- apps/sim/lib/table/rows/service.ts | 3 +- apps/sim/lib/table/select-options.ts | 87 +++ apps/sim/lib/table/select-values.ts | 2 +- apps/sim/lib/table/sql.ts | 46 +- apps/sim/lib/table/types.ts | 62 +- apps/sim/lib/table/validation.ts | 280 ++------ apps/sim/stores/table/types.ts | 3 + packages/emcn/src/icons/index.ts | 1 + packages/emcn/src/icons/type-currency.tsx | 26 + 62 files changed, 4142 insertions(+), 791 deletions(-) create mode 100644 .agents/skills/add-column-type/SKILL.md create mode 100644 .claude/commands/add-column-type.md create mode 120000 .claude/skills/add-column-type create mode 100644 .cursor/commands/add-column-type.md create mode 100644 apps/sim/app/api/table/[tableId]/columns/route.test.ts create mode 100644 apps/sim/lib/table/__tests__/column-type-registry.test.ts create mode 100644 apps/sim/lib/table/__tests__/currency.test.ts create mode 100644 apps/sim/lib/table/column-types/boolean.ts create mode 100644 apps/sim/lib/table/column-types/currency.ts create mode 100644 apps/sim/lib/table/column-types/date.ts create mode 100644 apps/sim/lib/table/column-types/index.ts create mode 100644 apps/sim/lib/table/column-types/json.ts create mode 100644 apps/sim/lib/table/column-types/number.ts create mode 100644 apps/sim/lib/table/column-types/registry.server.ts create mode 100644 apps/sim/lib/table/column-types/registry.ts create mode 100644 apps/sim/lib/table/column-types/select.ts create mode 100644 apps/sim/lib/table/column-types/string.ts create mode 100644 apps/sim/lib/table/column-types/types.server.ts create mode 100644 apps/sim/lib/table/column-types/types.ts create mode 100644 apps/sim/lib/table/currency.ts create mode 100644 apps/sim/lib/table/select-options.ts create mode 100644 packages/emcn/src/icons/type-currency.tsx diff --git a/.agents/skills/add-column-type/SKILL.md b/.agents/skills/add-column-type/SKILL.md new file mode 100644 index 00000000000..05f9816aa5f --- /dev/null +++ b/.agents/skills/add-column-type/SKILL.md @@ -0,0 +1,160 @@ +--- +name: add-column-type +description: Add a new table column type to Sim — registry entry, icon, storage shape, coercion, and the behavioral hooks the grid and API read. Use when adding a value kind under `apps/sim/lib/table/column-types/`. +argument-hint: +--- + +# Adding a Table Column Type + +A column type is **one file** in `apps/sim/lib/table/column-types/` plus a registry entry. Everything that varies per type — label, icon, storage cast, coercion, validation, conversion compatibility, formatting, editor, filter operators — lives on that one object, so no consumer needs editing. + +This was not always true: adding `currency` originally took ~40 edits across 32 `switch` arms and 26 UI branches, each of which failed **silently** when missed. The registry exists to make that impossible, so the rule is absolute: **if you find yourself adding a `case 'yourtype':` anywhere outside `column-types/`, the registry is missing a field. Add the field instead.** + +## Hard Rule: the compiler tells you what to do + +Do **not** hunt for places to edit. Add your type to the `ColumnType` union first and let `tsc` produce the list: + +```bash +cd apps/sim && bunx tsc --noEmit -p tsconfig.json +``` + +You will get two errors, naming `column-types/registry.ts` and `column-types/registry.server.ts`. Register in both. + +If your type owns metadata, adding its key to `TYPE_SPECIFIC_COLUMN_KEYS` produces two more legitimate errors — `FOREIGN_METADATA_VERB` in `validation.ts` (a `Record` over those keys) and the key's absence from `ColumnDefinition`. Those are the gate working, not sites to "fix". + +Any error beyond those four is a site reading a hardcoded type list that should read the registry — fix that site, don't work around it. + +## Directory Structure + +``` +apps/sim/lib/table/column-types/ +├── types.ts # ColumnTypeDefinition — the contract you implement +├── types.server.ts # ColumnTypeServerDefinition — cell migrations only +├── registry.ts # Record ← client-safe, the gate +├── registry.server.ts # Record ← adds migrations (drizzle) +├── index.ts # barrel + accessors (columnTypeOf, columnTypeById, …) +└── {type}.ts # one file per type — what you write +``` + +## Step 1: Pick the storage shape + +Decide what a cell literally holds in `user_table_rows.data` (JSONB). This drives almost everything else: + +| Storage | `jsonbCast` | Notes | +|---------|-------------|-------| +| number | `'numeric'` | Filters/sorts compare numerically. `currency` does this. | +| ISO string | `'timestamptz'` | `date` does this. | +| string / bool / object | `null` | Text comparison is correct. | + +**Prefer an existing primitive over a new shape.** `currency` stores a plain number and keeps its ISO code as *display metadata* — which is why filtering, sorting, uniqueness, and CSV export all reuse the numeric paths untouched, and why re-denominating a column rewrites zero rows. + +## Step 2: Add the icon + +Create `packages/emcn/src/icons/type-{name}.tsx`, copying the geometry conventions of its siblings exactly: + +```tsx +import type { SVGProps } from 'react' + +/** + * Type {name} icon component - {what the glyph is} for {name} columns + * @param props - SVG properties including className, fill, etc. + */ +export function Type{Pascal}(props: SVGProps) { + return ( + + ) +} +``` + +- `viewBox='-1.75 -1.5 24 24'` is the **`type-*` family** value, not the set-wide default. Match the family. +- Center the glyph on the viewBox's optical center (**y = 10.5**, **x = 10.25**) — every sibling does, and a few tenths off is visible at `size-[14px]`. +- Export alphabetically **by component name** in `packages/emcn/src/icons/index.ts`. + +## Step 3: Write the type file + +`apps/sim/lib/table/column-types/{name}.ts`. Copy the closest existing type and change what differs. Every field is required by the interface, so the compiler enumerates them for you — read the TSDoc in `types.ts` rather than guessing. + +The three that are easy to get wrong: + +- **`coerce`** is the *single* write-path implementation. The server runs it before persisting **and** the grid runs it to fill the optimistic cache. Accept every shape the value legitimately arrives in (paste, CSV, tool write), because rejecting means the cell is nulled. +- **`isCompatibleWith`** gates type conversion and must read the value **exactly as `coerce` will**, or a conversion will pass its check and then null the cell. +- **`ownedMetadata`** lists the `ColumnDefinition` keys your type owns. Anything you add must also be added to `TYPE_SPECIFIC_COLUMN_KEYS` in `types.ts` and given a phrase in `FOREIGN_METADATA_VERB` in `validation.ts` — both are `Record`-typed, so the compiler will tell you. + +## Step 4: Register + +Add the entry to `COLUMN_TYPE_REGISTRY` in `registry.ts` **and** `COLUMN_TYPE_SERVER_REGISTRY` in `registry.server.ts`. + +`COLUMN_TYPES` is declared in `types.ts` (not derived from the registry — the registry is annotated `Record` against it, which is the gate). `constants.ts` re-exports it, so `columnTypeSchema = z.enum(COLUMN_TYPES)` picks your type up with no edit. **Type-specific metadata does not** — see the next step. + +## Step 5: Migrations (only if the stored bytes change) + +If converting an existing column **to** your type must rewrite cells, add `migrateCellsTo` in `registry.server.ts`; if converting **away** must rewrite them, add `migrateCellsFrom`. + +This is load-bearing, not cosmetic: filters and sorts apply `jsonbCast` to whatever is stored, so leaving a non-castable string behind makes **every query on that column fail** — not merely render oddly. + +Prefer set-based SQL. When the transform genuinely needs JS (`currency`'s separator disambiguation), compute the values during the compatibility scan and pass them through `resolved`, then apply them in one batched statement. + +## Naming Convention + +- Type id: lowercase, singular — `currency`, not `Currency` or `currencies` +- File: `column-types/{id}.ts`, export `const {id}ColumnType` +- Icon: `type-{kebab}.tsx`, export `Type{Pascal}` + +## Watch out + +- **Import cycles.** `column-types/select.ts` imports `select-values.ts`, so `select-values.ts` must **not** import the registry — that closes a cycle and fails at module init. Inside a type's own helper module the string literal is the implementation, not a config leak. +- **The client-safe boundary.** `registry.ts` and everything it imports must stay free of `@sim/db`, `drizzle-orm`, and `next/server` — the tables grid imports it directly. A React icon is fine (it's a component *reference*, never called server-side). Only `registry.server.ts` may touch drizzle. +- **Don't re-export the registry from `@/lib/table`.** 44 server modules import that barrel; routing this through it pulls `@sim/emcn/icons` into all of them. Deep-import `@/lib/table/column-types`. +- **`import.ts`'s `coerceValue` is a SECOND write path and is not opt-in.** Importing into a column of your type always hits it, and its `default` arm silently `String(value)`s — so a missing `case` stores text in a column whose `jsonbCast` is numeric, and then every filter and sort on that column errors in Postgres. Add a `case`, even though the switch compiles without one. (It is deliberately separate from the registry's `coerce`: an import wants an unparseable value to survive as its raw string so the row error can name it.) +- **CSV inference** is an ordered heuristic in `import.ts`, deliberately not registry-driven. A new type is not inferred from a CSV unless you extend `inferColumnType` — usually you should not, since inference cannot supply configuration (an option set, a currency code). + +## If your type owns metadata, read this + +Registering the *type* is compiler-enforced. Registering its *metadata* is not, and that is where the remaining manual work lives. A key like `precision` has to be added in each of these, none of which will fail to compile if you forget: + +| Where | What happens if you forget | +|---|---| +| `lib/table/types.ts` `ColumnDefinition` | (this one DOES fail — the ownership loop indexes it) | +| `column-types/types.ts` `TYPE_SPECIFIC_COLUMN_KEYS` | it is never stripped on conversion, and poisons the target type | +| `lib/api/contracts/tables.ts` — the schema slot in all three column schemas, plus `refineColumnOptions` | zod strips it at the boundary; silently never saved | +| `columns/service.ts` `addTableColumn` param type | callers cannot pass it | +| A metadata-only update path (`updateColumnCurrency` is the model) + a branch in both column routes + the copilot tool | changing it on an existing column is a silent 200 no-op | +| `column-config-sidebar.tsx` | no UI to set it | +| `table-grid.tsx` delete-column undo + `use-table-undo.ts` restore | undo silently resets it to the default | + +`normalizeColumn`, `buildConvertedColumn`, and the undo snapshot read `TYPE_SPECIFIC_COLUMN_KEYS` generically, so those three are already zero-edit. + +**Known gap:** the metadata-only update path is ~6 near-identical copies (service + 2 routes + copilot). A `metadataUpdate` descriptor on `ColumnTypeServerDefinition` would collapse them; until that exists, copy `currency`'s. + +## Checklist Before Finishing + +- [ ] Added to the `ColumnType` union in `column-types/types.ts` +- [ ] `column-types/{id}.ts` created, every interface field filled in +- [ ] Registered in **both** `registry.ts` and `registry.server.ts` +- [ ] Icon added, centered on the family's optical center, exported alphabetically +- [ ] `migrateCellsTo` / `migrateCellsFrom` added if the stored bytes change +- [ ] New metadata keys added to `TYPE_SPECIFIC_COLUMN_KEYS` + `FOREIGN_METADATA_VERB` +- [ ] Unit tests for `coerce` / `isCompatibleWith` round-trips, verified to fail without the code +- [ ] Docs row added to `apps/docs/content/docs/en/tables/index.mdx` + +## Final Validation (Required) + +1. **`cd apps/sim && bunx tsc --noEmit -p tsconfig.json`** — must be clean. If any file *outside* `column-types/` errors, that file has a hardcoded type list; fix it to read the registry. +2. **Grep for leaks** — `grep -rnE "(===|!==) '{id}'|case '{id}':" apps/sim --include='*.ts' --include='*.tsx' | grep -v column-types/`. (All three forms: a plain `!==` and a `case` are how half of `currency`'s real branches are written.) Hits are expected; judge each. A hit is fine when it mounts a specific React component or encodes a genuinely one-off behavior (`json`'s mono textarea, `date`'s timezone-aware parsing). A hit is a **leak** when it restates something the registry could answer — an icon, a label, a colour, an operator set, a cast, a coercion. Leaks get a registry field, not a new branch. +3. **Run the suite** — `bunx vitest run lib/table 'app/workspace/[workspaceId]/tables' lib/api app/api/table app/api/v1 lib/copilot/tools/server/table`. Existing tests must pass **unchanged**; needing to edit one means you changed behavior for the other types. +4. **`bun run lint:check`, `bun run check:api-validation`, `bun run check:client-boundary`** from the repo root. +5. **Exercise it in the running app** on a table with one column of every type: create, edit inline / in the expanded popover / in the row modal, paste from a spreadsheet, filter, sort, convert to and from other types, export CSV, undo a column delete. diff --git a/.claude/commands/add-column-type.md b/.claude/commands/add-column-type.md new file mode 100644 index 00000000000..b390ccc0b98 --- /dev/null +++ b/.claude/commands/add-column-type.md @@ -0,0 +1,159 @@ +--- +description: Add a new table column type to Sim — registry entry, icon, storage shape, coercion, and the behavioral hooks the grid and API read. Use when adding a value kind under `apps/sim/lib/table/column-types/`. +argument-hint: +--- + +# Adding a Table Column Type + +A column type is **one file** in `apps/sim/lib/table/column-types/` plus a registry entry. Everything that varies per type — label, icon, storage cast, coercion, validation, conversion compatibility, formatting, editor, filter operators — lives on that one object, so no consumer needs editing. + +This was not always true: adding `currency` originally took ~40 edits across 32 `switch` arms and 26 UI branches, each of which failed **silently** when missed. The registry exists to make that impossible, so the rule is absolute: **if you find yourself adding a `case 'yourtype':` anywhere outside `column-types/`, the registry is missing a field. Add the field instead.** + +## Hard Rule: the compiler tells you what to do + +Do **not** hunt for places to edit. Add your type to the `ColumnType` union first and let `tsc` produce the list: + +```bash +cd apps/sim && bunx tsc --noEmit -p tsconfig.json +``` + +You will get two errors, naming `column-types/registry.ts` and `column-types/registry.server.ts`. Register in both. + +If your type owns metadata, adding its key to `TYPE_SPECIFIC_COLUMN_KEYS` produces two more legitimate errors — `FOREIGN_METADATA_VERB` in `validation.ts` (a `Record` over those keys) and the key's absence from `ColumnDefinition`. Those are the gate working, not sites to "fix". + +Any error beyond those four is a site reading a hardcoded type list that should read the registry — fix that site, don't work around it. + +## Directory Structure + +``` +apps/sim/lib/table/column-types/ +├── types.ts # ColumnTypeDefinition — the contract you implement +├── types.server.ts # ColumnTypeServerDefinition — cell migrations only +├── registry.ts # Record ← client-safe, the gate +├── registry.server.ts # Record ← adds migrations (drizzle) +├── index.ts # barrel + accessors (columnTypeOf, columnTypeById, …) +└── {type}.ts # one file per type — what you write +``` + +## Step 1: Pick the storage shape + +Decide what a cell literally holds in `user_table_rows.data` (JSONB). This drives almost everything else: + +| Storage | `jsonbCast` | Notes | +|---------|-------------|-------| +| number | `'numeric'` | Filters/sorts compare numerically. `currency` does this. | +| ISO string | `'timestamptz'` | `date` does this. | +| string / bool / object | `null` | Text comparison is correct. | + +**Prefer an existing primitive over a new shape.** `currency` stores a plain number and keeps its ISO code as *display metadata* — which is why filtering, sorting, uniqueness, and CSV export all reuse the numeric paths untouched, and why re-denominating a column rewrites zero rows. + +## Step 2: Add the icon + +Create `packages/emcn/src/icons/type-{name}.tsx`, copying the geometry conventions of its siblings exactly: + +```tsx +import type { SVGProps } from 'react' + +/** + * Type {name} icon component - {what the glyph is} for {name} columns + * @param props - SVG properties including className, fill, etc. + */ +export function Type{Pascal}(props: SVGProps) { + return ( + + ) +} +``` + +- `viewBox='-1.75 -1.5 24 24'` is the **`type-*` family** value, not the set-wide default. Match the family. +- Center the glyph on the viewBox's optical center (**y = 10.5**, **x = 10.25**) — every sibling does, and a few tenths off is visible at `size-[14px]`. +- Export alphabetically **by component name** in `packages/emcn/src/icons/index.ts`. + +## Step 3: Write the type file + +`apps/sim/lib/table/column-types/{name}.ts`. Copy the closest existing type and change what differs. Every field is required by the interface, so the compiler enumerates them for you — read the TSDoc in `types.ts` rather than guessing. + +The three that are easy to get wrong: + +- **`coerce`** is the *single* write-path implementation. The server runs it before persisting **and** the grid runs it to fill the optimistic cache. Accept every shape the value legitimately arrives in (paste, CSV, tool write), because rejecting means the cell is nulled. +- **`isCompatibleWith`** gates type conversion and must read the value **exactly as `coerce` will**, or a conversion will pass its check and then null the cell. +- **`ownedMetadata`** lists the `ColumnDefinition` keys your type owns. Anything you add must also be added to `TYPE_SPECIFIC_COLUMN_KEYS` in `types.ts` and given a phrase in `FOREIGN_METADATA_VERB` in `validation.ts` — both are `Record`-typed, so the compiler will tell you. + +## Step 4: Register + +Add the entry to `COLUMN_TYPE_REGISTRY` in `registry.ts` **and** `COLUMN_TYPE_SERVER_REGISTRY` in `registry.server.ts`. + +`COLUMN_TYPES` is declared in `types.ts` (not derived from the registry — the registry is annotated `Record` against it, which is the gate). `constants.ts` re-exports it, so `columnTypeSchema = z.enum(COLUMN_TYPES)` picks your type up with no edit. **Type-specific metadata does not** — see the next step. + +## Step 5: Migrations (only if the stored bytes change) + +If converting an existing column **to** your type must rewrite cells, add `migrateCellsTo` in `registry.server.ts`; if converting **away** must rewrite them, add `migrateCellsFrom`. + +This is load-bearing, not cosmetic: filters and sorts apply `jsonbCast` to whatever is stored, so leaving a non-castable string behind makes **every query on that column fail** — not merely render oddly. + +Prefer set-based SQL. When the transform genuinely needs JS (`currency`'s separator disambiguation), compute the values during the compatibility scan and pass them through `resolved`, then apply them in one batched statement. + +## Naming Convention + +- Type id: lowercase, singular — `currency`, not `Currency` or `currencies` +- File: `column-types/{id}.ts`, export `const {id}ColumnType` +- Icon: `type-{kebab}.tsx`, export `Type{Pascal}` + +## Watch out + +- **Import cycles.** `column-types/select.ts` imports `select-values.ts`, so `select-values.ts` must **not** import the registry — that closes a cycle and fails at module init. Inside a type's own helper module the string literal is the implementation, not a config leak. +- **The client-safe boundary.** `registry.ts` and everything it imports must stay free of `@sim/db`, `drizzle-orm`, and `next/server` — the tables grid imports it directly. A React icon is fine (it's a component *reference*, never called server-side). Only `registry.server.ts` may touch drizzle. +- **Don't re-export the registry from `@/lib/table`.** 44 server modules import that barrel; routing this through it pulls `@sim/emcn/icons` into all of them. Deep-import `@/lib/table/column-types`. +- **`import.ts`'s `coerceValue` is a SECOND write path and is not opt-in.** Importing into a column of your type always hits it, and its `default` arm silently `String(value)`s — so a missing `case` stores text in a column whose `jsonbCast` is numeric, and then every filter and sort on that column errors in Postgres. Add a `case`, even though the switch compiles without one. (It is deliberately separate from the registry's `coerce`: an import wants an unparseable value to survive as its raw string so the row error can name it.) +- **CSV inference** is an ordered heuristic in `import.ts`, deliberately not registry-driven. A new type is not inferred from a CSV unless you extend `inferColumnType` — usually you should not, since inference cannot supply configuration (an option set, a currency code). + +## If your type owns metadata, read this + +Registering the *type* is compiler-enforced. Registering its *metadata* is not, and that is where the remaining manual work lives. A key like `precision` has to be added in each of these, none of which will fail to compile if you forget: + +| Where | What happens if you forget | +|---|---| +| `lib/table/types.ts` `ColumnDefinition` | (this one DOES fail — the ownership loop indexes it) | +| `column-types/types.ts` `TYPE_SPECIFIC_COLUMN_KEYS` | it is never stripped on conversion, and poisons the target type | +| `lib/api/contracts/tables.ts` — the schema slot in all three column schemas, plus `refineColumnOptions` | zod strips it at the boundary; silently never saved | +| `columns/service.ts` `addTableColumn` param type | callers cannot pass it | +| A metadata-only update path (`updateColumnCurrency` is the model) + a branch in both column routes + the copilot tool | changing it on an existing column is a silent 200 no-op | +| `column-config-sidebar.tsx` | no UI to set it | +| `table-grid.tsx` delete-column undo + `use-table-undo.ts` restore | undo silently resets it to the default | + +`normalizeColumn`, `buildConvertedColumn`, and the undo snapshot read `TYPE_SPECIFIC_COLUMN_KEYS` generically, so those three are already zero-edit. + +**Known gap:** the metadata-only update path is ~6 near-identical copies (service + 2 routes + copilot). A `metadataUpdate` descriptor on `ColumnTypeServerDefinition` would collapse them; until that exists, copy `currency`'s. + +## Checklist Before Finishing + +- [ ] Added to the `ColumnType` union in `column-types/types.ts` +- [ ] `column-types/{id}.ts` created, every interface field filled in +- [ ] Registered in **both** `registry.ts` and `registry.server.ts` +- [ ] Icon added, centered on the family's optical center, exported alphabetically +- [ ] `migrateCellsTo` / `migrateCellsFrom` added if the stored bytes change +- [ ] New metadata keys added to `TYPE_SPECIFIC_COLUMN_KEYS` + `FOREIGN_METADATA_VERB` +- [ ] Unit tests for `coerce` / `isCompatibleWith` round-trips, verified to fail without the code +- [ ] Docs row added to `apps/docs/content/docs/en/tables/index.mdx` + +## Final Validation (Required) + +1. **`cd apps/sim && bunx tsc --noEmit -p tsconfig.json`** — must be clean. If any file *outside* `column-types/` errors, that file has a hardcoded type list; fix it to read the registry. +2. **Grep for leaks** — `grep -rnE "(===|!==) '{id}'|case '{id}':" apps/sim --include='*.ts' --include='*.tsx' | grep -v column-types/`. (All three forms: a plain `!==` and a `case` are how half of `currency`'s real branches are written.) Hits are expected; judge each. A hit is fine when it mounts a specific React component or encodes a genuinely one-off behavior (`json`'s mono textarea, `date`'s timezone-aware parsing). A hit is a **leak** when it restates something the registry could answer — an icon, a label, a colour, an operator set, a cast, a coercion. Leaks get a registry field, not a new branch. +3. **Run the suite** — `bunx vitest run lib/table 'app/workspace/[workspaceId]/tables' lib/api app/api/table app/api/v1 lib/copilot/tools/server/table`. Existing tests must pass **unchanged**; needing to edit one means you changed behavior for the other types. +4. **`bun run lint:check`, `bun run check:api-validation`, `bun run check:client-boundary`** from the repo root. +5. **Exercise it in the running app** on a table with one column of every type: create, edit inline / in the expanded popover / in the row modal, paste from a spreadsheet, filter, sort, convert to and from other types, export CSV, undo a column delete. diff --git a/.claude/skills/add-column-type b/.claude/skills/add-column-type new file mode 120000 index 00000000000..9d7b29aa20a --- /dev/null +++ b/.claude/skills/add-column-type @@ -0,0 +1 @@ +../../.agents/skills/add-column-type \ No newline at end of file diff --git a/.cursor/commands/add-column-type.md b/.cursor/commands/add-column-type.md new file mode 100644 index 00000000000..f0be823ab6e --- /dev/null +++ b/.cursor/commands/add-column-type.md @@ -0,0 +1,154 @@ +# Adding a Table Column Type + +A column type is **one file** in `apps/sim/lib/table/column-types/` plus a registry entry. Everything that varies per type — label, icon, storage cast, coercion, validation, conversion compatibility, formatting, editor, filter operators — lives on that one object, so no consumer needs editing. + +This was not always true: adding `currency` originally took ~40 edits across 32 `switch` arms and 26 UI branches, each of which failed **silently** when missed. The registry exists to make that impossible, so the rule is absolute: **if you find yourself adding a `case 'yourtype':` anywhere outside `column-types/`, the registry is missing a field. Add the field instead.** + +## Hard Rule: the compiler tells you what to do + +Do **not** hunt for places to edit. Add your type to the `ColumnType` union first and let `tsc` produce the list: + +```bash +cd apps/sim && bunx tsc --noEmit -p tsconfig.json +``` + +You will get two errors, naming `column-types/registry.ts` and `column-types/registry.server.ts`. Register in both. + +If your type owns metadata, adding its key to `TYPE_SPECIFIC_COLUMN_KEYS` produces two more legitimate errors — `FOREIGN_METADATA_VERB` in `validation.ts` (a `Record` over those keys) and the key's absence from `ColumnDefinition`. Those are the gate working, not sites to "fix". + +Any error beyond those four is a site reading a hardcoded type list that should read the registry — fix that site, don't work around it. + +## Directory Structure + +``` +apps/sim/lib/table/column-types/ +├── types.ts # ColumnTypeDefinition — the contract you implement +├── types.server.ts # ColumnTypeServerDefinition — cell migrations only +├── registry.ts # Record ← client-safe, the gate +├── registry.server.ts # Record ← adds migrations (drizzle) +├── index.ts # barrel + accessors (columnTypeOf, columnTypeById, …) +└── {type}.ts # one file per type — what you write +``` + +## Step 1: Pick the storage shape + +Decide what a cell literally holds in `user_table_rows.data` (JSONB). This drives almost everything else: + +| Storage | `jsonbCast` | Notes | +|---------|-------------|-------| +| number | `'numeric'` | Filters/sorts compare numerically. `currency` does this. | +| ISO string | `'timestamptz'` | `date` does this. | +| string / bool / object | `null` | Text comparison is correct. | + +**Prefer an existing primitive over a new shape.** `currency` stores a plain number and keeps its ISO code as *display metadata* — which is why filtering, sorting, uniqueness, and CSV export all reuse the numeric paths untouched, and why re-denominating a column rewrites zero rows. + +## Step 2: Add the icon + +Create `packages/emcn/src/icons/type-{name}.tsx`, copying the geometry conventions of its siblings exactly: + +```tsx +import type { SVGProps } from 'react' + +/** + * Type {name} icon component - {what the glyph is} for {name} columns + * @param props - SVG properties including className, fill, etc. + */ +export function Type{Pascal}(props: SVGProps) { + return ( + + ) +} +``` + +- `viewBox='-1.75 -1.5 24 24'` is the **`type-*` family** value, not the set-wide default. Match the family. +- Center the glyph on the viewBox's optical center (**y = 10.5**, **x = 10.25**) — every sibling does, and a few tenths off is visible at `size-[14px]`. +- Export alphabetically **by component name** in `packages/emcn/src/icons/index.ts`. + +## Step 3: Write the type file + +`apps/sim/lib/table/column-types/{name}.ts`. Copy the closest existing type and change what differs. Every field is required by the interface, so the compiler enumerates them for you — read the TSDoc in `types.ts` rather than guessing. + +The three that are easy to get wrong: + +- **`coerce`** is the *single* write-path implementation. The server runs it before persisting **and** the grid runs it to fill the optimistic cache. Accept every shape the value legitimately arrives in (paste, CSV, tool write), because rejecting means the cell is nulled. +- **`isCompatibleWith`** gates type conversion and must read the value **exactly as `coerce` will**, or a conversion will pass its check and then null the cell. +- **`ownedMetadata`** lists the `ColumnDefinition` keys your type owns. Anything you add must also be added to `TYPE_SPECIFIC_COLUMN_KEYS` in `types.ts` and given a phrase in `FOREIGN_METADATA_VERB` in `validation.ts` — both are `Record`-typed, so the compiler will tell you. + +## Step 4: Register + +Add the entry to `COLUMN_TYPE_REGISTRY` in `registry.ts` **and** `COLUMN_TYPE_SERVER_REGISTRY` in `registry.server.ts`. + +`COLUMN_TYPES` is declared in `types.ts` (not derived from the registry — the registry is annotated `Record` against it, which is the gate). `constants.ts` re-exports it, so `columnTypeSchema = z.enum(COLUMN_TYPES)` picks your type up with no edit. **Type-specific metadata does not** — see the next step. + +## Step 5: Migrations (only if the stored bytes change) + +If converting an existing column **to** your type must rewrite cells, add `migrateCellsTo` in `registry.server.ts`; if converting **away** must rewrite them, add `migrateCellsFrom`. + +This is load-bearing, not cosmetic: filters and sorts apply `jsonbCast` to whatever is stored, so leaving a non-castable string behind makes **every query on that column fail** — not merely render oddly. + +Prefer set-based SQL. When the transform genuinely needs JS (`currency`'s separator disambiguation), compute the values during the compatibility scan and pass them through `resolved`, then apply them in one batched statement. + +## Naming Convention + +- Type id: lowercase, singular — `currency`, not `Currency` or `currencies` +- File: `column-types/{id}.ts`, export `const {id}ColumnType` +- Icon: `type-{kebab}.tsx`, export `Type{Pascal}` + +## Watch out + +- **Import cycles.** `column-types/select.ts` imports `select-values.ts`, so `select-values.ts` must **not** import the registry — that closes a cycle and fails at module init. Inside a type's own helper module the string literal is the implementation, not a config leak. +- **The client-safe boundary.** `registry.ts` and everything it imports must stay free of `@sim/db`, `drizzle-orm`, and `next/server` — the tables grid imports it directly. A React icon is fine (it's a component *reference*, never called server-side). Only `registry.server.ts` may touch drizzle. +- **Don't re-export the registry from `@/lib/table`.** 44 server modules import that barrel; routing this through it pulls `@sim/emcn/icons` into all of them. Deep-import `@/lib/table/column-types`. +- **`import.ts`'s `coerceValue` is a SECOND write path and is not opt-in.** Importing into a column of your type always hits it, and its `default` arm silently `String(value)`s — so a missing `case` stores text in a column whose `jsonbCast` is numeric, and then every filter and sort on that column errors in Postgres. Add a `case`, even though the switch compiles without one. (It is deliberately separate from the registry's `coerce`: an import wants an unparseable value to survive as its raw string so the row error can name it.) +- **CSV inference** is an ordered heuristic in `import.ts`, deliberately not registry-driven. A new type is not inferred from a CSV unless you extend `inferColumnType` — usually you should not, since inference cannot supply configuration (an option set, a currency code). + +## If your type owns metadata, read this + +Registering the *type* is compiler-enforced. Registering its *metadata* is not, and that is where the remaining manual work lives. A key like `precision` has to be added in each of these, none of which will fail to compile if you forget: + +| Where | What happens if you forget | +|---|---| +| `lib/table/types.ts` `ColumnDefinition` | (this one DOES fail — the ownership loop indexes it) | +| `column-types/types.ts` `TYPE_SPECIFIC_COLUMN_KEYS` | it is never stripped on conversion, and poisons the target type | +| `lib/api/contracts/tables.ts` — the schema slot in all three column schemas, plus `refineColumnOptions` | zod strips it at the boundary; silently never saved | +| `columns/service.ts` `addTableColumn` param type | callers cannot pass it | +| A metadata-only update path (`updateColumnCurrency` is the model) + a branch in both column routes + the copilot tool | changing it on an existing column is a silent 200 no-op | +| `column-config-sidebar.tsx` | no UI to set it | +| `table-grid.tsx` delete-column undo + `use-table-undo.ts` restore | undo silently resets it to the default | + +`normalizeColumn`, `buildConvertedColumn`, and the undo snapshot read `TYPE_SPECIFIC_COLUMN_KEYS` generically, so those three are already zero-edit. + +**Known gap:** the metadata-only update path is ~6 near-identical copies (service + 2 routes + copilot). A `metadataUpdate` descriptor on `ColumnTypeServerDefinition` would collapse them; until that exists, copy `currency`'s. + +## Checklist Before Finishing + +- [ ] Added to the `ColumnType` union in `column-types/types.ts` +- [ ] `column-types/{id}.ts` created, every interface field filled in +- [ ] Registered in **both** `registry.ts` and `registry.server.ts` +- [ ] Icon added, centered on the family's optical center, exported alphabetically +- [ ] `migrateCellsTo` / `migrateCellsFrom` added if the stored bytes change +- [ ] New metadata keys added to `TYPE_SPECIFIC_COLUMN_KEYS` + `FOREIGN_METADATA_VERB` +- [ ] Unit tests for `coerce` / `isCompatibleWith` round-trips, verified to fail without the code +- [ ] Docs row added to `apps/docs/content/docs/en/tables/index.mdx` + +## Final Validation (Required) + +1. **`cd apps/sim && bunx tsc --noEmit -p tsconfig.json`** — must be clean. If any file *outside* `column-types/` errors, that file has a hardcoded type list; fix it to read the registry. +2. **Grep for leaks** — `grep -rnE "(===|!==) '{id}'|case '{id}':" apps/sim --include='*.ts' --include='*.tsx' | grep -v column-types/`. (All three forms: a plain `!==` and a `case` are how half of `currency`'s real branches are written.) Hits are expected; judge each. A hit is fine when it mounts a specific React component or encodes a genuinely one-off behavior (`json`'s mono textarea, `date`'s timezone-aware parsing). A hit is a **leak** when it restates something the registry could answer — an icon, a label, a colour, an operator set, a cast, a coercion. Leaks get a registry field, not a new branch. +3. **Run the suite** — `bunx vitest run lib/table 'app/workspace/[workspaceId]/tables' lib/api app/api/table app/api/v1 lib/copilot/tools/server/table`. Existing tests must pass **unchanged**; needing to edit one means you changed behavior for the other types. +4. **`bun run lint:check`, `bun run check:api-validation`, `bun run check:client-boundary`** from the repo root. +5. **Exercise it in the running app** on a table with one column of every type: create, edit inline / in the expanded popover / in the row modal, paste from a spreadsheet, filter, sort, convert to and from other types, export CSV, undo a column delete. diff --git a/AGENTS.md b/AGENTS.md index 9ce16b909d9..f36d633df61 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -468,3 +468,9 @@ Two hard rules that the skills assume: For the full authoring instructions — SubBlock property tables, `condition`/`dependsOn`/`required`/`mode`/`canonicalParamId` syntax, required block metadata (`integrationType`, `tags`, `authMode`, `docsLink`, `{Service}BlockMeta`), file-input/`normalizeFileInput` patterns, and checklists — use the skills: `/add-integration` (end-to-end), `/add-tools`, `/add-block`, `/add-trigger`. +## Tables + +Table column types are registry entries in `apps/sim/lib/table/column-types/` — one file per type owning its label, icon, storage cast, coercion, validation, conversion compatibility, formatting, and editor. `Record` on `registry.ts` and `registry.server.ts` is a compile-time completeness gate: adding a type to the union errors until both entries exist. + +Never add a `case 'sometype':` outside `column-types/` — a missing arm fails silently (a wrong `jsonbCast` breaks every filter on the column). If a consumer needs per-type knowledge, add a registry field. Use `/add-column-type` for the full procedure. + diff --git a/CLAUDE.md b/CLAUDE.md index a0632e40ad7..d2e376dbd9f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -478,3 +478,9 @@ Two hard rules that the skills assume: For the full authoring instructions — SubBlock property tables, `condition`/`dependsOn`/`required`/`mode`/`canonicalParamId` syntax, required block metadata (`integrationType`, `tags`, `authMode`, `docsLink`, `{Service}BlockMeta`), file-input/`normalizeFileInput` patterns, and checklists — use the skills: `/add-integration` (end-to-end), `/add-tools`, `/add-block`, `/add-trigger`. +## Tables + +Table column types are registry entries in `apps/sim/lib/table/column-types/` — one file per type owning its label, icon, storage cast, coercion, validation, conversion compatibility, formatting, and editor. `Record` on `registry.ts` and `registry.server.ts` is a compile-time completeness gate: adding a type to the union errors until both entries exist. + +Never add a `case 'sometype':` outside `column-types/` — a missing arm fails silently (a wrong `jsonbCast` breaks every filter on the column). If a consumer needs per-type knowledge, add a registry field. Use `/add-column-type` for the full procedure. + diff --git a/apps/docs/content/docs/en/integrations/table.mdx b/apps/docs/content/docs/en/integrations/table.mdx index df92f381e0e..4c26c6fa714 100644 --- a/apps/docs/content/docs/en/integrations/table.mdx +++ b/apps/docs/content/docs/en/integrations/table.mdx @@ -16,7 +16,7 @@ Tables allow you to create and manage custom data tables directly within Sim. St **Why Use Tables?** - **No external setup**: Create tables instantly without configuring external databases - **Workflow-native**: Data persists across workflow executions and is accessible from any workflow in your workspace -- **Flexible schema**: Define columns with types (string, number, boolean, date, json) and constraints (required, unique) +- **Flexible schema**: Define columns with types (string, number, currency, boolean, date, json, select) and constraints (required, unique) - **Powerful querying**: Filter, sort, and paginate data using MongoDB-style operators - **Agent-friendly**: Tables can be used as tools by AI agents for dynamic data storage and retrieval diff --git a/apps/docs/content/docs/en/tables/index.mdx b/apps/docs/content/docs/en/tables/index.mdx index ae70683af65..913711ecdfa 100644 --- a/apps/docs/content/docs/en/tables/index.mdx +++ b/apps/docs/content/docs/en/tables/index.mdx @@ -22,12 +22,16 @@ Every column has a type, which decides how its values are stored and validated. | --- | --- | --- | | **Text** | A free-form string | `"Acme Corp"` | | **Number** | A numeric value | `42` | +| **Currency** | An amount in a currency you pick per column | `$1,234.56` | | **Boolean** | `true` or `false` | `true` | | **Date** | A date | `2026-03-16` | | **JSON** | An object or array | `{ "tier": "pro" }` | +| **Select** | One of a fixed set of options, or several | `Pro` | Types are enforced as you enter values, so a Number column only takes numbers. +A Currency column stores a plain number and renders it in the currency you choose for that column, so filters, sorts, and exports all see the amount itself. Changing a column's currency relabels it — it does not convert the amounts. + ## Editing a table Open the **Tables** section in the sidebar and click **New table** to create one. Add columns from the column header, type into a cell to edit it, and paste rows from a spreadsheet to bulk-load. Filter and sort from the toolbar without changing the underlying data. The editor has full keyboard support; see [keyboard shortcuts](/keyboard-shortcuts). diff --git a/apps/sim/app/api/table/[tableId]/columns/route.test.ts b/apps/sim/app/api/table/[tableId]/columns/route.test.ts new file mode 100644 index 00000000000..4ac282861cd --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/columns/route.test.ts @@ -0,0 +1,320 @@ +/** + * @vitest-environment node + * + * The PATCH handler performs several writes, each in its own locked + * transaction, so one that fails leaves the earlier ones committed. Two things + * keep that from producing a partial update the caller cannot see or undo: the + * guards reject the knowable cases before any write, and the rename — the only + * write that is purely cosmetic — goes LAST, so a failed typed write leaves the + * column entirely untouched. These pin both. + */ +import { hybridAuthMockFns } from '@sim/testing' +import { getErrorMessage } from '@sim/utils/errors' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCheckAccess, + mockRenameColumn, + mockUpdateColumnType, + mockUpdateColumnCurrency, + mockUpdateColumnOptions, + mockUpdateColumnConstraints, + mockAddTableColumn, + mockDeleteColumn, +} = vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockRenameColumn: vi.fn(), + mockUpdateColumnType: vi.fn(), + mockUpdateColumnCurrency: vi.fn(), + mockUpdateColumnOptions: vi.fn(), + mockUpdateColumnConstraints: vi.fn(), + mockAddTableColumn: vi.fn(), + mockDeleteColumn: vi.fn(), +})) + +vi.mock('@/lib/table', () => ({ + addTableColumn: mockAddTableColumn, + deleteColumn: mockDeleteColumn, + renameColumn: mockRenameColumn, + updateColumnConstraints: mockUpdateColumnConstraints, + updateColumnCurrency: mockUpdateColumnCurrency, + updateColumnOptions: mockUpdateColumnOptions, + updateColumnType: mockUpdateColumnType, +})) +vi.mock('@/app/api/table/utils', () => ({ + accessError: () => new Response('denied', { status: 403 }), + checkAccess: mockCheckAccess, + normalizeColumn: (c: unknown) => c, + rootErrorMessage: (e: unknown) => getErrorMessage(e), + tableLockErrorResponse: () => null, +})) + +import { PATCH } from '@/app/api/table/[tableId]/columns/route' + +const WORKSPACE_ID = '11111111-1111-4111-8111-111111111111' + +function patch(updates: Record) { + return PATCH( + new NextRequest('http://localhost/api/table/t1/columns', { + method: 'PATCH', + body: JSON.stringify({ workspaceId: WORKSPACE_ID, columnName: 'amount', updates }), + headers: { 'content-type': 'application/json' }, + }), + { params: Promise.resolve({ tableId: 't1' }) } + ) +} + +describe('PATCH /api/table/[tableId]/columns — pre-flight guards', () => { + beforeEach(() => { + vi.clearAllMocks() + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'session', + }) + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { columns: [{ id: 'col_a', name: 'amount', type: 'number' }] }, + }, + }) + mockRenameColumn.mockResolvedValue({ schema: { columns: [] } }) + }) + + it('rejects a currency code on a non-currency column without renaming first', async () => { + const response = await patch({ name: 'renamed', currencyCode: 'USD' }) + + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('Cannot set currency'), + }) + // The whole point: the rename must not have been committed. + expect(mockRenameColumn).not.toHaveBeenCalled() + expect(mockUpdateColumnCurrency).not.toHaveBeenCalled() + }) + + it('rejects an unsupported currency code without renaming first', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { columns: [{ id: 'col_a', name: 'amount', type: 'currency' }] }, + }, + }) + + const response = await patch({ name: 'renamed', currencyCode: 'ZZZ' }) + + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('Invalid currency code'), + }) + expect(mockRenameColumn).not.toHaveBeenCalled() + }) + + it('still applies a rename when the currency it rides on is unchanged', async () => { + // `updateColumnCurrency` no-ops on an unchanged code. The rename folded into + // the same request must not be dropped with it. + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { + columns: [{ id: 'col_a', name: 'amount', type: 'currency', currencyCode: 'USD' }], + }, + }, + }) + mockUpdateColumnCurrency.mockResolvedValue({ schema: { columns: [] } }) + + const response = await patch({ name: 'renamed', currencyCode: 'USD' }) + + expect(response.status).toBe(200) + expect(mockUpdateColumnCurrency).toHaveBeenCalledWith( + expect.objectContaining({ currencyCode: 'USD', newName: 'renamed' }), + expect.any(String) + ) + }) + + it('renames standalone when there is no other write to ride on', async () => { + mockRenameColumn.mockResolvedValue({ schema: { columns: [] } }) + + const response = await patch({ name: 'renamed' }) + + expect(response.status).toBe(200) + expect(mockRenameColumn).toHaveBeenCalledWith( + expect.objectContaining({ oldName: 'col_a', newName: 'renamed' }), + expect.any(String) + ) + }) + + it('leaves the column untouched when a typed write fails', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { columns: [{ id: 'col_a', name: 'amount', type: 'currency' }] }, + }, + }) + // Stands in for the race the guards cannot close: the column stopped being + // a currency between the snapshot the guards read and this write. + mockUpdateColumnCurrency.mockRejectedValue( + new Error('Cannot set currency on column "amount" of type "string"') + ) + + const response = await patch({ name: 'renamed', currencyCode: 'USD' }) + + expect(response.status).toBe(400) + expect(mockRenameColumn).not.toHaveBeenCalled() + }) + + it('rejects a name already taken before any write runs', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { + columns: [ + { id: 'col_a', name: 'amount', type: 'currency' }, + { id: 'col_b', name: 'taken', type: 'string' }, + ], + }, + }, + }) + + const response = await patch({ name: 'taken', currencyCode: 'EUR' }) + + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('already exists'), + }) + // The typed write would otherwise have committed under a rename that fails. + expect(mockUpdateColumnCurrency).not.toHaveBeenCalled() + expect(mockRenameColumn).not.toHaveBeenCalled() + }) + + it('forwards unique to the retype so it validates post-conversion values', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { columns: [{ id: 'col_a', name: 'amount', type: 'string' }] }, + }, + }) + mockUpdateColumnType.mockResolvedValue({ schema: { columns: [] } }) + mockUpdateColumnConstraints.mockResolvedValue({ schema: { columns: [] } }) + + const response = await patch({ type: 'number', unique: true }) + + expect(response.status).toBe(200) + // The conversion itself can manufacture duplicates ("5" and "5.0" both + // coerce to 5), so the retype has to see `unique` — discovering it in the + // separate constraint write would report an error with the conversion + // already committed and the original text irrecoverably rewritten. + expect(mockUpdateColumnType).toHaveBeenCalledWith( + expect.objectContaining({ newType: 'number', unique: true }), + expect.any(String) + ) + }) + + it('applies a rename, retype and constraints in a single write', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { columns: [{ id: 'col_a', name: 'amount', type: 'string' }] }, + }, + }) + mockUpdateColumnType.mockResolvedValue({ schema: { columns: [] } }) + + const response = await patch({ name: 'total', type: 'number', required: true, unique: true }) + + expect(response.status).toBe(200) + // One transaction for the whole request: no separate rename, no separate + // constraint write, so no half of it can commit without the others. + expect(mockRenameColumn).not.toHaveBeenCalled() + expect(mockUpdateColumnConstraints).not.toHaveBeenCalled() + expect(mockUpdateColumnType).toHaveBeenCalledTimes(1) + expect(mockUpdateColumnType).toHaveBeenCalledWith( + expect.objectContaining({ + newType: 'number', + required: true, + unique: true, + newName: 'total', + }), + expect.any(String) + ) + }) + + it('still runs the constraint write when the type is unchanged', async () => { + mockUpdateColumnConstraints.mockResolvedValue({ schema: { columns: [] } }) + + const response = await patch({ required: true }) + + expect(response.status).toBe(200) + expect(mockUpdateColumnConstraints).toHaveBeenCalledTimes(1) + expect(mockUpdateColumnType).not.toHaveBeenCalled() + }) + + it('rejects constraint changes on a workflow-output column before any write', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { + columns: [{ id: 'col_a', name: 'amount', type: 'number', workflowGroupId: 'g1' }], + }, + }, + }) + + const response = await patch({ type: 'string', required: true }) + + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: expect.stringContaining('workflow-output column'), + }) + expect(mockUpdateColumnType).not.toHaveBeenCalled() + }) + + it('rejects unique on a type that cannot carry it without renaming first', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { + columns: [ + { id: 'col_a', name: 'amount', type: 'select', options: [{ id: 'o', name: 'O' }] }, + ], + }, + }, + }) + + const response = await patch({ name: 'renamed', unique: true }) + + expect(response.status).toBe(400) + expect(mockRenameColumn).not.toHaveBeenCalled() + }) + + it('folds a rename into the typed write instead of running it separately', async () => { + mockCheckAccess.mockResolvedValue({ + ok: true, + table: { + workspaceId: WORKSPACE_ID, + schema: { columns: [{ id: 'col_a', name: 'amount', type: 'currency' }] }, + }, + }) + mockUpdateColumnCurrency.mockResolvedValue({ schema: { columns: [] } }) + + const response = await patch({ name: 'renamed', currencyCode: 'eur' }) + + expect(response.status).toBe(200) + // One transaction, not two: the rename rides along with the currency write, + // so neither half can commit without the other. + expect(mockRenameColumn).not.toHaveBeenCalled() + expect(mockUpdateColumnCurrency).toHaveBeenCalledWith( + // Addressed by stable id; the contract upper-cases the code on the way in. + expect.objectContaining({ columnName: 'col_a', currencyCode: 'EUR', newName: 'renamed' }), + expect.any(String) + ) + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/columns/route.ts b/apps/sim/app/api/table/[tableId]/columns/route.ts index 227c1422b04..19289a0198c 100644 --- a/apps/sim/app/api/table/[tableId]/columns/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/route.ts @@ -15,10 +15,13 @@ import { deleteColumn, renameColumn, updateColumnConstraints, + updateColumnCurrency, updateColumnOptions, updateColumnType, } from '@/lib/table' -import { columnMatchesRef } from '@/lib/table/column-keys' +import { columnMatchesRef, getColumnId } from '@/lib/table/column-keys' +import { columnTypeById } from '@/lib/table/column-types' +import { isSupportedCurrencyCode } from '@/lib/table/currency' import { accessError, checkAccess, @@ -120,13 +123,6 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu const { updates } = validated let updatedTable = null - if (updates.name) { - updatedTable = await renameColumn( - { tableId, oldName: validated.columnName, newName: updates.name }, - requestId - ) - } - // A payload that repeats the current type must not go through // `updateColumnType` — it early-returns on an unchanged type and would drop // any `options` alongside it. Only a real type change routes there; an @@ -134,29 +130,121 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu const currentColumn = table.schema.columns.find((c) => columnMatchesRef(c, validated.columnName) ) + // Address every write below by the stable id, not the name: a rename folded + // into one of them must not break the next one's lookup. + const columnRef = currentColumn ? getColumnId(currentColumn) : validated.columnName + // The constraints write below is a separate, unconditional step, so it is + // the last one whenever it runs — that is the write the rename rides on. const typeChanging = updates.type !== undefined && updates.type !== currentColumn?.type + if (!currentColumn) { + return NextResponse.json( + { error: `Column "${validated.columnName}" not found` }, + { status: 404 } + ) + } - // Every write below is its own locked transaction, so any of them paired - // with a constraint write that is going to fail commits and then errors. + // A retype applies and validates the constraints itself, so the separate + // constraint write only runs when the type is unchanged. The rename rides + // whichever write actually runs last. + const typedWriteRuns = + typeChanging || + updates.currencyCode !== undefined || + updates.options !== undefined || + updates.multiple !== undefined + const constraintsWriteRuns = + !typedWriteRuns && (updates.required !== undefined || updates.unique !== undefined) + const renameWithTypedWrite = + updates.name && !constraintsWriteRuns ? { newName: updates.name } : {} + + // Every write below is its own locked transaction, so one that is going to + // fail leaves the earlier ones committed. These guards reject the knowable + // cases up front, before any write at all. // Gate on the type the column ENDS UP with, not on whether the type is // changing: an options-only update on an existing select column carries the // same hazard as a conversion does. const resultingType = updates.type ?? currentColumn?.type - if (updates.unique === true && resultingType === 'select') { - return NextResponse.json({ error: 'Cannot set a select column as unique' }, { status: 400 }) + if (updates.currencyCode !== undefined) { + if (resultingType !== 'currency') { + return NextResponse.json( + { + error: `Cannot set currency on column "${validated.columnName}" of type "${resultingType}"`, + }, + { status: 400 } + ) + } + if (!isSupportedCurrencyCode(updates.currencyCode)) { + return NextResponse.json( + { + error: `Invalid currency code "${updates.currencyCode}". Use an ISO 4217 code, e.g. USD`, + }, + { status: 400 } + ) + } + } + // The rename runs last (see below), so a name already taken would fail after + // the typed write committed. This is the only rename failure a caller can + // cause; catching it here leaves just the concurrent-collision race, which + // no pre-flight check can close. + if ( + updates.name && + table.schema.columns.some( + (c) => + c.name.toLowerCase() === updates.name?.toLowerCase() && + !columnMatchesRef(c, validated.columnName) + ) + ) { + return NextResponse.json( + { error: `Column "${updates.name}" already exists` }, + { status: 400 } + ) + } + if ( + currentColumn?.workflowGroupId && + (updates.required !== undefined || updates.unique !== undefined) + ) { + return NextResponse.json( + { + error: `Cannot change constraints on workflow-output column "${currentColumn.name}". Constraints aren't applicable to columns whose values come from workflow execution.`, + }, + { status: 400 } + ) + } + if (updates.unique === true && !columnTypeById(resultingType).supportsUnique) { + return NextResponse.json( + { error: `Cannot set a ${resultingType} column as unique` }, + { status: 400 } + ) } if (typeChanging) { updatedTable = await updateColumnType( { tableId, - columnName: updates.name ?? validated.columnName, + columnName: columnRef, newType: updates.type as NonNullable, ...(updates.options !== undefined ? { options: updates.options } : {}), ...(updates.multiple !== undefined ? { multiple: updates.multiple } : {}), + ...(updates.currencyCode !== undefined ? { currencyCode: updates.currencyCode } : {}), // Forwarded so the conversion validates against the constraint this // same request is about to set, not the column's current one. ...(updates.required !== undefined ? { required: updates.required } : {}), + ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...renameWithTypedWrite, + }, + requestId + ) + } else if (updates.currencyCode !== undefined) { + // Re-denominating an existing currency column: schema-only, no cell + // rewrite. Reached only when the type is unchanged — a conversion INTO + // currency carries the code through `updateColumnType` above. + updatedTable = await updateColumnCurrency( + { + tableId, + columnName: columnRef, + currencyCode: updates.currencyCode, + ...(updates.required !== undefined ? { required: updates.required } : {}), + ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...renameWithTypedWrite, }, requestId ) @@ -164,29 +252,46 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu updatedTable = await updateColumnOptions( { tableId, - columnName: updates.name ?? validated.columnName, + columnName: columnRef, options: updates.options ?? currentColumn?.options ?? [], ...(updates.multiple !== undefined ? { multiple: updates.multiple } : {}), // Forwarded so the removal guard validates against the constraint this // same request is about to set, not the column's current one. ...(updates.required !== undefined ? { required: updates.required } : {}), + ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...renameWithTypedWrite, }, requestId ) } - if (updates.required !== undefined || updates.unique !== undefined) { + // Skipped whenever a typed write ran: that write already applied and + // validated these, in one transaction with the change they accompany. + if (constraintsWriteRuns) { updatedTable = await updateColumnConstraints( { tableId, - columnName: updates.name ?? validated.columnName, + columnName: columnRef, ...(updates.required !== undefined ? { required: updates.required } : {}), ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...(updates.name ? { newName: updates.name } : {}), }, requestId ) } + // A rename rides along with the LAST write above, inside that write's + // transaction — a rename is metadata-only (rows key on the stable column + // id), so nothing forces it to be its own write, and folding it in is what + // stops a combined request from committing one half and then failing. Only + // a rename with nothing to ride on runs standalone. + if (updates.name && !updatedTable) { + updatedTable = await renameColumn( + { tableId, oldName: columnRef, newName: updates.name }, + requestId + ) + } + if (!updatedTable) { return NextResponse.json({ error: 'No updates specified' }, { status: 400 }) } @@ -217,7 +322,8 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu msg.includes('exceeds maximum') || msg.includes('incompatible') || msg.includes('duplicate') || - msg.includes('option') + msg.includes('option') || + msg.includes('currency') ) { return NextResponse.json({ error: msg }, { status: 400 }) } diff --git a/apps/sim/app/api/table/utils.ts b/apps/sim/app/api/table/utils.ts index 835866d829a..ceb399556c4 100644 --- a/apps/sim/app/api/table/utils.ts +++ b/apps/sim/app/api/table/utils.ts @@ -11,6 +11,7 @@ import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import type { MultipartError } from '@/lib/core/utils/multipart' import type { ColumnDefinition, Filter, TableDefinition, TablePredicate } from '@/lib/table' import { buildFilterClause, getTableById, TableQueryValidationError } from '@/lib/table' +import { typeMetadataOf } from '@/lib/table/column-types' import { USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableLockedError } from '@/lib/table/mutation-locks' import { isTablePredicate } from '@/lib/table/query-builder/converters' @@ -339,7 +340,8 @@ export function normalizeColumn(col: ColumnDefinition): ColumnDefinition { required: col.required ?? false, unique: col.unique ?? false, ...(col.workflowGroupId ? { workflowGroupId: col.workflowGroupId } : {}), - ...(col.options ? { options: col.options } : {}), - ...(col.multiple ? { multiple: true } : {}), + // Type-specific metadata is forwarded generically: naming keys here meant a + // new type's metadata was stored server-side but silently never returned. + ...typeMetadataOf(col), } } diff --git a/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts b/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts index f1751ee2120..56caed8353e 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/columns/route.ts @@ -14,10 +14,13 @@ import { deleteColumn, renameColumn, updateColumnConstraints, + updateColumnCurrency, updateColumnOptions, updateColumnType, } from '@/lib/table' -import { columnMatchesRef } from '@/lib/table/column-keys' +import { columnMatchesRef, getColumnId } from '@/lib/table/column-keys' +import { columnTypeById } from '@/lib/table/column-types' +import { isSupportedCurrencyCode } from '@/lib/table/currency' import { accessError, checkAccess, @@ -154,13 +157,6 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu const { updates } = validated let updatedTable = null - if (updates.name) { - updatedTable = await renameColumn( - { tableId, oldName: validated.columnName, newName: updates.name }, - requestId - ) - } - // A payload that repeats the current type must not go through // `updateColumnType` — it early-returns on an unchanged type and would drop // any `options` alongside it. Only a real type change routes there; an @@ -168,29 +164,121 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu const currentColumn = table.schema.columns.find((c) => columnMatchesRef(c, validated.columnName) ) + // Address every write below by the stable id, not the name: a rename folded + // into one of them must not break the next one's lookup. + const columnRef = currentColumn ? getColumnId(currentColumn) : validated.columnName + // The constraints write below is a separate, unconditional step, so it is + // the last one whenever it runs — that is the write the rename rides on. const typeChanging = updates.type !== undefined && updates.type !== currentColumn?.type + if (!currentColumn) { + return NextResponse.json( + { error: `Column "${validated.columnName}" not found` }, + { status: 404 } + ) + } - // Every write below is its own locked transaction, so any of them paired - // with a constraint write that is going to fail commits and then errors. + // A retype applies and validates the constraints itself, so the separate + // constraint write only runs when the type is unchanged. The rename rides + // whichever write actually runs last. + const typedWriteRuns = + typeChanging || + updates.currencyCode !== undefined || + updates.options !== undefined || + updates.multiple !== undefined + const constraintsWriteRuns = + !typedWriteRuns && (updates.required !== undefined || updates.unique !== undefined) + const renameWithTypedWrite = + updates.name && !constraintsWriteRuns ? { newName: updates.name } : {} + + // Every write below is its own locked transaction, so one that is going to + // fail leaves the earlier ones committed. These guards reject the knowable + // cases up front, before any write at all. // Gate on the type the column ENDS UP with, not on whether the type is // changing: an options-only update on an existing select column carries the // same hazard as a conversion does. const resultingType = updates.type ?? currentColumn?.type - if (updates.unique === true && resultingType === 'select') { - return NextResponse.json({ error: 'Cannot set a select column as unique' }, { status: 400 }) + if (updates.currencyCode !== undefined) { + if (resultingType !== 'currency') { + return NextResponse.json( + { + error: `Cannot set currency on column "${validated.columnName}" of type "${resultingType}"`, + }, + { status: 400 } + ) + } + if (!isSupportedCurrencyCode(updates.currencyCode)) { + return NextResponse.json( + { + error: `Invalid currency code "${updates.currencyCode}". Use an ISO 4217 code, e.g. USD`, + }, + { status: 400 } + ) + } + } + // The rename runs last (see below), so a name already taken would fail after + // the typed write committed. This is the only rename failure a caller can + // cause; catching it here leaves just the concurrent-collision race, which + // no pre-flight check can close. + if ( + updates.name && + table.schema.columns.some( + (c) => + c.name.toLowerCase() === updates.name?.toLowerCase() && + !columnMatchesRef(c, validated.columnName) + ) + ) { + return NextResponse.json( + { error: `Column "${updates.name}" already exists` }, + { status: 400 } + ) + } + if ( + currentColumn?.workflowGroupId && + (updates.required !== undefined || updates.unique !== undefined) + ) { + return NextResponse.json( + { + error: `Cannot change constraints on workflow-output column "${currentColumn.name}". Constraints aren't applicable to columns whose values come from workflow execution.`, + }, + { status: 400 } + ) + } + if (updates.unique === true && !columnTypeById(resultingType).supportsUnique) { + return NextResponse.json( + { error: `Cannot set a ${resultingType} column as unique` }, + { status: 400 } + ) } if (typeChanging) { updatedTable = await updateColumnType( { tableId, - columnName: updates.name ?? validated.columnName, + columnName: columnRef, newType: updates.type as NonNullable, ...(updates.options !== undefined ? { options: updates.options } : {}), ...(updates.multiple !== undefined ? { multiple: updates.multiple } : {}), + ...(updates.currencyCode !== undefined ? { currencyCode: updates.currencyCode } : {}), // Forwarded so the conversion validates against the constraint this // same request is about to set, not the column's current one. ...(updates.required !== undefined ? { required: updates.required } : {}), + ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...renameWithTypedWrite, + }, + requestId + ) + } else if (updates.currencyCode !== undefined) { + // Re-denominating an existing currency column: schema-only, no cell + // rewrite. Reached only when the type is unchanged — a conversion INTO + // currency carries the code through `updateColumnType` above. + updatedTable = await updateColumnCurrency( + { + tableId, + columnName: columnRef, + currencyCode: updates.currencyCode, + ...(updates.required !== undefined ? { required: updates.required } : {}), + ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...renameWithTypedWrite, }, requestId ) @@ -198,29 +286,46 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu updatedTable = await updateColumnOptions( { tableId, - columnName: updates.name ?? validated.columnName, + columnName: columnRef, options: updates.options ?? currentColumn?.options ?? [], ...(updates.multiple !== undefined ? { multiple: updates.multiple } : {}), // Forwarded so the removal guard validates against the constraint this // same request is about to set, not the column's current one. ...(updates.required !== undefined ? { required: updates.required } : {}), + ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...renameWithTypedWrite, }, requestId ) } - if (updates.required !== undefined || updates.unique !== undefined) { + // Skipped whenever a typed write ran: that write already applied and + // validated these, in one transaction with the change they accompany. + if (constraintsWriteRuns) { updatedTable = await updateColumnConstraints( { tableId, - columnName: updates.name ?? validated.columnName, + columnName: columnRef, ...(updates.required !== undefined ? { required: updates.required } : {}), ...(updates.unique !== undefined ? { unique: updates.unique } : {}), + ...(updates.name ? { newName: updates.name } : {}), }, requestId ) } + // A rename rides along with the LAST write above, inside that write's + // transaction — a rename is metadata-only (rows key on the stable column + // id), so nothing forces it to be its own write, and folding it in is what + // stops a combined request from committing one half and then failing. Only + // a rename with nothing to ride on runs standalone. + if (updates.name && !updatedTable) { + updatedTable = await renameColumn( + { tableId, oldName: columnRef, newName: updates.name }, + requestId + ) + } + if (!updatedTable) { return NextResponse.json({ error: 'No updates specified' }, { status: 400 }) } @@ -262,7 +367,8 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Colu msg.includes('exceeds maximum') || msg.includes('incompatible') || msg.includes('duplicate') || - msg.includes('option') + msg.includes('option') || + msg.includes('currency') ) { return NextResponse.json({ error: msg }, { status: 400 }) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx index 2c0a9cffda4..af2a0c797c6 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-config-sidebar.tsx @@ -6,6 +6,11 @@ import { X } from '@sim/emcn/icons' import { toError } from '@sim/utils/errors' import { findValidationIssue, isValidationError } from '@/lib/api/client/errors' import type { ColumnDefinition, SelectOption } from '@/lib/table' +import { + DEFAULT_CURRENCY_CODE, + getCurrencyOptions, + resolveCurrencyCode, +} from '@/lib/table/currency' import { FieldError, RequiredLabel, @@ -19,6 +24,15 @@ function isSelectType(type: ColumnDefinition['type']): boolean { return type === 'select' } +/** + * Picker entries, built once at module load: the option list is derived from the + * runtime's currency data and never varies per column. + */ +const CURRENCY_COMBOBOX_OPTIONS = getCurrencyOptions().map((c) => ({ + value: c.code, + label: `${c.code} · ${c.name}`, +})) + function optionsEqual(a: SelectOption[], b: SelectOption[]): boolean { return JSON.stringify(a) === JSON.stringify(b) } @@ -110,6 +124,11 @@ function ColumnConfigBody({ const [multipleInput, setMultipleInput] = useState(() => config.mode === 'edit' ? !!existingColumn?.multiple : false ) + const [currencyInput, setCurrencyInput] = useState(() => + config.mode === 'edit' + ? resolveCurrencyCode(existingColumn?.currencyCode) + : DEFAULT_CURRENCY_CODE + ) const [showValidation, setShowValidation] = useState(false) const [nameError, setNameError] = useState(null) const [optionsError, setOptionsError] = useState(null) @@ -117,6 +136,7 @@ function ColumnConfigBody({ const saveDisabled = updateColumn.isPending || addColumn.isPending const trimmedName = nameInput.trim() const wantsOptions = isSelectType(typeInput) + const wantsCurrency = typeInput === 'currency' const trimmedOptions = optionsInput.map((o) => ({ ...o, name: o.name.trim() })) /** Client-side option validation mirroring the server rules; returns an error message or null. */ @@ -150,6 +170,7 @@ function ColumnConfigBody({ ...(!wantsOptions && uniqueInput ? { unique: true } : {}), ...(wantsOptions ? { options: trimmedOptions } : {}), ...(wantsOptions && multipleInput ? { multiple: true } : {}), + ...(wantsCurrency ? { currencyCode: currencyInput } : {}), }) toast.success(`Added "${trimmedName}"`) onClose() @@ -168,6 +189,8 @@ function ColumnConfigBody({ const optionsChanged = wantsOptions && !optionsEqual(existingColumn?.options ?? [], trimmedOptions) const multipleChanged = wantsOptions && !!existingColumn?.multiple !== multipleInput + const currencyChanged = + wantsCurrency && resolveCurrencyCode(existingColumn?.currencyCode) !== currencyInput const updates: { name?: string @@ -175,6 +198,7 @@ function ColumnConfigBody({ unique?: boolean options?: SelectOption[] multiple?: boolean + currencyCode?: string } = { ...(renamed ? { name: trimmedName } : {}), ...(typeChanged ? { type: typeInput } : {}), @@ -182,6 +206,9 @@ function ColumnConfigBody({ ...(uniqueCleared ? { unique: false } : {}), ...(wantsOptions && (typeChanged || optionsChanged) ? { options: trimmedOptions } : {}), ...(wantsOptions && (typeChanged || multipleChanged) ? { multiple: multipleInput } : {}), + ...(wantsCurrency && (typeChanged || currencyChanged) + ? { currencyCode: currencyInput } + : {}), } if (Object.keys(updates).length === 0) { onClose() @@ -261,6 +288,24 @@ function ColumnConfigBody({ )} + {wantsCurrency && ( + <> + +
+ Currency + +
+ + )} + {wantsOptions && ( <> diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts index 6ca8023e80b..2f235137f17 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts @@ -1,14 +1,7 @@ import type React from 'react' -import { - Calendar as CalendarIcon, - PlayOutline, - TagIcon, - TypeBoolean, - TypeJson, - TypeNumber, - TypeText, -} from '@sim/emcn/icons' +import { PlayOutline } from '@sim/emcn/icons' import type { ColumnDefinition } from '@/lib/table' +import { ALL_COLUMN_TYPES } from '@/lib/table/column-types' /** * UI-only column type. `'workflow'` is the virtual entry users pick from the @@ -23,13 +16,17 @@ export interface ColumnTypeOption { icon: React.ComponentType<{ className?: string }> } +/** + * Real column types come from the registry — adding one there makes it appear + * in every picker automatically. `workflow` is appended because it is a UI + * affordance, not a storable type. + */ export const COLUMN_TYPE_OPTIONS: ColumnTypeOption[] = [ - { type: 'string', label: 'Text', icon: TypeText }, - { type: 'number', label: 'Number', icon: TypeNumber }, - { type: 'boolean', label: 'Boolean', icon: TypeBoolean }, - { type: 'date', label: 'Date', icon: CalendarIcon }, - { type: 'json', label: 'JSON', icon: TypeJson }, - { type: 'select', label: 'Select', icon: TagIcon }, + ...ALL_COLUMN_TYPES.map((definition) => ({ + type: definition.id, + label: definition.label, + icon: definition.icon, + })), { type: 'workflow', label: 'Workflow', icon: PlayOutline }, ] diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx index efee32d04e8..092e7046e79 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx @@ -18,6 +18,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useParams } from 'next/navigation' import type { ColumnDefinition, TableInfo, TableRow } from '@/lib/table' +import { columnTypeOf } from '@/lib/table/column-types' +import { resolveCurrencyCode } from '@/lib/table/currency' import { useTimezone } from '@/hooks/queries/general-settings' import { useDeleteTableRow, useDeleteTableRows, useUpdateTableRow } from '@/hooks/queries/tables' import { @@ -209,9 +211,16 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { )} ) - const hint = `Type: ${column.type}${column.required ? '' : ' (optional)'}` + // Currency names its code — the modal edits the bare amount, so without it + // there is nothing on screen saying which currency the number is in. + const typeLabel = + column.type === 'currency' + ? `currency (${resolveCurrencyCode(column.currencyCode)})` + : column.type + const hint = `Type: ${typeLabel}${column.required ? '' : ' (optional)'}` + const definition = columnTypeOf(column) - if (column.type === 'boolean') { + if (definition.editor === 'toggle') { return (
@@ -231,6 +240,9 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { ) } + // The one type wanting a mono multi-line field; `editor: 'text'` covers both + // this and a plain input, so it stays explicit rather than inventing a field + // only one type would ever set. if (column.type === 'json') { return ( @@ -276,7 +288,7 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { ) } - if (column.type === 'select') { + if (definition.editor === 'select') { return ( @@ -290,7 +302,12 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { title={title} required={column.required} hint={hint} - inputType={column.type === 'number' ? 'number' : 'text'} + // A native number input rejects the formatted amounts this type's parser + // exists to accept, so those types take a text field — the same shape the + // grid's inline editor uses. + inputType={ + definition.inputMode === 'decimal' && !definition.acceptsFormattedInput ? 'number' : 'text' + } value={formatValueForInput(value, column.type)} onChange={onChange} placeholder={`Enter ${column.name}`} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx index 283dfa21d8c..4e16d03912b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx @@ -6,6 +6,7 @@ import { Badge, Checkbox, cn, Tooltip } from '@sim/emcn' import { parse } from 'tldts' import { faviconUrl } from '@/lib/core/utils/favicon' import type { RowExecutionMetadata, SelectOption } from '@/lib/table' +import { columnTypeOf } from '@/lib/table/column-types' import { StatusBadge } from '@/app/workspace/[workspaceId]/logs/utils' import { storageToDisplay } from '../../../utils' import { resolveSelectOptions, SelectPill } from '../../select-field' @@ -128,6 +129,13 @@ export function resolveCellRender({ return { kind: 'select', options: resolveSelectOptions(column, value) } } if (isNull) return { kind: 'empty' } + // Formatted here rather than in a render branch because the symbol and + // fraction digits come from the COLUMN's currency, which the render switch + // (keyed on kind alone) no longer has. Renders as plain text — a currency + // cell is a number cell with a symbol, so it stays left-aligned like one. + if (column.type === 'currency') { + return { kind: 'text', text: columnTypeOf(column).formatForDisplay(value, column) } + } if (column.type === 'json') return { kind: 'json', text: JSON.stringify(value) } if (column.type === 'date') return { kind: 'date', text: String(value) } if (column.type === 'string') { diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx index 84a61ebab39..9d1d1208e1d 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx @@ -4,6 +4,7 @@ import type React from 'react' import { useEffect, useEffectEvent, useLayoutEffect, useMemo, useRef, useState } from 'react' import { Button } from '@sim/emcn' import type { TableRow as TableRowType } from '@/lib/table' +import { columnTypeOf } from '@/lib/table/column-types' import { useTimezone } from '@/hooks/queries/general-settings' import type { EditingCell, SaveReason } from '../../../types' import { @@ -58,11 +59,11 @@ export function ExpandedCellPopover({ return { row, column, colIndex, value: row.data[column.key] } }, [expandedCell, rows, columns]) - const isBooleanCell = target?.column.type === 'boolean' + const isTogglingCell = target ? columnTypeOf(target.column).editor === 'toggle' : false // Workflow-output cells are editable in the expanded view too — the user // can override the workflow's value. Booleans toggle inline; the expanded // popover only handles text-shaped inputs. - const isEditable = Boolean(target) && canEdit && !isBooleanCell + const isEditable = Boolean(target) && canEdit && !isTogglingCell const displayText = useMemo(() => { if (!target) return '' @@ -74,6 +75,9 @@ export function ExpandedCellPopover({ if (target.column.type === 'date' && typeof value === 'string') { return storageToDisplay(value, { seconds: true }) } + if (target.column.type === 'currency') { + return columnTypeOf(target.column).formatForDisplay(value, target.column) + } if (typeof value === 'string') return value return JSON.stringify(value, null, 2) }, [target]) @@ -231,14 +235,12 @@ function ExpandedCellEditor({ setParseError('Invalid JSON') return } - /** `cleanCellValue` nulls unparseable dates/numbers instead of throwing — reject rather than silently clear. */ - if ( - cleaned === null && - draftValue.trim() !== '' && - (column.type === 'date' || column.type === 'number') - ) { - setParseError(column.type === 'date' ? 'Invalid date' : 'Invalid number') - return + if (cleaned === null && draftValue.trim() !== '') { + const message = columnTypeOf(column).parseErrorMessage + if (message) { + setParseError(message) + return + } } onSave(rowId, column.key, cleaned, 'blur') onClose() diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx index e3caa259fe8..b93cea863d5 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx @@ -15,6 +15,7 @@ import { } from '@sim/emcn' import { Check } from '@sim/emcn/icons' import type { ColumnDefinition } from '@/lib/table' +import { columnTypeOf } from '@/lib/table/column-types' import { isCalendarDateString } from '@/lib/table/dates' import { useTimezone } from '@/hooks/queries/general-settings' import type { SaveReason } from '../../../types' @@ -242,7 +243,7 @@ function InlineDateEditor({ ) } -/** Inline editor for `string`/`number`/`json` columns — single-line text input. Number columns use `type="number"` so the browser rejects non-numeric input. */ +/** Inline editor for `string`/`number`/`currency`/`json` columns — single-line text input. Numeric columns get a decimal keypad and reject a draft that cannot be parsed. */ function InlineTextEditor({ value, column, @@ -291,8 +292,11 @@ function InlineTextEditor({ rejectDraft('Invalid JSON', reason) return } - if (column.type === 'number' && cleaned === null && draft.trim() !== '') { - rejectDraft('Invalid number', reason) + // `cleanCellValue` nulls an unparseable draft rather than throwing; types + // that declare a message reject it instead of silently clearing the cell. + const parseError = columnTypeOf(column).parseErrorMessage + if (cleaned === null && draft.trim() !== '' && parseError) { + rejectDraft(parseError, reason) return } doneRef.current = true @@ -313,13 +317,13 @@ function InlineTextEditor({ } } - const isNumber = column.type === 'number' + const inputMode = columnTypeOf(column).inputMode return ( { setDraft(e.target.value) @@ -425,13 +429,16 @@ function InlineSelectEditor({ value, column, onSave, onCancel }: InlineEditorPro ) } -/** Dispatches to the right editor variant based on the column type. */ +/** Dispatches to the editor variant the column type declares. */ export function InlineEditor(props: InlineEditorProps) { - if (props.column.type === 'date') { - return + switch (columnTypeOf(props.column).editor) { + case 'date': + return + case 'select': + return + // `toggle` types never open an editor — the grid flips them in place — so + // reaching here at all means a text draft is the sane fallback. + default: + return } - if (props.column.type === 'select') { - return - } - return } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-type-icon.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-type-icon.tsx index dfaf662fbf6..5331129a248 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-type-icon.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/column-type-icon.tsx @@ -2,25 +2,17 @@ import type React from 'react' import { Tooltip } from '@sim/emcn' -import { - Calendar as CalendarIcon, - PlayOutline, - TagIcon, - TypeBoolean, - TypeJson, - TypeNumber, - TypeText, - WorkflowX, -} from '@sim/emcn/icons' +import { PlayOutline, WorkflowX } from '@sim/emcn/icons' +import { columnTypeById } from '@/lib/table/column-types' import type { BlockIconInfo } from '../types' -export const COLUMN_TYPE_ICONS: Record = { - string: TypeText, - number: TypeNumber, - boolean: TypeBoolean, - date: CalendarIcon, - json: TypeJson, - select: TagIcon, +/** + * Icon for a column type. Reads the column-type registry rather than restating + * it — this and the type picker's list used to be two hand-maintained copies + * that had to be edited together. Unknown types fall back to the text icon. + */ +export function columnTypeIcon(type: string): React.ComponentType<{ className?: string }> { + return columnTypeById(type).icon } interface ColumnTypeIconProps { @@ -72,6 +64,6 @@ export function ColumnTypeIcon({ const Icon = blockIconInfo?.icon ?? PlayOutline return } - const Icon = COLUMN_TYPE_ICONS[type] ?? TypeText + const Icon = columnTypeIcon(type) return } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/index.ts index 8c8ef9f9dc2..17ba6d7c7d2 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/headers/index.ts @@ -1,3 +1,3 @@ export { ColumnHeaderMenu } from './column-header-menu' -export { COLUMN_TYPE_ICONS, ColumnTypeIcon } from './column-type-icon' +export { ColumnTypeIcon, columnTypeIcon } from './column-type-icon' export { ColumnOptionsMenu, WorkflowGroupMetaCell } from './workflow-group-meta-cell' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index d7bf866e721..e2b8ef12aa7 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -20,6 +20,7 @@ import type { WorkflowGroup, } from '@/lib/table' import { getColumnId } from '@/lib/table/column-keys' +import { columnTypeOf } from '@/lib/table/column-types' import { TABLE_LIMITS } from '@/lib/table/constants' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import type { BlockedTableAction } from '@/app/workspace/[workspaceId]/tables/[tableId]/lock-copy' @@ -43,15 +44,10 @@ import { extractCreatedRowId, useTableUndo } from '@/hooks/use-table-undo' import type { DeletedRowSnapshot } from '@/stores/table/types' import { useContextMenu, useTable } from '../../hooks' import type { EditingCell, QueryOptions, SaveReason } from '../../types' -import { - cleanCellValue, - generateColumnName as sharedGenerateColumnName, - storageToDisplay, -} from '../../utils' +import { cleanCellValue, generateColumnName as sharedGenerateColumnName } from '../../utils' import type { ColumnConfig } from '../column-config-sidebar' import { ContextMenu } from '../context-menu' import { NewColumnDropdown } from '../new-column-dropdown' -import { resolveSelectOptions } from '../select-field' import type { WorkflowConfig } from '../workflow-sidebar' import { ExpandedCellPopover } from './cells' import { ADD_COL_WIDTH, COL_WIDTH, SELECTION_TINT_BG } from './constants' @@ -294,10 +290,10 @@ interface TableGridProps { */ function cellToText(value: unknown, column?: DisplayColumn): string { if (value === null || value === undefined) return '' - if (column?.type === 'select') { - return resolveSelectOptions(column, value) - .map((o) => o.name) - .join(', ') + // Types storing opaque ids copy their labels — the clipboard should hold what + // the user sees, and pasting it back round-trips through the same resolver. + if (column && columnTypeOf(column).storesOpaqueIds) { + return columnTypeOf(column).formatForDisplay(value, column) } return typeof value === 'object' ? JSON.stringify(value) : String(value) } @@ -1106,7 +1102,7 @@ export function TableGrid({ function handleContextMenuEditCell() { if (contextMenu.row && contextMenu.columnName) { const column = columnsRef.current.find((c) => getColumnId(c) === contextMenu.columnName) - if (column?.type === 'boolean') { + if (column && columnTypeOf(column).editor === 'toggle') { toggleBooleanCell( contextMenu.row.id, contextMenu.columnName, @@ -1576,7 +1572,8 @@ export function TableGrid({ if (colIndex === -1) return const column = cols[colIndex] - if (column.type === 'boolean') return + // A toggle renders a fixed-size control, so there is no text to fit to. + if (columnTypeOf(column).editor === 'toggle') return const host = containerRef.current ?? document.body const currentRows = rowsRef.current @@ -1595,27 +1592,18 @@ export function TableGrid({ for (const row of currentRows) { const val = row.data[column.key] if (val == null) continue + // Measure what the cell actually RENDERS, not the stored value — + // otherwise auto-fit sizes a select column to its opaque option ids and + // a currency column to a bare number without its symbol or separators. let text: string - if (column.type === 'json') { - if (typeof val === 'string') { - text = val - } else { - try { - text = JSON.stringify(val) - } catch { - text = String(val) - } + if (column.type === 'json' && typeof val !== 'string') { + try { + text = JSON.stringify(val) + } catch { + text = String(val) } - } else if (column.type === 'date') { - text = storageToDisplay(String(val), { seconds: true }) - } else if (column.type === 'select') { - // Cells store option ids; measure the rendered pill labels instead so - // auto-fit doesn't size the column to opaque ids. - text = resolveSelectOptions(column, val) - .map((o) => o.name) - .join(', ') } else { - text = String(val) + text = columnTypeOf(column).formatForDisplay(val, column) } measure.textContent = text maxWidth = Math.max(maxWidth, measure.getBoundingClientRect().width + 17) @@ -2178,7 +2166,7 @@ export function TableGrid({ const handleCellClick = useCallback( (rowId: string, columnName: string, options?: { toggleBoolean?: boolean }) => { const column = columnsRef.current.find((c) => c.key === columnName) - if (column?.type === 'boolean') { + if (column && columnTypeOf(column).editor === 'toggle') { if (!options?.toggleBoolean || !canEditCellRef.current) return const row = rowsRef.current.find((r) => r.id === rowId) if (row) { @@ -2198,7 +2186,7 @@ export function TableGrid({ const handleCellDoubleClick = useCallback( (rowId: string, columnName: string, columnKey: string) => { const column = columnsRef.current.find((c) => c.key === columnKey) - if (column?.type === 'boolean') return + if (column && columnTypeOf(column).editor === 'toggle') return // Double-click means "edit this cell". On an update-locked table, say so // rather than opening the expanded viewer — which looks like an editor @@ -2213,8 +2201,9 @@ export function TableGrid({ setSelectionFocus(null) setIsColumnSelection(false) - // Date/number: use inline editor (calendar picker / numeric input). - if ((column?.type === 'date' || column?.type === 'number') && canEditCellRef.current) { + // Types with a bounded value edit in place (calendar picker, numeric + // input); only free-form prose opens the big expanded popover. + if (column && !columnTypeOf(column).expandable && canEditCellRef.current) { setEditingCell({ rowId, columnName }) setInitialCharacter(null) return @@ -2470,7 +2459,7 @@ export function TableGrid({ const row = currentRows[anchor.rowIndex] if (!row) return - if (col.type === 'boolean') { + if (columnTypeOf(col).editor === 'toggle') { toggleBooleanCellRef.current(row.id, col.key, row.data[col.key]) return } @@ -2712,9 +2701,12 @@ export function TableGrid({ // Workflow-output cells are editable: the user can override the // workflow's value if they want. Booleans toggle on space/click — // typeahead doesn't apply to them. - if (!col || col.type === 'boolean') return - if (col.type === 'number' && !/[\d.-]/.test(e.key)) return - if (col.type === 'date' && !/[\d\-/]/.test(e.key)) return + if (!col || columnTypeOf(col).editor === 'toggle') return + // Types that parse their input only start an edit on a key they could + // actually accept, so a stray letter doesn't open an editor that can + // never save. + const typeahead = columnTypeOf(col).typeaheadPattern + if (typeahead && !typeahead.test(e.key)) return e.preventDefault() const row = currentRows[anchor.rowIndex] @@ -3472,6 +3464,7 @@ export function TableGrid({ // invalid with no options, and the saved cell data is option ids. ...(entry.def?.options ? { columnOptions: entry.def.options } : {}), ...(entry.def?.multiple ? { columnMultiple: true } : {}), + ...(entry.def?.currencyCode ? { columnCurrencyCode: entry.def.currencyCode } : {}), cellData, previousOrder: orderSnapshot, previousWidth, diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/workflow-sidebar/workflow-sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/workflow-sidebar/workflow-sidebar.tsx index b6702c721f8..d5d67dc2a83 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/workflow-sidebar/workflow-sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/workflow-sidebar/workflow-sidebar.tsx @@ -43,6 +43,7 @@ import type { } from '@/lib/table' import { getColumnId } from '@/lib/table/column-keys' import { columnTypeForLeaf, deriveOutputColumnName } from '@/lib/table/column-naming' +import { columnTypeById } from '@/lib/table/column-types' import { type FlattenOutputsBlockInput, type FlattenOutputsEdgeInput, @@ -161,16 +162,7 @@ interface WorkflowStatePayload { } function tableColumnTypeToInputType(colType: ColumnDefinition['type'] | undefined): string { - switch (colType) { - case 'number': - return 'number' - case 'boolean': - return 'boolean' - case 'json': - return 'object' - default: - return 'string' - } + return columnTypeById(colType).workflowInputType } const TagIcon: React.FC<{ diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 146c02c103a..553d032101f 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -71,7 +71,7 @@ import { WorkflowSidebar, } from './components' import { COLUMN_SIDEBAR_WIDTH } from './components/table-grid/constants' -import { COLUMN_TYPE_ICONS } from './components/table-grid/headers' +import { columnTypeIcon } from './components/table-grid/headers' import { useTable, useTableEventStream } from './hooks' import { type BlockedTableAction, describeBlockedAction, lockedNouns } from './lock-copy' import { @@ -1047,7 +1047,7 @@ export function Table({ id: getColumnId(col), label: col.name, type: col.type, - icon: COLUMN_TYPE_ICONS[col.type], + icon: columnTypeIcon(col.type), })), [columns] ) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts index cbeb8c122d4..d4c3fc5b3ce 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts @@ -1,12 +1,7 @@ -import type { ColumnDefinition } from '@/lib/table' -import { - formatDateCellDisplay, - getWallClockParts, - normalizeDateCellValue, - storedDateToEditable, -} from '@/lib/table/dates' - -type BadgeVariant = 'green' | 'blue' | 'purple' | 'orange' | 'teal' | 'gray' +import type { ColumnDefinition, JsonValue } from '@/lib/table' +import type { ColumnType } from '@/lib/table/column-types' +import { columnTypeById, columnTypeOf } from '@/lib/table/column-types' +import { formatDateCellDisplay, getWallClockParts, normalizeDateCellValue } from '@/lib/table/dates' /** * Pick a fresh "untitled[_N]" name not already taken by `columns`. Used by @@ -23,26 +18,6 @@ export function generateColumnName(columns: ReadonlyArray<{ name: string }>): st return name } -/** - * Returns the appropriate badge color variant for a column type - */ -export function getTypeBadgeVariant(type: string): BadgeVariant { - switch (type) { - case 'string': - return 'green' - case 'number': - return 'blue' - case 'boolean': - return 'purple' - case 'json': - return 'orange' - case 'date': - return 'teal' - default: - return 'gray' - } -} - /** * Coerce a raw input value to the appropriate type for a column. * Throws on invalid JSON. @@ -52,11 +27,9 @@ export function cleanCellValue( column: ColumnDefinition, timeZone?: string ): unknown { - if (column.type === 'number') { - if (value === '') return null - const num = Number(value) - return Number.isNaN(num) ? null : num - } + // These three read the browser's own context (the viewer's timezone, a JSON + // draft that must throw so the editor can show a parse error, a checkbox's + // truthiness) so they cannot come from the shared coercion. if (column.type === 'json') { if (typeof value === 'string') { if (value === '') return null @@ -64,56 +37,17 @@ export function cleanCellValue( } return value } - if (column.type === 'boolean') { - return Boolean(value) - } + if (column.type === 'boolean') return Boolean(value) if (column.type === 'date') { if (value === '' || value === null || value === undefined) return null return displayToStorage(String(value), timeZone) } - if (column.type === 'select') { - return cleanSelectValue(value, column) - } - return value || null -} - -/** - * Client-side mirror of the server's `select` coercion: a cell stores option - * ids, but pasted or imported text carries names. Resolving here — not only on - * the server — is what keeps the optimistic cache holding ids, so the pasted - * cell renders its pill immediately instead of blanking until the refetch. - */ -function cleanSelectValue(value: unknown, column: ColumnDefinition): unknown { - const options = column.options ?? [] - const resolve = (raw: unknown): string | null => { - if (typeof raw !== 'string') return null - const match = - options.find((o) => o.id === raw) ?? - options.find((o) => o.name === raw) ?? - options.find((o) => o.name.toLowerCase() === raw.toLowerCase()) - return match ? match.id : null - } + if (value === '' || value === null || value === undefined) return null - if (column.multiple) { - // Comma-delimited is the multi cell's own clipboard/CSV format, so a paste - // of one round-trips. Option names containing commas are a known ambiguity. - const raw = Array.isArray(value) - ? value - : typeof value === 'string' - ? value - .split(',') - .map((part) => part.trim()) - .filter((part) => part !== '') - : [] - const ids: string[] = [] - for (const entry of raw) { - const id = resolve(entry) - if (id !== null && !ids.includes(id)) ids.push(id) - } - return ids - } - - return resolve(Array.isArray(value) ? value[0] : value) + // Everything else runs the SAME coercion the server will run, so the + // optimistic cache holds exactly the value that gets persisted. + const coerced = columnTypeOf(column).coerce(value as JsonValue, column) + return coerced.ok ? coerced.value : null } /** @@ -125,14 +59,15 @@ function cleanSelectValue(value: unknown, column: ColumnDefinition): unknown { */ export function formatValueForInput(value: unknown, type: string): string { if (value === null || value === undefined) return '' - if (type === 'json') { - return typeof value === 'string' ? value : JSON.stringify(value) - } - if (type === 'date' && value) { - return storedDateToEditable(String(value)) + const definition = columnTypeById(type) + // Shape-drift guard, kept ahead of the registry: a column whose declared type + // lags its actual data (a workflow column mid-remap, where the schema cache + // hasn't refetched but row data already holds the new mapping's value) would + // otherwise render `[object Object]` through a scalar type's formatter. + if (typeof value === 'object' && !definition.storesOpaqueIds && type !== 'json') { + return JSON.stringify(value) } - if (typeof value === 'object') return JSON.stringify(value) - return String(value) + return definition.formatForInput(value, { name: '', type: type as ColumnType }) } /** A canonical date-cell value split into its wall-clock editing parts. */ diff --git a/apps/sim/hooks/use-table-undo.ts b/apps/sim/hooks/use-table-undo.ts index c1d4c72c9d6..205e52b8b53 100644 --- a/apps/sim/hooks/use-table-undo.ts +++ b/apps/sim/hooks/use-table-undo.ts @@ -390,6 +390,7 @@ export function useTableUndo({ // cell data restored below is keyed by those option ids. ...(action.columnOptions ? { options: action.columnOptions } : {}), ...(action.columnMultiple ? { multiple: true } : {}), + ...(action.columnCurrencyCode ? { currencyCode: action.columnCurrencyCode } : {}), position: action.columnPosition, }, { diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 5bccaa52069..075a1a8a199 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -55,19 +55,45 @@ export const selectOptionsSchema = z .array(selectOptionSchema) .max(MAX_SELECT_OPTIONS, `A select column cannot have more than ${MAX_SELECT_OPTIONS} options`) +/** + * ISO 4217 code for a `currency` column, normalized to upper case. + * + * Deliberately shape-only. Whether a code is one the runtime can actually + * format is checked server-side in `validateColumnDefinition`, because this + * same schema parses RESPONSES: pinning the boundary to the *client's* ICU + * table would make any divergence between the two runtimes' currency lists + * reject an entire table schema over one column's code. + */ +export const currencyCodeSchema = z + .string() + .regex(/^[A-Za-z]{3}$/, 'Must be a 3-letter ISO 4217 currency code, e.g. USD') + .transform((code) => code.toUpperCase()) + /** * Cross-field rule: a `select` column must declare a non-empty option set; - * other types must not carry options or `multiple`. Skipped when `type` is - * absent (an options-only update on an existing select column). + * other types must not carry options or `multiple`, and only a `currency` + * column may carry `currencyCode`. Skipped when `type` is absent (a + * metadata-only update on an existing column). */ function refineColumnOptions( data: { type?: (typeof COLUMN_TYPES)[number] options?: z.infer multiple?: boolean + currencyCode?: string }, ctx: z.RefinementCtx ): void { + // `currencyCode` on a non-currency column is inert until a later + // convert-to-currency inherits it, silently overriding the currency the user + // picked in that request. + if (data.type !== undefined && data.type !== 'currency' && data.currencyCode !== undefined) { + ctx.addIssue({ + code: 'custom', + path: ['currencyCode'], + message: 'currencyCode is only allowed on currency columns', + }) + } if (data.type === 'select') { if (!data.options || data.options.length === 0) { ctx.addIssue({ @@ -165,6 +191,8 @@ export const tableColumnSchema = z options: selectOptionsSchema.optional(), /** A `select` column that accepts multiple options per cell. */ multiple: z.boolean().optional(), + /** ISO 4217 code for a `currency` column. */ + currencyCode: currencyCodeSchema.optional(), }) .superRefine(refineColumnOptions) @@ -242,6 +270,7 @@ export const createTableColumnBodySchema = z.object({ position: z.number().int().min(0).optional(), options: selectOptionsSchema.optional(), multiple: z.boolean().optional(), + currencyCode: currencyCodeSchema.optional(), }) .superRefine(refineColumnOptions), }) @@ -257,6 +286,7 @@ export const updateTableColumnBodySchema = z.object({ unique: z.boolean().optional(), options: selectOptionsSchema.optional(), multiple: z.boolean().optional(), + currencyCode: currencyCodeSchema.optional(), }) .superRefine(refineColumnOptions), }) diff --git a/apps/sim/lib/copilot/tools/server/table/user-table.ts b/apps/sim/lib/copilot/tools/server/table/user-table.ts index 9f66adcb9da..036ceb2cd40 100644 --- a/apps/sim/lib/copilot/tools/server/table/user-table.ts +++ b/apps/sim/lib/copilot/tools/server/table/user-table.ts @@ -34,15 +34,18 @@ import { sortSpecNamesToIds, } from '@/lib/table/column-keys' import { columnTypeForLeaf, deriveOutputColumnName } from '@/lib/table/column-naming' +import { columnTypeById } from '@/lib/table/column-types' import { addTableColumn, deleteColumn, deleteColumns, renameColumn, updateColumnConstraints, + updateColumnCurrency, updateColumnOptions, updateColumnType, } from '@/lib/table/columns/service' +import { isSupportedCurrencyCode } from '@/lib/table/currency' import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner' import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' @@ -1540,6 +1543,7 @@ export const userTableServerTool: BaseServerTool position?: number options?: unknown multiple?: boolean + currencyCode?: string } | undefined if (!col?.name || !col?.type) { @@ -1554,6 +1558,12 @@ export const userTableServerTool: BaseServerTool } const requestId = generateId().slice(0, 8) assertNotAborted() + if (col.currencyCode !== undefined && !isSupportedCurrencyCode(col.currencyCode)) { + return { + success: false, + message: `Invalid currency code "${col.currencyCode}". Use an ISO 4217 code, e.g. USD`, + } + } // Agent authors select options by name; generate their stable ids here. const columnToAdd = col.type === 'select' @@ -1653,15 +1663,24 @@ export const userTableServerTool: BaseServerTool const uniqFlag = (args as Record).unique as boolean | undefined const rawOptions = (args as Record).options const multiple = (args as Record).multiple as boolean | undefined + const currencyCode = (args as Record).currencyCode as string | undefined if ( newType === undefined && uniqFlag === undefined && rawOptions === undefined && - multiple === undefined + multiple === undefined && + currencyCode === undefined ) { return { success: false, - message: 'At least one of newType, unique, options, or multiple must be provided', + message: + 'At least one of newType, unique, options, multiple, or currencyCode must be provided', + } + } + if (currencyCode !== undefined && !isSupportedCurrencyCode(currencyCode)) { + return { + success: false, + message: `Invalid currency code "${currencyCode}". Use an ISO 4217 code, e.g. USD`, } } const tableForUpdate = await getTableById(args.tableId) @@ -1693,10 +1712,10 @@ export const userTableServerTool: BaseServerTool // update on an existing select column carries the same hazard as a // conversion. Same guard the HTTP column routes apply. const resultingType = newType ?? currentColumn?.type - if (uniqFlag === true && resultingType === 'select') { + if (uniqFlag === true && !columnTypeById(resultingType).supportsUnique) { return { success: false, - message: `Cannot set column "${colName}" as unique: select columns cannot be unique.`, + message: `Cannot set column "${colName}" as unique: ${resultingType} columns cannot be unique.`, } } if (typeChanging) { @@ -1708,6 +1727,27 @@ export const userTableServerTool: BaseServerTool newType: newType as (typeof COLUMN_TYPES)[number], options, multiple, + ...(currencyCode !== undefined ? { currencyCode } : {}), + ...(uniqFlag !== undefined ? { unique: uniqFlag } : {}), + }, + requestId + ) + } else if (currencyCode !== undefined) { + // Re-denominating an existing currency column: schema-only, no cell + // rewrite. Mirrors the HTTP columns routes. + if (currentColumn?.type !== 'currency') { + return { + success: false, + message: `Column "${colName}" is not a currency column. Pass newType: "currency" with currencyCode to convert it.`, + } + } + assertNotAborted() + result = await updateColumnCurrency( + { + tableId: args.tableId, + columnName: colName, + currencyCode, + ...(uniqFlag !== undefined ? { unique: uniqFlag } : {}), }, requestId ) @@ -1725,11 +1765,20 @@ export const userTableServerTool: BaseServerTool } assertNotAborted() result = await updateColumnOptions( - { tableId: args.tableId, columnName: colName, options: nextOptions, multiple }, + { + tableId: args.tableId, + columnName: colName, + options: nextOptions, + multiple, + ...(uniqFlag !== undefined ? { unique: uniqFlag } : {}), + }, requestId ) } - if (uniqFlag !== undefined) { + // Skipped when a typed write ran: that write already applied and + // validated the constraint, in one transaction with the change it + // accompanies. Mirrors the HTTP columns routes. + if (uniqFlag !== undefined && result === undefined) { assertNotAborted() result = await updateColumnConstraints( { tableId: args.tableId, columnName: colName, unique: uniqFlag }, diff --git a/apps/sim/lib/table/__tests__/column-conversion.test.ts b/apps/sim/lib/table/__tests__/column-conversion.test.ts index 71726cf00e0..3e7c1bf8a55 100644 --- a/apps/sim/lib/table/__tests__/column-conversion.test.ts +++ b/apps/sim/lib/table/__tests__/column-conversion.test.ts @@ -5,7 +5,13 @@ * coverage and every case below was a shipped defect caught in review. */ import { describe, expect, it } from 'vitest' -import { isValueCompatibleWithType, selectValueForConversion } from '@/lib/table/columns/service' +import { COLUMN_TYPE_REGISTRY } from '@/lib/table/column-types' +import { + applyPendingRename, + isValueCompatibleWithType, + selectValueForConversion, +} from '@/lib/table/columns/service' +import { resolveSelectOptionId } from '@/lib/table/select-options' import type { ColumnDefinition, SelectOption } from '@/lib/table/types' const OPTIONS: SelectOption[] = [ @@ -113,3 +119,140 @@ describe('isValueCompatibleWithType — string target', () => { expect(isValueCompatibleWithType(flattened, 'string')).toBe(true) }) }) + +describe('isValueCompatibleWithType — currency', () => { + it('accepts numbers and the formatted shapes a text column holds', () => { + expect(isValueCompatibleWithType(1234.56, 'currency')).toBe(true) + expect(isValueCompatibleWithType(0, 'currency')).toBe(true) + expect(isValueCompatibleWithType('$1,234.56', 'currency')).toBe(true) + expect(isValueCompatibleWithType('1.234,56 €', 'currency')).toBe(true) + expect(isValueCompatibleWithType('(12.00)', 'currency')).toBe(true) + }) + + it('rejects values that would leave un-castable text in a numeric column', () => { + expect(isValueCompatibleWithType('ask sales', 'currency')).toBe(false) + expect(isValueCompatibleWithType('', 'currency')).toBe(false) + expect(isValueCompatibleWithType(true, 'currency')).toBe(false) + expect(isValueCompatibleWithType({ amount: 1 }, 'currency')).toBe(false) + }) + + it('treats an absent cell as convertible, like every other target type', () => { + expect(isValueCompatibleWithType(null, 'currency')).toBe(true) + expect(isValueCompatibleWithType(undefined, 'currency')).toBe(true) + }) + + it('accepts a currency cell converting back to a plain number', () => { + expect(isValueCompatibleWithType(1234.56, 'number')).toBe(true) + }) +}) + +describe('blank cells during conversion', () => { + // A text column with a single empty cell could not be converted to a number + // at all: `''` is incompatible with every numeric type, and the scan counted + // it as a hard blocker even when the target was optional — reporting it with + // an error that said "to a required ..." regardless. + it('treats an empty cell as incompatible with the numeric types', () => { + for (const type of ['number', 'currency'] as const) { + expect(isValueCompatibleWithType('', type)).toBe(false) + } + }) + + it('still accepts an empty string for text, which can legitimately hold it', () => { + expect(isValueCompatibleWithType('', 'string')).toBe(true) + expect(isValueCompatibleWithType('', 'json')).toBe(true) + }) + + it('lets a cleared select cell through only when the target is optional', () => { + expect(isValueCompatibleWithType('', 'select', OPTIONS, false, false)).toBe(true) + expect(isValueCompatibleWithType('', 'select', OPTIONS, false, true)).toBe(false) + }) +}) + +describe('rename folded into another write', () => { + // A rename is metadata-only — rows key on the stable column id — so nothing + // forces it to be its own transaction. Folding it into whichever write a + // request already carries is what makes a combined PATCH all-or-nothing: the + // name collision is detected against the same schema snapshot the other + // change is being applied to, and both abort together. + it('rejects a collision against the schema the write is landing in', () => { + const columns: ColumnDefinition[] = [ + { id: 'col_a', name: 'amount', type: 'currency' }, + { id: 'col_b', name: 'taken', type: 'string' }, + ] + expect(() => applyPendingRename(columns, 0, 'taken')).toThrow(/already exists/) + expect(() => applyPendingRename(columns, 0, 'TAKEN')).toThrow(/already exists/) + }) + + it('signals a no-op by identity, which is how callers detect nothing to write', () => { + // The early returns in `updateColumnType` / `updateColumnCurrency` rely on + // this: same reference means there is genuinely nothing to persist. + const columns: ColumnDefinition[] = [{ id: 'col_a', name: 'amount', type: 'currency' }] + expect(applyPendingRename(columns, 0, undefined)).toBe(columns[0]) + expect(applyPendingRename(columns, 0, 'amount')).toBe(columns[0]) + expect(applyPendingRename(columns, 0, 'renamed')).not.toBe(columns[0]) + }) + + it('applies a valid rename and is a no-op without one', () => { + const columns: ColumnDefinition[] = [{ id: 'col_a', name: 'amount', type: 'currency' }] + expect(applyPendingRename(columns, 0, 'total').name).toBe('total') + expect(applyPendingRename(columns, 0, undefined)).toBe(columns[0]) + expect(applyPendingRename(columns, 0, 'amount')).toBe(columns[0]) + }) + + it('rejects a name the column-name rules forbid', () => { + const columns: ColumnDefinition[] = [{ id: 'col_a', name: 'amount', type: 'currency' }] + expect(() => applyPendingRename(columns, 0, '1bad')).toThrow(/must start with/) + expect(() => applyPendingRename(columns, 0, 'a'.repeat(200))).toThrow(/maximum length/) + }) +}) + +describe('select accepts scalar cells', () => { + // The resolver stringifies a scalar before matching, so a `number` or + // `boolean` column whose values equal option NAMES converts. The migration's + // JSONB predicate has to cover those types too — matching only `'string'` + // left the cells as raw numbers inside a select column, where they render as + // nothing and fail option membership on the next write. + const NUMERIC_OPTIONS: SelectOption[] = [ + { id: 'opt_1', name: '123' }, + { id: 'opt_t', name: 'true' }, + ] + + it('resolves a numeric or boolean cell to its option id', () => { + expect(resolveSelectOptionId(123, NUMERIC_OPTIONS)).toBe('opt_1') + expect(resolveSelectOptionId(true, NUMERIC_OPTIONS)).toBe('opt_t') + }) + + it('reports those cells as convertible, which is what obliges the migration', () => { + expect(isValueCompatibleWithType(123, 'select', NUMERIC_OPTIONS)).toBe(true) + expect(isValueCompatibleWithType(true, 'select', NUMERIC_OPTIONS)).toBe(true) + expect(isValueCompatibleWithType(999, 'select', NUMERIC_OPTIONS)).toBe(false) + }) + + it('leaves structured values unresolvable', () => { + expect(resolveSelectOptionId({ a: 1 } as never, NUMERIC_OPTIONS)).toBeNull() + }) +}) + +describe('constraint validation reads post-migration values', () => { + // The ordering that matters: `updateColumnOptions` rewrites stored cells (a + // single<->multi toggle changes the shape; removing an option clears cells), + // and `updateColumnType` rewrites them through the coercion write-back. A + // `unique` scan run BEFORE those would read values that no longer exist by + // the time the constraint is persisted, pass, and let the rewrite produce the + // duplicates it was supposed to prevent. + // + // The coercion case is the concrete one: two distinct strings can collapse to + // one number. + it('shows how a conversion manufactures duplicates the pre-scan cannot see', () => { + const column: ColumnDefinition = { name: 'sku', type: 'number' } + const before = ['5', '5.0'] + expect(new Set(before).size).toBe(2) + + const after = before.map((value) => { + const coerced = COLUMN_TYPE_REGISTRY.number.coerce(value, column) + return coerced.ok ? coerced.value : value + }) + expect(after).toEqual([5, 5]) + expect(new Set(after).size).toBe(1) + }) +}) diff --git a/apps/sim/lib/table/__tests__/column-type-registry.test.ts b/apps/sim/lib/table/__tests__/column-type-registry.test.ts new file mode 100644 index 00000000000..73c5ffc424c --- /dev/null +++ b/apps/sim/lib/table/__tests__/column-type-registry.test.ts @@ -0,0 +1,205 @@ +/** + * @vitest-environment node + * + * Guards for the column-type registry itself, rather than for any one type. + * + * The registry replaced ~40 hand-maintained `switch` arms whose failure mode + * was silence — a missing arm compared numbers as text or blocked every + * conversion, with nothing to notice. These assert the properties that used to + * be spread across those arms, so a new type either satisfies them or fails + * here. + */ +import { describe, expect, it } from 'vitest' +import type { ColumnType } from '@/lib/table/column-types' +import { + ALL_COLUMN_TYPES, + COLUMN_TYPE_REGISTRY, + COLUMN_TYPES, + columnTypeById, + isColumnType, + isValueCompatible, +} from '@/lib/table/column-types' +import type { ColumnDefinition } from '@/lib/table/types' +import { validateColumnDefinition } from '@/lib/table/validation' + +describe('registry shape', () => { + it('keys every entry by its own id', () => { + for (const [key, definition] of Object.entries(COLUMN_TYPE_REGISTRY)) { + expect(definition.id).toBe(key) + } + }) + + it('derives COLUMN_TYPES from the registry, with no drift', () => { + expect([...COLUMN_TYPES].sort()).toEqual(Object.keys(COLUMN_TYPE_REGISTRY).sort()) + expect(ALL_COLUMN_TYPES).toHaveLength(COLUMN_TYPES.length) + }) + + it('falls back to string for an unknown type instead of throwing', () => { + // A malformed or future schema must render as text, not crash mid-render. + expect(columnTypeById('percent').id).toBe('string') + expect(columnTypeById(undefined).id).toBe('string') + expect(isColumnType('percent')).toBe(false) + expect(isColumnType('currency')).toBe(true) + }) + + it('only casts to numeric/timestamptz for types whose storage is actually that', () => { + // A wrong cast makes every filter and sort on the column fail in SQL. + for (const definition of ALL_COLUMN_TYPES) { + if (definition.jsonbCast === null) continue + expect(['numeric', 'timestamptz']).toContain(definition.jsonbCast) + } + expect(COLUMN_TYPE_REGISTRY.currency.jsonbCast).toBe(COLUMN_TYPE_REGISTRY.number.jsonbCast) + }) + + it('restricts filter operators only for types storing opaque ids', () => { + // Restricting a comparable type would silently drop valid filters. + for (const definition of ALL_COLUMN_TYPES) { + const restricted = definition.filterOperatorsFor?.({ name: 'c', type: definition.id }) + if (restricted) expect(definition.storesOpaqueIds).toBe(true) + } + }) + + it('never lets a type with its own metadata be unique-constrained implicitly', () => { + // Uniqueness compares the stored value; for a type whose storage is an + // opaque id that caps each option at one row for the whole table. + for (const definition of ALL_COLUMN_TYPES) { + if (definition.storesOpaqueIds) expect(definition.supportsUnique).toBe(false) + } + }) + + it('never asks for a native number input on a type accepting formatted text', () => { + // `` rejects `$1,234.56` outright, so a type whose + // parser exists to accept that must get a text field with a numeric keypad. + for (const definition of ALL_COLUMN_TYPES) { + if (!definition.acceptsFormattedInput) continue + expect(definition.inputMode).toBe('decimal') + } + }) + + it('gives every type that can reject a draft a message to show', () => { + // Without one, `cleanCellValue` nulls the draft and the edit vanishes with + // no explanation. + for (const definition of ALL_COLUMN_TYPES) { + if (definition.typeaheadPattern) expect(definition.parseErrorMessage).toBeTruthy() + } + }) +}) + +describe('conversion write-back', () => { + // A retype is allowed exactly when the target's `coerce` accepts the value, + // and `coerce` often TRANSFORMS it. The conversion must therefore write the + // transformed value back — filters and sorts apply `jsonbCast` to whatever is + // stored, so a value left in its old shape breaks every query on the column. + it.each` + type | stored | expected + ${'date'} | ${1700000000000} | ${'2023-11-14T22:13:20.000Z'} + ${'date'} | ${'2024-01-01'} | ${'2024-01-01'} + ${'currency'} | ${'$1,234.56'} | ${1234.56} + ${'currency'} | ${'1.234,56'} | ${1234.56} + ${'number'} | ${'1999'} | ${1999} + `('$type coerces $stored to a value its jsonbCast can read', ({ type, stored, expected }) => { + const column = { name: 'c', type } as ColumnDefinition + const result = COLUMN_TYPE_REGISTRY[type as ColumnType].coerce(stored, column) + expect(result.ok && result.value).toEqual(expected) + }) + + it('never leaves a numeric-cast type holding something Postgres cannot cast', () => { + // The concrete failure this guards: an epoch number left in a `date` + // column makes `(data->>'col')::timestamptz` throw on every query. + for (const definition of ALL_COLUMN_TYPES) { + if (definition.jsonbCast !== 'timestamptz') continue + const coerced = definition.coerce(1700000000000, { name: 'c', type: definition.id }) + expect(coerced.ok).toBe(true) + expect(typeof (coerced as { value: unknown }).value).toBe('string') + } + }) +}) + +describe('intentional divergences from the pre-registry behavior', () => { + // A differential run of the registry against the pre-refactor implementations + // (55 values x 7 column shapes) found ZERO coercion differences and exactly + // these compatibility differences. Both are deliberate fixes; pinning them + // here so neither can be silently reverted or quietly widened. + + it('rejects boolean conversions the old gate accepted and then nulled', () => { + // The old gate accepted '1'/'0'/1/0 but the write path only ever accepted + // 'true'/'false' — so the conversion reported zero incompatible rows and + // then nulled every one of them. Rejecting is the honest answer. + const column: ColumnDefinition = { name: 'b', type: 'boolean' } + for (const value of ['0', '1', 0, 1]) { + expect(isValueCompatible(value, column)).toBe(false) + expect(COLUMN_TYPE_REGISTRY.boolean.coerce(value as never, column).ok).toBe(false) + } + for (const value of [true, false, 'true', 'false']) { + expect(isValueCompatible(value, column)).toBe(true) + } + }) + + it('refuses to bulk-convert a number column to date', () => { + // `date.coerce` accepts an epoch for a single deliberate write, but + // reinterpreting a whole numeric column as epoch milliseconds is + // destructive and irreversible — 1, 5, 42 would become three timestamps in + // January 1970. The gate may be stricter than `coerce`, never looser. + const column: ColumnDefinition = { name: 'd', type: 'date' } + for (const value of [0, 1, 42, 1700000000]) { + expect(isValueCompatible(value, column)).toBe(false) + // The write path still accepts it. + expect(COLUMN_TYPE_REGISTRY.date.coerce(value as never, column).ok).toBe(true) + } + expect(isValueCompatible('2024-01-01', column)).toBe(true) + }) + + it('never lets a gate be LOOSER than its write path', () => { + // The dangerous direction: a gate that accepts what `coerce` rejects + // reports zero incompatible rows and then nulls every one of them. + const samples: unknown[] = ['', '0', '1', 'abc', 0, 1, 42, true, '2024-01-01', '$1.50'] + for (const definition of ALL_COLUMN_TYPES) { + if (definition.id === 'select') continue + const column: ColumnDefinition = { name: 'c', type: definition.id } + for (const value of samples) { + if (!isValueCompatible(value, column)) continue + expect( + definition.coerce(value as never, column).ok, + `${definition.id} gate accepts ${JSON.stringify(value)} but coerce rejects it` + ).toBe(true) + } + } + }) +}) + +describe('metadata ownership', () => { + const column = (over: Partial): ColumnDefinition => + ({ name: 'c', type: 'string', ...over }) as ColumnDefinition + const options = [{ id: 'opt_a', name: 'A' }] + + it.each` + label | definition | valid | needle + ${'options on select'} | ${column({ type: 'select', options })} | ${true} | ${''} + ${'options on string'} | ${column({ type: 'string', options })} | ${false} | ${'cannot define options'} + ${'options on currency'} | ${column({ type: 'currency', options })} | ${false} | ${'cannot define options'} + ${'multiple on number'} | ${column({ type: 'number', multiple: true })} | ${false} | ${'cannot be multiple'} + ${'code on currency'} | ${column({ type: 'currency', currencyCode: 'USD' })} | ${true} | ${''} + ${'code on number'} | ${column({ type: 'number', currencyCode: 'USD' })} | ${false} | ${'cannot define a currency'} + ${'code on select'} | ${column({ type: 'select', currencyCode: 'USD', options })} | ${false} | ${'cannot define a currency'} + ${'unsupported code'} | ${column({ type: 'currency', currencyCode: 'ZZZ' })} | ${false} | ${'invalid currency code'} + ${'unique on select'} | ${column({ type: 'select', unique: true, options })} | ${false} | ${'cannot be unique'} + ${'unique on currency'} | ${column({ type: 'currency', unique: true })} | ${true} | ${''} + ${'select with no option'} | ${column({ type: 'select' })} | ${false} | ${'at least one option'} + ${'unknown type'} | ${column({ type: 'percent' as ColumnDefinition['type'] })} | ${false} | ${'invalid type'} + `( + 'rejects $label', + ({ + definition, + valid, + needle, + }: { + definition: ColumnDefinition + valid: boolean + needle: string + }) => { + const result = validateColumnDefinition(definition) + expect(result.valid, result.errors.join('; ')).toBe(valid) + if (!valid) expect(result.errors.join(' ').toLowerCase()).toContain(needle.toLowerCase()) + } + ) +}) diff --git a/apps/sim/lib/table/__tests__/currency.test.ts b/apps/sim/lib/table/__tests__/currency.test.ts new file mode 100644 index 00000000000..e7b7802552b --- /dev/null +++ b/apps/sim/lib/table/__tests__/currency.test.ts @@ -0,0 +1,279 @@ +/** + * @vitest-environment node + * + * Guards for the `currency` column type. The parser is the load-bearing piece: + * filters and sorts cast a currency cell to `numeric`, so anything it lets + * through unparsed would break every query against the column. + */ +import { describe, expect, it } from 'vitest' +import { + DEFAULT_CURRENCY_CODE, + formatCurrencyDisplay, + formatCurrencyForInput, + getCurrencyOptions, + isSupportedCurrencyCode, + parseCurrencyInput, + resolveCurrencyCode, +} from '@/lib/table/currency' + +describe('parseCurrencyInput', () => { + it('passes finite numbers through and rejects non-finite ones', () => { + expect(parseCurrencyInput(1234.56)).toBe(1234.56) + expect(parseCurrencyInput(0)).toBe(0) + expect(parseCurrencyInput(-5)).toBe(-5) + expect(parseCurrencyInput(Number.NaN)).toBeNull() + expect(parseCurrencyInput(Number.POSITIVE_INFINITY)).toBeNull() + }) + + it('strips symbols, ISO codes, and every flavor of space', () => { + expect(parseCurrencyInput('$1234.56')).toBe(1234.56) + expect(parseCurrencyInput('€ 12')).toBe(12) + expect(parseCurrencyInput('USD 12.50')).toBe(12.5) + expect(parseCurrencyInput('12,50 €')).toBe(12.5) + // Non-breaking and narrow-no-break spaces are what fr/ru locales group with. + expect(parseCurrencyInput('1 234,56 €')).toBe(1234.56) + expect(parseCurrencyInput('1 234.56')).toBe(1234.56) + }) + + it('reads accounting negatives', () => { + expect(parseCurrencyInput('(1,234.56)')).toBe(-1234.56) + expect(parseCurrencyInput('($12)')).toBe(-12) + expect(parseCurrencyInput('-$12.50')).toBe(-12.5) + }) + + it('treats the later separator as the decimal point when both appear', () => { + expect(parseCurrencyInput('1,234.56')).toBe(1234.56) + expect(parseCurrencyInput('1.234,56')).toBe(1234.56) + expect(parseCurrencyInput('1,234,567.89')).toBe(1234567.89) + expect(parseCurrencyInput('1.234.567,89')).toBe(1234567.89) + }) + + it('disambiguates a lone comma by the digits that follow it', () => { + // Exactly three trailing digits reads as grouping... + expect(parseCurrencyInput('1,500')).toBe(1500) + expect(parseCurrencyInput('100,000')).toBe(100000) + // ...anything else is a decimal comma. + expect(parseCurrencyInput('1,5')).toBe(1.5) + expect(parseCurrencyInput('0,25')).toBe(0.25) + expect(parseCurrencyInput('12,3456')).toBe(12.3456) + }) + + it('treats repeated dots as grouping and a lone dot as the decimal point', () => { + expect(parseCurrencyInput('1.234.567')).toBe(1234567) + expect(parseCurrencyInput('1.5')).toBe(1.5) + expect(parseCurrencyInput('1234.5')).toBe(1234.5) + }) + + it('reads exponent form at face value', () => { + // `String()` emits exponent form past 1e21, so a stored amount round-trips + // through the editor as `1e+21`. Treating the `e` as decoration to strip + // read that back as 121 — a silent 19-orders-of-magnitude loss on the next + // edit of an untouched cell. + expect(parseCurrencyInput('1e5')).toBe(100000) + expect(parseCurrencyInput('1e+21')).toBe(1e21) + expect(parseCurrencyInput('1.5e-3')).toBe(0.0015) + expect(parseCurrencyInput('-1e5')).toBe(-100000) + expect(parseCurrencyInput('(1e5)')).toBe(-100000) + expect(parseCurrencyInput('$1e5')).toBe(100000) + }) + + it('does not mistake an ISO code for an exponent', () => { + // `EUR` survives the symbol strip with its `E` intact; it must still parse + // through the ordinary separator path. + expect(parseCurrencyInput('12 EUR')).toBe(12) + expect(parseCurrencyInput('EUR 12,50')).toBe(12.5) + expect(parseCurrencyInput('USD 1,234.56')).toBe(1234.56) + }) + + it('round-trips a magnitude that stringifies to exponent form', () => { + expect(parseCurrencyInput(formatCurrencyForInput(1e21))).toBe(1e21) + }) + + it('rejects a sign that is not leading, so dates do not read as amounts', () => { + // A date column converting to currency previously turned `2024-01-01` into + // 20240101 — the hyphens were dropped as decoration and the digit groups + // joined. Every cell in the column would have been silently corrupted. + expect(parseCurrencyInput('2024-01-01')).toBeNull() + expect(parseCurrencyInput('2024-01-01T10:30:00Z')).toBeNull() + expect(parseCurrencyInput('1-2-3')).toBeNull() + expect(parseCurrencyInput('12--3')).toBeNull() + // A leading sign is still a sign. + expect(parseCurrencyInput('-12')).toBe(-12) + expect(parseCurrencyInput('+12')).toBe(12) + expect(parseCurrencyInput('-$12.50')).toBe(-12.5) + }) + + it('parses what Intl emits, across locales and signs', () => { + // The realistic input: a user pastes a cell from a spreadsheet. Generated + // rather than hand-listed so a parser change cannot quietly regress a + // locale nobody thought to write down. + const pairs: Array<[string, string]> = [ + ['en-US', 'USD'], + ['de-DE', 'EUR'], + ['fr-FR', 'EUR'], + ['pt-BR', 'BRL'], + ['en-IN', 'INR'], + ['ja-JP', 'JPY'], + ['en-GB', 'GBP'], + ['de-CH', 'CHF'], + ['sv-SE', 'SEK'], + ['da-DK', 'DKK'], + ['pl-PL', 'PLN'], + ['ru-RU', 'RUB'], + ['it-IT', 'EUR'], + ['nl-NL', 'EUR'], + ['tr-TR', 'TRY'], + ['ko-KR', 'KRW'], + ['zh-CN', 'CNY'], + ['en-CA', 'CAD'], + ['en-AU', 'AUD'], + ['he-IL', 'ILS'], + ] + const amounts = [0, 12, 1234.56, 1234567.89, -12.5, -1234.56] + + for (const [locale, currency] of pairs) { + for (const amount of amounts) { + const formatted = new Intl.NumberFormat(locale, { + style: 'currency', + currency, + }).format(amount) + const parsed = parseCurrencyInput(formatted) + expect(parsed, `${locale}/${currency} ${JSON.stringify(formatted)}`).not.toBeNull() + // Zero-decimal currencies round, so compare within one unit. + expect( + Math.abs((parsed as number) - amount), + `${locale}/${currency} ${JSON.stringify(formatted)} -> ${parsed}` + ).toBeLessThanOrEqual(1) + } + } + }) + + it('parses the locale formats of the currencies the picker pins', () => { + // These are exactly what `Intl.NumberFormat` emits, i.e. what a user pastes + // from a spreadsheet. Rejecting them would make the pinned currencies + // unusable in their own conventional notation. + expect(parseCurrencyInput('R$ 1.234,56')).toBe(1234.56) + expect(parseCurrencyInput('₹12,34,567.89')).toBe(1234567.89) + expect(parseCurrencyInput('1 234,56 kr')).toBe(1234.56) + expect(parseCurrencyInput('1.234,56 kr.')).toBe(1234.56) + expect(parseCurrencyInput('1234,56 zł')).toBe(1234.56) + expect(parseCurrencyInput('CHF 1’234.56')).toBe(1234.56) + }) + + it('rejects an identifier whose letters touch its digits', () => { + // The distinguishing rule: a currency marker is always separated from the + // number by a space or a symbol, so letters touching digits mean this is a + // part number, not an amount. Without it, converting a column of SKUs to + // currency rewrote every cell with an invented value. + expect(parseCurrencyInput('SKU400')).toBeNull() + expect(parseCurrencyInput('ABC1234')).toBeNull() + expect(parseCurrencyInput('A1B2')).toBeNull() + // A marker separated properly still parses. + expect(parseCurrencyInput('USD 400')).toBe(400) + expect(parseCurrencyInput('$400')).toBe(400) + }) + + it('rejects text that merely contains digits', () => { + // Scraping digits out of arbitrary text invents a value. A string column of + // SKUs, phone numbers, or US-format dates converting to currency would + // otherwise report zero incompatible rows and rewrite every cell. + expect(parseCurrencyInput('01/02/2024')).toBeNull() + expect(parseCurrencyInput('Room 101')).toBeNull() + expect(parseCurrencyInput('Invoice 2024')).toBeNull() + expect(parseCurrencyInput('1_000')).toBeNull() + }) + + it('rejects malformed separator runs and invalid grouping', () => { + // `0.1.2` and `1,000,00` would read as 12 and 100000 under a plain + // strip-the-separator rule. + expect(parseCurrencyInput('1..2')).toBeNull() + expect(parseCurrencyInput('1,,2')).toBeNull() + expect(parseCurrencyInput('0.1.2')).toBeNull() + expect(parseCurrencyInput('1,000,00')).toBeNull() + // Valid grouping still works, western and Indian. + expect(parseCurrencyInput('1.234.567')).toBe(1234567) + expect(parseCurrencyInput('1,234,567.89')).toBe(1234567.89) + expect(parseCurrencyInput('12,34,567')).toBe(1234567) + }) + + it('rejects values carrying no amount', () => { + expect(parseCurrencyInput('')).toBeNull() + expect(parseCurrencyInput(' ')).toBeNull() + expect(parseCurrencyInput('n/a')).toBeNull() + expect(parseCurrencyInput('$')).toBeNull() + expect(parseCurrencyInput(null)).toBeNull() + expect(parseCurrencyInput(undefined)).toBeNull() + expect(parseCurrencyInput(true)).toBeNull() + expect(parseCurrencyInput(['12'])).toBeNull() + expect(parseCurrencyInput({ amount: 12 })).toBeNull() + }) + + it('round-trips its own display output', () => { + for (const amount of [0, 12, -12.5, 1234.56, 1234567.89]) { + const rendered = formatCurrencyDisplay(amount, 'USD', 'en-US') + expect(parseCurrencyInput(rendered)).toBe(amount) + } + }) +}) + +describe('formatCurrencyDisplay', () => { + it('uses the currency’s own symbol and fraction digits', () => { + expect(formatCurrencyDisplay(1234.56, 'USD', 'en-US')).toBe('$1,234.56') + // JPY has no minor unit, so it rounds to whole yen. + expect(formatCurrencyDisplay(1234.56, 'JPY', 'en-US')).toBe('¥1,235') + }) + + it('falls back to the default currency when the column declares none', () => { + expect(formatCurrencyDisplay(12, undefined, 'en-US')).toBe( + formatCurrencyDisplay(12, DEFAULT_CURRENCY_CODE, 'en-US') + ) + }) + + it('renders an unreadable value verbatim rather than blanking it', () => { + // What a string → currency conversion can leave behind. + expect(formatCurrencyDisplay('pending', 'USD', 'en-US')).toBe('pending') + expect(formatCurrencyDisplay(null, 'USD', 'en-US')).toBe('') + }) + + it('formats a stored string as if it were the number it encodes', () => { + expect(formatCurrencyDisplay('1234.56', 'USD', 'en-US')).toBe('$1,234.56') + }) +}) + +describe('formatCurrencyForInput', () => { + it('renders the bare amount, with no symbol or grouping', () => { + expect(formatCurrencyForInput(1234.56)).toBe('1234.56') + expect(formatCurrencyForInput('$1,234.56')).toBe('1234.56') + expect(formatCurrencyForInput(null)).toBe('') + expect(formatCurrencyForInput(undefined)).toBe('') + }) + + it('keeps unparseable text so an edit does not silently erase it', () => { + expect(formatCurrencyForInput('pending')).toBe('pending') + }) +}) + +describe('currency codes', () => { + it('accepts ISO 4217 codes case-insensitively and rejects the rest', () => { + expect(isSupportedCurrencyCode('USD')).toBe(true) + expect(isSupportedCurrencyCode('eur')).toBe(true) + expect(isSupportedCurrencyCode('ZZZ')).toBe(false) + expect(isSupportedCurrencyCode('US')).toBe(false) + expect(isSupportedCurrencyCode('DOLLAR')).toBe(false) + expect(isSupportedCurrencyCode('')).toBe(false) + }) + + it('upper-cases and defaults', () => { + expect(resolveCurrencyCode('eur')).toBe('EUR') + expect(resolveCurrencyCode(undefined)).toBe(DEFAULT_CURRENCY_CODE) + expect(resolveCurrencyCode('')).toBe(DEFAULT_CURRENCY_CODE) + }) + + it('offers the pinned codes first and no duplicates', () => { + const options = getCurrencyOptions() + expect(options[0].code).toBe('USD') + const codes = options.map((c) => c.code) + expect(new Set(codes).size).toBe(codes.length) + expect(codes).toContain('JPY') + }) +}) diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index f6b8be7dc36..fc3b77ed574 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -131,6 +131,38 @@ describe('Validation', () => { }) }) + describe('validateColumnDefinition — currency', () => { + const base: ColumnDefinition = { name: 'price', type: 'currency' } + + it('accepts a currency column with no code (it defaults on write)', () => { + expect(validateColumnDefinition(base).valid).toBe(true) + }) + + it('accepts a supported ISO 4217 code', () => { + expect(validateColumnDefinition({ ...base, currencyCode: 'JPY' }).valid).toBe(true) + }) + + it('rejects a code no runtime can format', () => { + const result = validateColumnDefinition({ ...base, currencyCode: 'ZZZ' }) + expect(result.valid).toBe(false) + expect(result.errors.join(' ')).toContain('invalid currency code') + }) + + it('rejects a currency code stashed on a non-currency column', () => { + const result = validateColumnDefinition({ + name: 'price', + type: 'number', + currencyCode: 'USD', + }) + expect(result.valid).toBe(false) + expect(result.errors.join(' ')).toContain('cannot define a currency') + }) + + it('allows a unique constraint, unlike select', () => { + expect(validateColumnDefinition({ ...base, unique: true }).valid).toBe(true) + }) + }) + describe('validateTableSchema', () => { it('should accept valid schema', () => { const schema: TableSchema = { @@ -470,6 +502,40 @@ describe('Validation', () => { expect(patch.tags).toEqual(['opt_a', 'opt_b']) }) }) + + describe('currency coercion', () => { + const currencySchema: TableSchema = { + columns: [ + { id: 'price', name: 'price', type: 'currency', currencyCode: 'USD' }, + { id: 'cost', name: 'cost', type: 'currency', currencyCode: 'EUR', required: true }, + ], + } + + it('parses a formatted amount down to a bare number', () => { + const patch: Record = { price: '$1,234.56' } + coerceRowValues(patch as never, currencySchema) + expect(patch.price).toBe(1234.56) + }) + + it('leaves an already-numeric cell untouched', () => { + const patch: Record = { price: 42 } + coerceRowValues(patch as never, currencySchema) + expect(patch.price).toBe(42) + }) + + it('nulls an unreadable amount on an optional column', () => { + const patch: Record = { price: 'ask sales' } + coerceRowValues(patch as never, currencySchema) + expect(patch.price).toBeNull() + }) + + it('leaves an unreadable amount in place on a required column so validation reports it', () => { + const patch: Record = { cost: 'ask sales' } + coerceRowValues(patch as never, currencySchema) + expect(patch.cost).toBe('ask sales') + expect(validateRowAgainstSchema(patch as never, currencySchema).valid).toBe(false) + }) + }) }) describe('getUniqueColumns', () => { diff --git a/apps/sim/lib/table/cell-format.ts b/apps/sim/lib/table/cell-format.ts index 9fe544885e2..2461cb286e4 100644 --- a/apps/sim/lib/table/cell-format.ts +++ b/apps/sim/lib/table/cell-format.ts @@ -11,6 +11,7 @@ */ import { getColumnId } from '@/lib/table/column-keys' +import { columnTypeOf } from '@/lib/table/column-types' import { selectValueToNames } from '@/lib/table/select-values' import type { ColumnDefinition, JsonValue, RowData } from '@/lib/table/types' @@ -24,7 +25,7 @@ import type { ColumnDefinition, JsonValue, RowData } from '@/lib/table/types' * must reach consumers byte-identical to what is stored. */ export function formatCellValue(value: unknown, column: ColumnDefinition): JsonValue { - if (column.type === 'select') return selectValueToNames(column, value) + if (columnTypeOf(column).storesOpaqueIds) return selectValueToNames(column, value) return value as JsonValue } diff --git a/apps/sim/lib/table/column-naming.ts b/apps/sim/lib/table/column-naming.ts index 1124e8da2df..eba112ee0d1 100644 --- a/apps/sim/lib/table/column-naming.ts +++ b/apps/sim/lib/table/column-naming.ts @@ -6,6 +6,7 @@ * get from the sidebar. */ +import { COLUMN_TYPE_REGISTRY, isColumnType } from '@/lib/table/column-types' import type { ColumnDefinition } from '@/lib/table/types' /** @@ -43,14 +44,11 @@ export function deriveOutputColumnName(path: string, taken: Set): string * union falls back to `json`, the most permissive shape that still validates. */ export function columnTypeForLeaf(leafType: string | undefined): ColumnDefinition['type'] { - switch (leafType) { - case 'string': - case 'number': - case 'boolean': - case 'date': - case 'json': - return leafType - default: - return 'json' + // A block output can only land on a type that carries no configuration of its + // own — a `select` needs an option set and a `currency` needs a code, neither + // of which a leaf type supplies. + if (isColumnType(leafType) && COLUMN_TYPE_REGISTRY[leafType].ownedMetadata.length === 0) { + return leafType } + return 'json' } diff --git a/apps/sim/lib/table/column-types/boolean.ts b/apps/sim/lib/table/column-types/boolean.ts new file mode 100644 index 00000000000..e3f5aacc624 --- /dev/null +++ b/apps/sim/lib/table/column-types/boolean.ts @@ -0,0 +1,40 @@ +import { TypeBoolean } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' + +export const booleanColumnType: ColumnTypeDefinition = { + id: 'boolean', + label: 'Boolean', + icon: TypeBoolean, + jsonbCast: null, + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: true, + ownedMetadata: [], + workflowInputType: 'boolean', + // Toggled in place on click, Enter, and fill — never opens an editor, so it + // has no `typeaheadPattern` and the expanded popover skips it entirely. + editor: 'toggle', + expandable: false, + + coerce(value) { + if (typeof value === 'boolean') return { ok: true, value } + if (typeof value === 'string') { + const normalized = value.trim().toLowerCase() + if (normalized === 'true') return { ok: true, value: true } + if (normalized === 'false') return { ok: true, value: false } + } + return { ok: false } + }, + + validateCell(value, column) { + return typeof value === 'boolean' ? null : `${column.name} must be boolean` + }, + + formatForDisplay(value) { + return String(value) + }, + + formatForInput(value) { + return String(value) + }, +} diff --git a/apps/sim/lib/table/column-types/currency.ts b/apps/sim/lib/table/column-types/currency.ts new file mode 100644 index 00000000000..16ee515f830 --- /dev/null +++ b/apps/sim/lib/table/column-types/currency.ts @@ -0,0 +1,68 @@ +import { TypeCurrency } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' +import { + formatCurrencyDisplay, + formatCurrencyForInput, + isSupportedCurrencyCode, + parseCurrencyInput, + resolveCurrencyCode, +} from '@/lib/table/currency' + +export const currencyColumnType: ColumnTypeDefinition = { + id: 'currency', + label: 'Currency', + icon: TypeCurrency, + jsonbCast: 'numeric', + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: 123, + ownedMetadata: ['currencyCode'], + workflowInputType: 'number', + editor: 'text', + expandable: false, + inputMode: 'decimal', + acceptsFormattedInput: true, + // Also accepts the grouping separator and the symbol the user is likely to + // type first — `parseCurrencyInput` strips both. + typeaheadPattern: /[\d.,\-\p{Sc}]/u, + parseErrorMessage: 'Invalid amount', + + coerce(value) { + // Stored as a bare number, but accepts the formatted shapes an amount + // arrives in — `$1,234.56`, `1 234,56 €`, `(12.00)` — so a paste, CSV + // import, or tool write lands as a number rather than being nulled. + const parsed = parseCurrencyInput(value) + return parsed === null ? { ok: false } : { ok: true, value: parsed } + }, + + validateCell(value, column) { + // A currency cell is a plain number; `currencyCode` is display metadata and + // never part of the stored value. + return typeof value === 'number' && !Number.isNaN(value) + ? null + : `${column.name} must be number` + }, + + validateDefinition(column) { + if (column.currencyCode !== undefined && !isSupportedCurrencyCode(column.currencyCode)) { + return [ + `Column "${column.name}" has invalid currency code "${column.currencyCode}". Use an ISO 4217 code, e.g. USD`, + ] + } + return [] + }, + + formatForDisplay(value, column) { + return formatCurrencyDisplay(value, column.currencyCode) + }, + + formatForInput(value) { + return formatCurrencyForInput(value) + }, + + defaultMetadata(column) { + // Always stamped, so the schema states the rendered currency explicitly + // rather than leaving readers to know the default. + return { currencyCode: resolveCurrencyCode(column.currencyCode) } + }, +} diff --git a/apps/sim/lib/table/column-types/date.ts b/apps/sim/lib/table/column-types/date.ts new file mode 100644 index 00000000000..11b980eeacd --- /dev/null +++ b/apps/sim/lib/table/column-types/date.ts @@ -0,0 +1,63 @@ +import { Calendar as CalendarIcon } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' +import { + formatDateCellDisplay, + normalizeDateCellValue, + storedDateToEditable, +} from '@/lib/table/dates' +import type { JsonValue } from '@/lib/table/types' + +export const dateColumnType: ColumnTypeDefinition = { + id: 'date', + label: 'Date', + icon: CalendarIcon, + jsonbCast: 'timestamptz', + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: '2024-01-31', + ownedMetadata: [], + workflowInputType: 'string', + editor: 'date', + expandable: false, + typeaheadPattern: /[\d\-/]/, + parseErrorMessage: 'Invalid date', + + coerce(value) { + if (typeof value === 'string') { + const normalized = normalizeDateCellValue(value) + return normalized === null ? { ok: false } : { ok: true, value: normalized } + } + // Date instances and epoch numbers may still be out of the representable + // range (>±8.64e15ms) — guard `toISOString()`, which throws RangeError on + // an Invalid Date, so an over-range value degrades to `{ ok: false }` + // rather than crashing the write. + const date = value instanceof Date ? value : typeof value === 'number' ? new Date(value) : null + if (date && !Number.isNaN(date.getTime())) return { ok: true, value: date.toISOString() } + return { ok: false } + }, + + isCompatibleWith(value) { + // Stricter than `coerce` on purpose. Writing a number into a date cell is a + // deliberate act — the caller means epoch milliseconds. Reinterpreting a + // whole NUMBER column as epochs is not: a column of 1, 5, 42 would become + // three timestamps in January 1970, irreversibly, and a Unix-seconds column + // would land in 1970 rather than the year it means. Refuse the bulk + // conversion; single writes still accept epochs. + if (typeof value === 'number') return false + return dateColumnType.coerce(value as JsonValue, { name: '', type: 'date' }).ok + }, + + validateCell(value, column) { + const valid = + value instanceof Date || (typeof value === 'string' && !Number.isNaN(Date.parse(value))) + return valid ? null : `${column.name} must be valid date` + }, + + formatForDisplay(value) { + return formatDateCellDisplay(String(value), { seconds: true }) + }, + + formatForInput(value) { + return storedDateToEditable(String(value)) + }, +} diff --git a/apps/sim/lib/table/column-types/index.ts b/apps/sim/lib/table/column-types/index.ts new file mode 100644 index 00000000000..9aebb2b1002 --- /dev/null +++ b/apps/sim/lib/table/column-types/index.ts @@ -0,0 +1,21 @@ +/** + * Column-type registry barrel. + * + * Client-safe — importing this never pulls `@sim/db` or `drizzle-orm`. Server + * code that needs the retype cell migrations imports `registry.server` instead. + * + * Deliberately NOT re-exported from `@/lib/table`: 44 server modules import + * that barrel, and routing this through it would pull `@sim/emcn/icons` into + * every one of them. Import `@/lib/table/column-types` directly, the same way + * `column-keys`, `constants`, and `dates` are already imported. + */ + +export * from '@/lib/table/column-types/registry' +export type { + CoerceResult, + ColumnCellEditor, + ColumnType, + ColumnTypeDefinition, + TypeSpecificColumnKey, +} from '@/lib/table/column-types/types' +export { TYPE_SPECIFIC_COLUMN_KEYS } from '@/lib/table/column-types/types' diff --git a/apps/sim/lib/table/column-types/json.ts b/apps/sim/lib/table/column-types/json.ts new file mode 100644 index 00000000000..027f57cea64 --- /dev/null +++ b/apps/sim/lib/table/column-types/json.ts @@ -0,0 +1,38 @@ +import { TypeJson } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' + +export const jsonColumnType: ColumnTypeDefinition = { + id: 'json', + label: 'JSON', + icon: TypeJson, + jsonbCast: null, + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: 'value', + ownedMetadata: [], + workflowInputType: 'object', + editor: 'text', + expandable: true, + + coerce(value) { + // Anything JSON-serializable is already valid — this is the widest type. + return { ok: true, value } + }, + + validateCell(value, column) { + try { + JSON.stringify(value) + return null + } catch { + return `${column.name} must be valid JSON` + } + }, + + formatForDisplay(value) { + return JSON.stringify(value) + }, + + formatForInput(value) { + return typeof value === 'string' ? value : JSON.stringify(value) + }, +} diff --git a/apps/sim/lib/table/column-types/number.ts b/apps/sim/lib/table/column-types/number.ts new file mode 100644 index 00000000000..92c5e321ba7 --- /dev/null +++ b/apps/sim/lib/table/column-types/number.ts @@ -0,0 +1,45 @@ +import { TypeNumber } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' + +export const numberColumnType: ColumnTypeDefinition = { + id: 'number', + label: 'Number', + icon: TypeNumber, + jsonbCast: 'numeric', + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: 123, + ownedMetadata: [], + workflowInputType: 'number', + editor: 'text', + expandable: false, + inputMode: 'decimal', + typeaheadPattern: /[\d.-]/, + parseErrorMessage: 'Invalid number', + + coerce(value) { + if (typeof value === 'number') { + return Number.isFinite(value) ? { ok: true, value } : { ok: false } + } + if (typeof value === 'string' && value.trim() !== '') { + const parsed = Number(value) + return Number.isFinite(parsed) ? { ok: true, value: parsed } : { ok: false } + } + return { ok: false } + }, + + validateCell(value, column) { + return typeof value === 'number' && !Number.isNaN(value) + ? null + : `${column.name} must be number` + }, + + formatForDisplay(value) { + return String(value) + }, + + formatForInput(value) { + if (typeof value === 'object') return JSON.stringify(value) + return String(value) + }, +} diff --git a/apps/sim/lib/table/column-types/registry.server.ts b/apps/sim/lib/table/column-types/registry.server.ts new file mode 100644 index 00000000000..a87eda3604a --- /dev/null +++ b/apps/sim/lib/table/column-types/registry.server.ts @@ -0,0 +1,238 @@ +/** + * Server-only half of the column-type registry: the cell rewrites that run + * inside the retype transaction. + * + * Kept out of `registry.ts` because these need a drizzle transaction, and the + * tables grid imports that module. Mirrors `connectors/registry.server.ts`. + * + * A migration is keyed by *direction*: `migrateCellsTo` runs when a column is + * converted into the type, `migrateCellsFrom` when it is converted out of it. + * `select` needs both — its cells store opaque option ids that mean nothing + * under any other type. `currency` needs only the inbound one. + */ + +import { userTableRows } from '@sim/db/schema' +import { sql } from 'drizzle-orm' +import { COLUMN_TYPE_REGISTRY } from '@/lib/table/column-types/registry' +import type { ColumnType } from '@/lib/table/column-types/types' +import type { + ColumnCellMigration, + ColumnTypeServerEntry, +} from '@/lib/table/column-types/types.server' +import type { DbTransaction } from '@/lib/table/planner' +import type { JsonValue, SelectOption } from '@/lib/table/types' + +/** + * Rewrites a column's cells from stored option **ids** to option **names**, for + * a column that is ceasing to be a `select`. A multi cell joins comma-separated + * — the same shape it exports as. + * + * An id whose option no longer exists becomes null, matching + * {@link selectValueForConversion}, which is what the compatibility check ran + * on. Passing it through instead would leave an opaque `opt_…` in a typed cell + * the check had already accounted as empty. + * + * Set-based: one statement per stored shape, driven by a jsonb id→name map, so + * cost is independent of row count. + */ +async function migrateSelectCellsToNames( + trx: DbTransaction, + tableId: string, + columnKey: string, + options: SelectOption[] +): Promise { + const nameById = JSON.stringify(Object.fromEntries(options.map((o) => [o.id, o.name]))) + await trx.execute( + sql`UPDATE ${userTableRows} + SET data = jsonb_set(data, ARRAY[${columnKey}::text], + COALESCE(${nameById}::jsonb -> (data->>${columnKey}::text), 'null'::jsonb)) + WHERE table_id = ${tableId} + AND jsonb_typeof(data->${columnKey}::text) = 'string'` + ) + await trx.execute( + sql`UPDATE ${userTableRows} + SET data = jsonb_set(data, ARRAY[${columnKey}::text], COALESCE(to_jsonb(( + SELECT string_agg(${nameById}::jsonb ->> e.v, ', ' ORDER BY e.ord) + FROM jsonb_array_elements_text(data->${columnKey}::text) WITH ORDINALITY AS e(v, ord) + WHERE ${nameById}::jsonb ? e.v + )), 'null'::jsonb)) + WHERE table_id = ${tableId} + AND jsonb_typeof(data->${columnKey}::text) = 'array'` + ) +} + +/** Rows rewritten per statement, bounding the size of the jsonb map parameter. */ +const COERCED_WRITE_BACK_BATCH_SIZE = 5000 + +/** + * Writes back the values a conversion's coercion produced. + * + * A retype is allowed exactly when the target type's `coerce` accepts the + * value, and `coerce` frequently *transforms* it — an epoch number becomes an + * ISO date, `$1,234.56` becomes `1234.56`. Without this, the cell keeps its old + * bytes under the new type, and since filters and sorts apply the type's + * `jsonbCast` to whatever is stored, an epoch left in a `date` column makes + * `::timestamptz` fail on EVERY query against that column. + * + * The values arrive already computed (the compatibility scan derived them), so + * this is purely the write. It cannot be expressed set-based — the coercions + * are JS, and a naive SQL equivalent would mangle exactly the inputs they + * disambiguate — so the map is folded into one statement per batch. + */ +export async function writeBackCoercedCells( + trx: DbTransaction, + tableId: string, + columnKey: string, + valueByRowId: ReadonlyMap +): Promise { + if (valueByRowId.size === 0) return + + const entries = [...valueByRowId] + for (let start = 0; start < entries.length; start += COERCED_WRITE_BACK_BATCH_SIZE) { + const batch = JSON.stringify( + Object.fromEntries(entries.slice(start, start + COERCED_WRITE_BACK_BATCH_SIZE)) + ) + await trx.execute( + sql`UPDATE ${userTableRows} AS r + SET data = jsonb_set(r.data, ARRAY[${columnKey}::text], m.value) + FROM jsonb_each(${batch}::jsonb) AS m(key, value) + WHERE r.table_id = ${tableId} AND r.id = m.key` + ) + } +} + +/** + * Rewrites a column's cells into the canonical `select` storage shape: the + * option **id**, wrapped in an array when the column is `multiple`. + * + * Needed in both directions of a select change. Converting *to* select, cells + * hold option names (that is what made them compatible) but every reader — + * pills, filters, exports — resolves by id. Toggling single→multi, cells hold a + * scalar id while multi filters compile to array containment, which a scalar + * never matches. Either way the cell silently drops out until it is re-edited. + * + * Scalar cells count, not just strings: `resolveSelectOptionId` stringifies a + * number or boolean before matching, so a `number` column whose values equal + * option NAMES passes the compatibility gate. Matching only `jsonb_typeof = + * 'string'` left those cells as raw numbers inside a select column, where they + * render as nothing and fail option membership on the next write. `data->>key` + * yields the text form for every scalar, so one widened predicate covers them. + * + * The map keys ids, names, and lower-cased names — ids so re-running is a no-op, + * lower-cased names because `resolveSelectOptionId` accepts a case-mismatched + * name and a cell that passed that check must actually migrate. Duplicate option + * names are rejected case-insensitively at validation, so the folded key is + * unambiguous; lookups still try the exact form first to preserve its precedence. + */ +async function migrateCellsToSelectIds( + trx: DbTransaction, + tableId: string, + columnKey: string, + options: SelectOption[], + multiple: boolean +): Promise { + // Ids are written last so an id always outranks any option's name, matching + // `resolveSelectOptionId`'s id-before-name precedence. A single flat pass + // would let a later option whose *name* equals an earlier option's *id* + // overwrite that id entry and repoint its cells at the wrong option. + // A Map, not plain-object assignment: an option named `__proto__` would set + // the prototype instead of an own key and drop out of the serialized map. + const refs = new Map() + for (const o of options) { + refs.set(o.name.toLowerCase(), o.id) + refs.set(o.name, o.id) + } + for (const o of options) { + refs.set(o.id, o.id) + } + const idByRef = JSON.stringify(Object.fromEntries(refs)) + + if (multiple) { + // A string cell reaching a multi target is either one option name or the + // comma-joined form a multiselect converts to text as. Try the whole string + // first so an option whose own name contains a comma still wins, then split + // — mirroring `splitMultiSelectInput` on the write path, including its + // first-occurrence dedup. + await trx.execute( + sql`UPDATE ${userTableRows} + SET data = jsonb_set(data, ARRAY[${columnKey}::text], + CASE WHEN data->>${columnKey}::text = '' THEN '[]'::jsonb + WHEN COALESCE(${idByRef}::jsonb -> (data->>${columnKey}::text), ${idByRef}::jsonb -> lower(data->>${columnKey}::text)) IS NOT NULL + THEN jsonb_build_array(COALESCE(${idByRef}::jsonb -> (data->>${columnKey}::text), ${idByRef}::jsonb -> lower(data->>${columnKey}::text))) + ELSE COALESCE(( + SELECT jsonb_agg(v ORDER BY ord) FROM ( + SELECT COALESCE(${idByRef}::jsonb -> btrim(part), ${idByRef}::jsonb -> lower(btrim(part)), to_jsonb(btrim(part))) AS v, + min(o) AS ord + FROM unnest(string_to_array(data->>${columnKey}::text, ',')) WITH ORDINALITY AS u(part, o) + WHERE btrim(part) <> '' + GROUP BY 1 + ) d), '[]'::jsonb) + END) + WHERE table_id = ${tableId} + AND jsonb_typeof(data->${columnKey}::text) IN ('string', 'number', 'boolean')` + ) + await trx.execute( + sql`UPDATE ${userTableRows} + SET data = jsonb_set(data, ARRAY[${columnKey}::text], COALESCE(( + SELECT jsonb_agg(COALESCE(${idByRef}::jsonb -> e.v, ${idByRef}::jsonb -> lower(e.v), to_jsonb(e.v)) ORDER BY e.ord) + FROM jsonb_array_elements_text(data->${columnKey}::text) WITH ORDINALITY AS e(v, ord) + ), '[]'::jsonb)) + WHERE table_id = ${tableId} + AND jsonb_typeof(data->${columnKey}::text) = 'array'` + ) + return + } + + // A cleared cell is stored as '' — compatibility lets it through, so it has + // to land as null rather than an '' that fails option membership on the next + // write. + await trx.execute( + sql`UPDATE ${userTableRows} + SET data = jsonb_set(data, ARRAY[${columnKey}::text], + CASE WHEN data->>${columnKey}::text = '' THEN 'null'::jsonb + ELSE COALESCE(${idByRef}::jsonb -> (data->>${columnKey}::text), ${idByRef}::jsonb -> lower(data->>${columnKey}::text), data->${columnKey}::text) + END) + WHERE table_id = ${tableId} + AND jsonb_typeof(data->${columnKey}::text) IN ('string', 'number', 'boolean')` + ) + // Compatibility already rejected multi-valued cells for a single target, so + // any array here holds at most one option. + await trx.execute( + sql`UPDATE ${userTableRows} + SET data = jsonb_set(data, ARRAY[${columnKey}::text], + COALESCE(${idByRef}::jsonb -> (data->${columnKey}::text->>0), ${idByRef}::jsonb -> lower(data->${columnKey}::text->>0), data->${columnKey}::text->0, 'null'::jsonb)) + WHERE table_id = ${tableId} + AND jsonb_typeof(data->${columnKey}::text) = 'array'` + ) +} + +/** + * Every column type plus its migrations. The `Record` + * annotation is the same completeness gate the client-safe registry uses: a + * new type will not compile until it appears here too. + */ +export const COLUMN_TYPE_SERVER_REGISTRY: Record = { + string: COLUMN_TYPE_REGISTRY.string, + number: COLUMN_TYPE_REGISTRY.number, + boolean: COLUMN_TYPE_REGISTRY.boolean, + date: COLUMN_TYPE_REGISTRY.date, + json: COLUMN_TYPE_REGISTRY.json, + select: { + ...COLUMN_TYPE_REGISTRY.select, + migrateCellsTo: ({ trx, tableId, columnKey, target }) => + migrateCellsToSelectIds(trx, tableId, columnKey, target.options ?? [], !!target.multiple), + migrateCellsFrom: ({ trx, tableId, columnKey, previous }) => + migrateSelectCellsToNames(trx, tableId, columnKey, previous.options ?? []), + }, + currency: COLUMN_TYPE_REGISTRY.currency, +} + +/** The inbound migration for a target type, if it has one. */ +export function migrationTo(type: ColumnType): ColumnCellMigration | undefined { + return COLUMN_TYPE_SERVER_REGISTRY[type]?.migrateCellsTo +} + +/** The outbound migration for a source type, if it has one. */ +export function migrationFrom(type: ColumnType): ColumnCellMigration | undefined { + return COLUMN_TYPE_SERVER_REGISTRY[type]?.migrateCellsFrom +} diff --git a/apps/sim/lib/table/column-types/registry.ts b/apps/sim/lib/table/column-types/registry.ts new file mode 100644 index 00000000000..8bc336a1dc1 --- /dev/null +++ b/apps/sim/lib/table/column-types/registry.ts @@ -0,0 +1,117 @@ +/** + * The column-type registry — one entry per table column type. + * + * Client-safe: this module and everything it imports stay free of `@sim/db`, + * `drizzle-orm`, and `next/server`, so the tables grid can import it directly. + * The server-only half (retype cell migrations) lives in `registry.server.ts`. + * + * ## Adding a column type + * + * 1. Add its id to {@link ColumnType} in `types.ts`. + * 2. Write `column-types/.ts` exporting a `ColumnTypeDefinition`. + * 3. Add it to `COLUMN_TYPE_REGISTRY` below. + * + * Step 3 is not optional and cannot be forgotten: the `Record` + * annotation makes step 1 a **compile error** until the entry exists, and the + * `ColumnTypeDefinition` interface then makes it an error until every field is + * filled in. That is the whole point of this file — adding a type used to mean + * remembering ~40 scattered `switch` arms, each of which failed silently when + * missed. + */ + +import { booleanColumnType } from '@/lib/table/column-types/boolean' +import { currencyColumnType } from '@/lib/table/column-types/currency' +import { dateColumnType } from '@/lib/table/column-types/date' +import { jsonColumnType } from '@/lib/table/column-types/json' +import { numberColumnType } from '@/lib/table/column-types/number' +import { + MULTI_SELECT_OPERATORS, + SINGLE_SELECT_OPERATORS, + selectColumnType, +} from '@/lib/table/column-types/select' +import { stringColumnType } from '@/lib/table/column-types/string' +import type { ColumnType, ColumnTypeDefinition } from '@/lib/table/column-types/types' +import { COLUMN_TYPES, TYPE_SPECIFIC_COLUMN_KEYS } from '@/lib/table/column-types/types' +import type { ColumnDefinition, JsonValue } from '@/lib/table/types' + +export { COLUMN_TYPES } +export { MULTI_SELECT_OPERATORS, SINGLE_SELECT_OPERATORS } + +/** + * Every column type, keyed by id. The annotation is the completeness gate — + * see the module doc. + */ +export const COLUMN_TYPE_REGISTRY: Record = { + string: stringColumnType, + number: numberColumnType, + boolean: booleanColumnType, + date: dateColumnType, + json: jsonColumnType, + select: selectColumnType, + currency: currencyColumnType, +} + +/** Every definition, in the same order as {@link COLUMN_TYPES}. */ +export const ALL_COLUMN_TYPES: readonly ColumnTypeDefinition[] = COLUMN_TYPES.map( + (id) => COLUMN_TYPE_REGISTRY[id] +) + +/** Whether `value` is a known column type. */ +export function isColumnType(value: unknown): value is ColumnType { + // `in` would also match inherited keys, so `'toString'` would type-guard as a + // column type and then resolve to `Function.prototype.toString`. + return typeof value === 'string' && Object.hasOwn(COLUMN_TYPE_REGISTRY, value) +} + +/** + * The definition for a column's type. Falls back to `string` for a column whose + * declared type is not (or is no longer) a known one, so a malformed schema + * renders as text instead of throwing mid-render. + */ +export function columnTypeOf(column: Pick): ColumnTypeDefinition { + return COLUMN_TYPE_REGISTRY[column.type] ?? stringColumnType +} + +/** The definition for a type id, or `string`'s when the id is unknown. */ +export function columnTypeById(type: string | undefined): ColumnTypeDefinition { + return (isColumnType(type) && COLUMN_TYPE_REGISTRY[type]) || stringColumnType +} + +/** + * Whether an existing cell survives a conversion **to** `target`'s type. + * + * Falls back to "whatever the type's `coerce` accepts". Only `select` + * overrides, because its rules (a cleared `''` against `required`, and single + * vs multi cardinality) are about the column, not the value. + */ +export function isValueCompatible(value: unknown, target: ColumnDefinition): boolean { + const definition = columnTypeOf(target) + if (definition.isCompatibleWith) return definition.isCompatibleWith(value, target) + return definition.coerce(value as JsonValue, target).ok +} + +/** This type's own metadata errors; types carrying no metadata report none. */ +export function validateTypeMetadata(column: ColumnDefinition): string[] { + return columnTypeOf(column).validateDefinition?.(column) ?? [] +} + +/** + * A column's type-specific metadata, as a spreadable object. + * + * Callers that copy a column — the API response serializer, the undo snapshot — + * used to name `options`/`multiple`/`currencyCode` by hand, so a new type's + * metadata was stored but silently dropped on the way out. Reading the key list + * keeps them zero-edit. + */ +export function typeMetadataOf(column: ColumnDefinition): Partial { + const metadata: Partial = {} + for (const key of TYPE_SPECIFIC_COLUMN_KEYS) { + if (column[key] !== undefined) Object.assign(metadata, { [key]: column[key] }) + } + return metadata +} + +/** Wire operators a column accepts, or `null` for "all operators". */ +export function filterOperatorsFor(column: ColumnDefinition): ReadonlySet | null { + return columnTypeOf(column).filterOperatorsFor?.(column) ?? null +} diff --git a/apps/sim/lib/table/column-types/select.ts b/apps/sim/lib/table/column-types/select.ts new file mode 100644 index 00000000000..88c3f65ec09 --- /dev/null +++ b/apps/sim/lib/table/column-types/select.ts @@ -0,0 +1,143 @@ +import { TagIcon } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' +import { MAX_SELECT_OPTIONS } from '@/lib/table/constants' +import { + optionIds, + resolveSelectCellValue, + resolveSelectOptionId, + splitMultiSelectInput, +} from '@/lib/table/select-options' +import { selectValueToNames } from '@/lib/table/select-values' +import type { JsonValue } from '@/lib/table/types' + +/** + * Operators that make sense on a `select` column (whose values are opaque option + * ids), split by cardinality. A single-select cell holds one id, so it compares + * for equality; a multi-select cell holds an array of ids, so the question is + * membership — hence contains / does-not-contain. `$eq` against an array cell + * can never be true (`{"t":["a"]} @> {"t":"a"}` is false in Postgres), so + * allowing it would silently match nothing. + */ +export const SINGLE_SELECT_OPERATORS: ReadonlySet = new Set([ + '$eq', + '$ne', + '$in', + '$nin', + '$empty', +]) +export const MULTI_SELECT_OPERATORS: ReadonlySet = new Set([ + '$contains', + '$ncontains', + '$empty', +]) + +export const selectColumnType: ColumnTypeDefinition = { + id: 'select', + label: 'Select', + icon: TagIcon, + // Cells hold opaque option ids; comparison is by id, never by cast. + jsonbCast: null, + storesOpaqueIds: true, + supportsUnique: false, + sampleValue: 'Option', + ownedMetadata: ['options', 'multiple'], + workflowInputType: 'string', + editor: 'select', + expandable: false, + + filterOperatorsFor(column) { + return column.multiple ? MULTI_SELECT_OPERATORS : SINGLE_SELECT_OPERATORS + }, + + coerce(value, column) { + const resolved = resolveSelectCellValue(value, column) + // A multi target always resolves (to `[]` at worst); a single target that + // matches no option has nothing safe to store. + return resolved === null ? { ok: false } : { ok: true, value: resolved } + }, + + validateCell(value, column) { + const ids = optionIds(column) + if (column.multiple) { + if (!Array.isArray(value)) return `${column.name} must be a list of options` + if (!value.every((v) => typeof v === 'string' && ids.has(v))) { + return `${column.name} must only contain defined options` + } + if (column.required && value.length === 0) return `Missing required field: ${column.name}` + return null + } + return typeof value === 'string' && ids.has(value) + ? null + : `${column.name} must be one of the defined options` + }, + + validateDefinition(column) { + const errors: string[] = [] + const options = column.options + if (!Array.isArray(options) || options.length === 0) { + errors.push( + `Column "${column.name}" of type "${column.type}" must define at least one option` + ) + return errors + } + if (options.length > MAX_SELECT_OPTIONS) { + errors.push(`Column "${column.name}" cannot have more than ${MAX_SELECT_OPTIONS} options`) + } + + const ids = new Set() + const names = new Set() + for (const opt of options) { + if (!opt.id || typeof opt.id !== 'string') { + errors.push(`Column "${column.name}" has an option missing an id`) + } else if (ids.has(opt.id)) { + errors.push(`Column "${column.name}" has duplicate option id "${opt.id}"`) + } else { + ids.add(opt.id) + } + if (!opt.name || typeof opt.name !== 'string') { + errors.push(`Column "${column.name}" has an option missing a name`) + } else { + const key = opt.name.toLowerCase() + if (names.has(key)) { + errors.push(`Column "${column.name}" has duplicate option name "${opt.name}"`) + } else { + names.add(key) + } + } + } + return errors + }, + + isCompatibleWith(value, target) { + // A cleared select cell is written as '' — still convertible, unless the + // target is required. Required only rejects null/undefined on a write, so + // a required string column legitimately holds ''; the migration turns that + // into null (or [] for a multi), and every later update of that row would + // then fail its own required check. + if (value === '') return !target.required + // Read the value exactly as the write-path coercion will. A multi target + // splits a comma-delimited string, so a multiselect → text → multiselect + // round-trip (text holding this feature's own `Bug, Docs` export shape) + // stays convertible instead of being rejected as one unknown option. + const parts = target.multiple + ? splitMultiSelectInput(value as JsonValue) + : Array.isArray(value) + ? value + : [value] + // A single-select target can't hold several options. `updateColumnOptions` + // blocks the same transition; without this the next coerce would silently + // keep only the first id. + if (!target.multiple && parts.length > 1) return false + const options = target.options ?? [] + return parts.every((v) => resolveSelectOptionId(v as JsonValue, options) !== null) + }, + + formatForDisplay(value, column) { + const resolved = selectValueToNames(column, value) + return Array.isArray(resolved) ? resolved.join(', ') : (resolved ?? '') + }, + + formatForInput(value, column) { + return selectColumnType.formatForDisplay(value, column) + }, +} diff --git a/apps/sim/lib/table/column-types/string.ts b/apps/sim/lib/table/column-types/string.ts new file mode 100644 index 00000000000..4dc531c7b34 --- /dev/null +++ b/apps/sim/lib/table/column-types/string.ts @@ -0,0 +1,39 @@ +import { TypeText } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' + +export const stringColumnType: ColumnTypeDefinition = { + id: 'string', + label: 'Text', + icon: TypeText, + jsonbCast: null, + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: 'example', + ownedMetadata: [], + workflowInputType: 'string', + editor: 'text', + expandable: true, + + coerce(value) { + if (typeof value === 'string') return { ok: true, value } + if (typeof value === 'number' || typeof value === 'boolean') { + return { ok: true, value: String(value) } + } + return { ok: false } + }, + + validateCell(value, column) { + return typeof value === 'string' ? null : `${column.name} must be string, got ${typeof value}` + }, + + formatForDisplay(value) { + if (typeof value === 'string') return value + if (value === null || value === undefined) return '' + return JSON.stringify(value) + }, + + formatForInput(value) { + if (typeof value === 'object') return JSON.stringify(value) + return String(value) + }, +} diff --git a/apps/sim/lib/table/column-types/types.server.ts b/apps/sim/lib/table/column-types/types.server.ts new file mode 100644 index 00000000000..c650c31c72e --- /dev/null +++ b/apps/sim/lib/table/column-types/types.server.ts @@ -0,0 +1,47 @@ +/** + * The server-only half of a column type: rewriting stored cells when a column + * is converted into or out of this type. + * + * Separate from `types.ts` so the client-safe definition never references a + * drizzle transaction type. Mirrors `connectors/`'s `ConnectorMeta` / + * `ConnectorConfig` split. + */ + +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' +import type { DbTransaction } from '@/lib/table/planner' +import type { ColumnDefinition, JsonValue } from '@/lib/table/types' + +export interface ColumnCellMigrationContext { + trx: DbTransaction + tableId: string + /** JSONB storage key for the column (its stable id). */ + columnKey: string + /** The column definition as it was before the conversion. */ + previous: ColumnDefinition + /** The column definition the table will end up with. */ + target: ColumnDefinition + /** + * Row id → replacement value, for types whose migration is computed in JS + * during the compatibility scan rather than expressed set-based in SQL. + */ + resolved: ReadonlyMap +} + +export type ColumnCellMigration = (context: ColumnCellMigrationContext) => Promise + +export interface ColumnTypeServerDefinition { + /** + * Rewrites cells into this type's canonical storage shape when a column is + * converted **to** it. Omitted when the stored bytes are already correct. + */ + readonly migrateCellsTo?: ColumnCellMigration + /** + * Rewrites cells out of this type's storage shape when a column is converted + * **away from** it. Needed when the stored value is meaningless under any + * other type — a `select`'s option ids, for instance. + */ + readonly migrateCellsFrom?: ColumnCellMigration +} + +/** A column type plus its server-only migrations. */ +export type ColumnTypeServerEntry = ColumnTypeDefinition & ColumnTypeServerDefinition diff --git a/apps/sim/lib/table/column-types/types.ts b/apps/sim/lib/table/column-types/types.ts new file mode 100644 index 00000000000..0de148ac1d0 --- /dev/null +++ b/apps/sim/lib/table/column-types/types.ts @@ -0,0 +1,199 @@ +/** + * The shape of a table column type. + * + * Everything that varies per column type — how it looks, how it stores, how it + * coerces, how it compares in SQL — lives on one of these, so adding a type is + * "write one file and register it" rather than finding ~40 `switch` arms. + * + * Split in two, on the axis that actually constrains us: + * + * - {@link ColumnTypeDefinition} is **client-safe**. It may carry a React icon + * (an icon is a component *reference*; server code never calls it, and + * `scripts/check-client-boundary-imports.ts` only forbids calling a + * `'use client'` export from a server surface). It must NOT reach `@sim/db`, + * `drizzle-orm`, or `next/server` — the tables grid imports it directly. + * - `ColumnTypeServerDefinition` (in `types.server.ts`) adds the one genuinely + * server-only concern: rewriting stored cells inside a transaction. + * + * This mirrors `connectors/types.ts`'s `ConnectorMeta` / `ConnectorConfig` + * split and its `registry.ts` / `registry.server.ts` pair. + */ + +import type React from 'react' +import type { ColumnDefinition, JsonValue } from '@/lib/table/types' + +/** + * Every column type id, in picker order. Declared here — not derived from the + * registry — so `constants.ts` can re-export it without dragging the registry's + * icon imports into the 44 server modules that read the `@/lib/table` barrel. + * + * The registry's `Record` annotation is what keeps the two in + * step: adding an id here fails compilation until both registries have an entry. + */ +export const COLUMN_TYPES = [ + 'string', + 'number', + 'currency', + 'boolean', + 'date', + 'json', + 'select', +] as const + +export type ColumnType = (typeof COLUMN_TYPES)[number] + +/** Which inline editor the grid mounts for a cell of this type. */ +export type ColumnCellEditor = + /** Single-line text input. Numeric types additionally set `inputMode`. */ + | 'text' + /** Calendar + time picker. */ + | 'date' + /** Option dropdown. */ + | 'select' + /** Not editable inline — the grid toggles it in place instead. */ + | 'toggle' + +/** + * Optional `ColumnDefinition` keys that belong to a specific column type rather + * than to every column. Each type declares which it owns; a key appearing on + * any other type is rejected generically, so adding metadata for a new type + * means extending this list and that type's `ownedMetadata` — not editing the + * validator. + */ +export const TYPE_SPECIFIC_COLUMN_KEYS = ['options', 'multiple', 'currencyCode'] as const + +export type TypeSpecificColumnKey = (typeof TYPE_SPECIFIC_COLUMN_KEYS)[number] + +/** Result of coercing a raw value toward a column's declared type. */ +export type CoerceResult = { ok: true; value: JsonValue } | { ok: false } + +export interface ColumnTypeDefinition { + readonly id: ColumnType + + /** Human label in the type picker, column header menu, and docs. */ + readonly label: string + /** Type icon. A component reference only — never invoked server-side. */ + readonly icon: React.ComponentType<{ className?: string }> + /** + * Postgres cast needed to compare this type's JSONB text, or `null` when text + * comparison is correct. Single source for both filter ranges and sort order. + */ + readonly jsonbCast: 'numeric' | 'timestamptz' | null + + /** + * Wire operators a column of this type accepts, or `null` for "all + * operators". Only types whose stored value is opaque (a `select`'s option + * id) need to restrict. Takes the column, so a type whose answer depends on + * its own configuration — select's single vs multi cardinality — owns that + * rule instead of the registry special-casing it. + */ + filterOperatorsFor?(column: ColumnDefinition): ReadonlySet | null + + /** + * True when the stored value is an opaque identifier that must be resolved to + * a display label for search, filtering, export, and clipboard. Only `select` + * sets this; it is why those paths special-case it. + */ + readonly storesOpaqueIds: boolean + + /** + * Whether a column of this type can carry a `unique` constraint. False for + * types whose stored value is opaque: uniqueness would compare the stored + * option id, capping each option at one row for the whole table. + */ + readonly supportsUnique: boolean + + /** + * A representative value, used to show an LLM what this column's cells look + * like. Keeps prompt examples from restating per-type knowledge. + */ + readonly sampleValue: JsonValue + + /** + * Optional `ColumnDefinition` keys this type owns. Any type-specific key + * present on a column of a *different* type is rejected, generically — a + * stored `multiple` or `currencyCode` on the wrong type is inert until a + * later conversion inherits it and silently overrides what that request + * asked for. Declaring ownership here is what lets a new type add metadata + * without touching the validator. + */ + readonly ownedMetadata: readonly TypeSpecificColumnKey[] + + /** Workflow/block param type a column of this type maps onto. */ + readonly workflowInputType: 'string' | 'number' | 'boolean' | 'object' + + /** Inline editor variant. */ + readonly editor: ColumnCellEditor + /** + * Whether double-clicking a cell opens the large expanded popover instead of + * the compact inline editor. True for free-form prose (`string`, `json`) + * where a cell can hold far more than one line; false for types with a + * bounded, structured value. + */ + readonly expandable: boolean + /** `inputMode` for the text editor, when the type wants a specific keypad. */ + readonly inputMode?: 'decimal' + /** + * Whether the editor must accept text an `` would reject. + * A currency cell legitimately takes `$1,234.56` or `1.234,56`, so it needs a + * text input with a numeric keypad; a plain number takes neither and keeps + * the native numeric input with its spinner and validation. + */ + readonly acceptsFormattedInput?: boolean + /** + * Keys that may start a type-ahead edit. Absent means any printable key + * starts one — only types that parse their input restrict it, so a stray + * letter can't open an editor whose draft could never save. + */ + readonly typeaheadPattern?: RegExp + /** + * Message shown when a draft cannot be parsed. Absent means any text is + * valid, so a draft always saves. + */ + readonly parseErrorMessage?: string + + /** + * Coerces a non-null raw value toward this type. The single write-path + * implementation — the server calls it before persisting and the grid calls + * it to fill the optimistic cache, so the two can no longer disagree. + */ + coerce(value: JsonValue, column: ColumnDefinition): CoerceResult + + /** Validates a stored cell's shape. Returns an error message, or null when valid. */ + validateCell(value: JsonValue, column: ColumnDefinition): string | null + + /** + * Validates this type's own column metadata (a `select`'s options, a + * `currency`'s code). Omitted by types that carry none. + */ + validateDefinition?(column: ColumnDefinition): string[] + + /** + * Whether an existing cell survives a conversion **to** this type. + * + * Defaults to "whatever {@link coerce} accepts", which is what makes the + * retype gate and the write path incapable of disagreeing — a gate that is + * more permissive than the write path reports zero incompatible rows and + * then rewrites every one of them. + * + * An override may only ever be **stricter** than `coerce`, never looser. + * Stricter is safe: the bulk conversion is refused while individual writes + * still work. Looser is the direction that corrupts data. Use it when a + * coercion that is reasonable for a single deliberate write would be + * destructive applied to a whole column at once. + */ + isCompatibleWith?(value: unknown, target: ColumnDefinition): boolean + + /** Stored value → display text (grid cell, CSV, clipboard, width measurement). */ + formatForDisplay(value: unknown, column: ColumnDefinition): string + + /** Stored value → the text an editor input starts with. */ + formatForInput(value: unknown, column: ColumnDefinition): string + + /** + * Metadata stamped onto a newly created column of this type, so the schema + * states the type's configuration explicitly instead of leaving readers to + * know the default. Returns nothing for types that carry no metadata. + */ + defaultMetadata?(column: ColumnDefinition): Partial +} diff --git a/apps/sim/lib/table/columns/service.ts b/apps/sim/lib/table/columns/service.ts index 4226dc935f9..d8f627fc117 100644 --- a/apps/sim/lib/table/columns/service.ts +++ b/apps/sim/lib/table/columns/service.ts @@ -12,9 +12,22 @@ import { db } from '@sim/db' import { userTableDefinitions, userTableRows } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import { omit } from '@sim/utils/object' import { and, count, eq, sql } from 'drizzle-orm' import { columnMatchesRef, generateColumnId, getColumnId } from '@/lib/table/column-keys' +import { + columnTypeById, + columnTypeOf, + isValueCompatible, + TYPE_SPECIFIC_COLUMN_KEYS, +} from '@/lib/table/column-types' +import { + migrationFrom, + migrationTo, + writeBackCoercedCells, +} from '@/lib/table/column-types/registry.server' import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' +import { resolveCurrencyCode } from '@/lib/table/currency' import { assertColumnDestructive, assertSchemaMutable } from '@/lib/table/mutation-locks' import type { DbTransaction } from '@/lib/table/planner' import { stripGroupExecutions } from '@/lib/table/rows/executions' @@ -32,14 +45,11 @@ import type { TableMetadata, TableSchema, UpdateColumnConstraintsData, + UpdateColumnCurrencyData, UpdateColumnOptionsData, UpdateColumnTypeData, } from '@/lib/table/types' -import { - resolveSelectOptionId, - splitMultiSelectInput, - validateColumnDefinition, -} from '@/lib/table/validation' +import { validateColumnDefinition } from '@/lib/table/validation' import { assertValidSchema, stripGroupDeps } from '@/lib/table/workflow-columns' const logger = createLogger('TableColumnService') @@ -64,6 +74,7 @@ export async function addTableColumn( position?: number options?: SelectOption[] multiple?: boolean + currencyCode?: string }, requestId: string ): Promise { @@ -108,6 +119,7 @@ export async function addTableColumn( unique: column.unique ?? false, ...(column.options ? { options: column.options } : {}), ...(column.multiple ? { multiple: true } : {}), + ...columnTypeById(column.type).defaultMetadata?.(column as ColumnDefinition), } const columnValidation = validateColumnDefinition(newColumn) @@ -474,6 +486,181 @@ export async function deleteColumns( return def } +/** + * Validates a constraint change against the column's stored data, and returns + * the column with those constraints applied. + * + * Shared by every write that can carry constraints, for the reason the + * duplicate scan and {@link countEmptyCells} are shared: three copies of these + * rules is the drift that produced the original required-check bug. Applying + * them in the same write as the change they accompany is what stops a combined + * request from committing one half and then failing on the other. + */ +async function applyConstraints( + trx: DbTransaction, + tableId: string, + column: ColumnDefinition, + columnKey: string, + data: { required?: boolean; unique?: boolean } +): Promise { + if (data.required === undefined && data.unique === undefined) return column + + if (column.workflowGroupId) { + throw new Error( + `Cannot change constraints on workflow-output column "${column.name}". Constraints aren't applicable to columns whose values come from workflow execution.` + ) + } + if (data.required === true && !column.required) { + const emptyCount = await countEmptyCells(trx, tableId, columnKey) + if (emptyCount > 0) { + throw new Error( + `Cannot set column "${column.name}" as required: ${emptyCount} row(s) have null, missing, or empty values` + ) + } + } + if (data.unique === true && !column.unique) { + if (!columnTypeOf(column).supportsUnique) { + throw new Error( + `Cannot set column "${column.name}" as unique: ${column.type} columns compare stored values that would allow only one row per value.` + ) + } + if (await hasDuplicateValues(trx, tableId, columnKey)) { + throw new Error(`Cannot set column "${column.name}" as unique: duplicate values exist`) + } + } + return { + ...column, + ...(data.required !== undefined ? { required: data.required } : {}), + ...(data.unique !== undefined ? { unique: data.unique } : {}), + } +} + +/** Persists a column list as the table's schema and returns the updated definition. */ +async function persistColumns( + trx: DbTransaction, + table: TableDefinition, + columns: ColumnDefinition[] +): Promise { + const updatedSchema: TableSchema = { ...table.schema, columns } + const now = new Date() + await trx + .update(userTableDefinitions) + .set({ schema: updatedSchema, updatedAt: now }) + .where(eq(userTableDefinitions.id, table.id)) + return { ...table, schema: updatedSchema, updatedAt: now } +} + +/** + * Whether any two rows share a stored value in this column. + * + * Shared by the constraint write and the retype's pre-validation so the two + * cannot drift — the same reason {@link countEmptyCells} is shared. A retype + * that sets `unique` in the same request has to run this against the values the + * conversion is ABOUT to write, not the ones on disk: coercing `"5"` and `"5.0"` + * to a number manufactures a duplicate that no pre-scan of the raw text sees. + */ +async function hasDuplicateValues( + trx: DbTransaction, + tableId: string, + columnKey: string +): Promise { + const duplicates = (await trx.execute( + sql`SELECT ${userTableRows.data}->>${columnKey}::text AS val, count(*) AS cnt FROM ${userTableRows} WHERE table_id = ${tableId} AND ${userTableRows.data} ? ${columnKey} AND ${userTableRows.data}->>${columnKey}::text IS NOT NULL GROUP BY val HAVING count(*) > 1 LIMIT 1` + )) as { val: string; cnt: number }[] + return duplicates.length > 0 +} + +/** + * Validates a pending rename against the schema it will land in, and returns + * the renamed column. + * + * Exists so a rename can be folded into whatever OTHER column write a request + * carries, inside that write's transaction. Each write is its own locked + * transaction, so a standalone rename alongside one of them means either order + * can commit and then fail — and since a rename is metadata-only (rows key on + * the stable column id), there is nothing forcing it to be its own write. + * + * Returns the column unchanged when there is no rename to apply. Exported so + * the collision and name-shape rules are testable without a transaction. + */ +export function applyPendingRename( + columns: ColumnDefinition[], + columnIndex: number, + newName: string | undefined +): ColumnDefinition { + const column = columns[columnIndex] + if (newName === undefined || newName === column.name) return column + + if (!NAME_PATTERN.test(newName)) { + throw new Error( + `Invalid column name "${newName}". Column names must start with a letter or underscore, followed by alphanumeric characters or underscores.` + ) + } + if (newName.length > TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH) { + throw new Error( + `Column name exceeds maximum length (${TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH} characters)` + ) + } + if (columns.some((c, i) => i !== columnIndex && c.name.toLowerCase() === newName.toLowerCase())) { + throw new Error(`Column "${newName}" already exists`) + } + return { ...column, name: newName } +} + +/** + * The column definition a retype produces: prior per-type metadata dropped, + * then only what the TARGET type declares it owns carried forward, then that + * type's own defaults stamped on. + */ +function buildConvertedColumn( + column: ColumnDefinition, + data: UpdateColumnTypeData, + { isSelectType, targetMultiple }: { isSelectType: boolean; targetMultiple: boolean } +): ColumnDefinition { + // Strip EVERY type-specific key generically, so a future type's metadata + // cannot ride through `...rest` onto a target that does not own it — which + // `validateColumnDefinition` would then reject on every later write. + const rest = omit(column, [...TYPE_SPECIFIC_COLUMN_KEYS]) as ColumnDefinition + // Constraints arriving with the retype are APPLIED here, not left to a second + // transaction. `updateColumnType` already validates against them (empty cells + // for `required`, post-conversion duplicates for `unique`), so applying them + // in the same write is what makes a combined request all-or-nothing. + const withConstraints: ColumnDefinition = { + ...rest, + ...(data.required !== undefined ? { required: data.required } : {}), + ...(data.unique !== undefined ? { unique: data.unique } : {}), + } + + if (isSelectType) { + return { + ...withConstraints, + type: data.newType, + options: data.options ?? column.options, + ...(targetMultiple ? { multiple: true } : {}), + // Select columns carry no unique constraint: it would compare the stored + // option id, capping each option at one row table-wide, and the UI hides + // the toggle so it could never be cleared again. Dropped here rather than + // in each caller — the sidebar was the only one clearing it, leaving the + // v1 and agent paths to strand it. + unique: false, + } + } + + // Then carry back only the keys the TARGET type declares it owns, preferring + // the value this request supplied over the column's existing one. Iterating + // the key list rather than naming keys is what keeps this zero-edit for a + // future type. + const definition = columnTypeById(data.newType) + const owned = new Set(definition.ownedMetadata) + const carried: ColumnDefinition = { ...withConstraints, type: data.newType } + for (const key of TYPE_SPECIFIC_COLUMN_KEYS) { + if (!owned.has(key)) continue + const value = data[key] ?? column[key] + if (value !== undefined) Object.assign(carried, { [key]: value }) + } + return { ...carried, ...definition.defaultMetadata?.(carried) } +} + /** * Changes the type of a column. Validates that existing data is compatible. * @@ -514,7 +701,30 @@ export async function updateColumnType( const column = schema.columns[columnIndex] if (column.type === data.newType) { - return table + // Callers gate on the type actually changing, but they compute that from + // a schema read taken before this transaction took the lock — so a + // concurrent change can land us here with real work still to do. Only a + // rename can be honoured without a conversion; anything else would be + // silently discarded, and answering success for a change that never + // happened is the worst outcome available. + const carriesOtherWork = + data.required !== undefined || + data.unique !== undefined || + data.options !== undefined || + data.multiple !== undefined || + data.currencyCode !== undefined + if (carriesOtherWork) { + throw new Error( + `Column "${column.name}" is already type "${data.newType}"; re-issue the request without a type change.` + ) + } + const renamed = applyPendingRename(schema.columns, columnIndex, data.newName) + if (renamed === column) return table + return persistColumns( + trx, + table, + schema.columns.map((c, i) => (i === columnIndex ? renamed : c)) + ) } const columnKey = getColumnId(column) @@ -557,8 +767,34 @@ export async function updateColumnType( } } + /** + * The column definition the table ends up with. Built before the scan so + * the coercion below reads the same metadata (option set, currency) the + * stored value will be validated against afterwards. + */ + const convertedColumn = buildConvertedColumn(column, data, { + isSelectType, + targetMultiple: !!targetMultiple, + }) + let incompatibleCount = 0 let blankCount = 0 + /** + * Row id → the value the cell must END UP holding. + * + * Collected during the compatibility scan rather than re-derived later, so + * it reads the same `effective` value the check accepted — which for a + * `select` source is the option name, not the stored id. + * + * Load-bearing: a conversion is allowed exactly when the target type's + * `coerce` accepts the value, and `coerce` frequently *transforms* it (an + * epoch number becomes an ISO date, a formatted amount becomes a number). + * Without writing the transformed value back, the cell keeps its old bytes + * under the new type — and since filters and sorts apply the type's + * `jsonbCast` to whatever is stored, an epoch left in a `date` column makes + * `::timestamptz` fail on EVERY query against it. + */ + const coercedByRowId = new Map() for (const row of rows) { const rowData = row.data as RowData const value = rowData[columnKey] @@ -575,8 +811,28 @@ export async function updateColumnType( targetRequired ) ) { - if (effective === null || effective === '') blankCount++ - else incompatibleCount++ + // A cell the target cannot read but that is merely EMPTY is not a + // conversion failure — the write path already turns an unreadable value + // into null on an optional column, so the conversion does the same. Only + // a required target has a real problem with it, and the guard above has + // already reported those. Blocking here meant a text column with a + // single blank cell could not be converted to a number at all. + if (effective === null || effective === '') { + if (targetRequired) blankCount++ + else coercedByRowId.set(row.id, null) + } else { + incompatibleCount++ + } + continue + } + + // `select` keeps its own id↔name migrations; everything else writes back + // whatever `coerce` produced, when that differs from what is stored. + if (!isSelectType && effective !== null) { + const coerced = columnTypeById(data.newType).coerce(effective as JsonValue, convertedColumn) + if (coerced.ok && !Object.is(coerced.value, value)) { + coercedByRowId.set(row.id, coerced.value) + } } } @@ -592,25 +848,10 @@ export async function updateColumnType( ) } - const updatedColumns = schema.columns.map((c, i) => { - if (i !== columnIndex) return c - // Drop any prior select config, then re-add when the target type uses it. - const { options: _prevOptions, multiple: _prevMultiple, ...rest } = c - return isSelectType - ? { - ...rest, - type: data.newType, - options: data.options ?? c.options, - ...(targetMultiple ? { multiple: true } : {}), - // Select columns carry no unique constraint: it would compare the - // stored option id, capping each option at one row table-wide, and - // the UI hides the toggle so it could never be cleared again. Drop - // it here rather than in each caller — the sidebar was the only one - // clearing it, leaving the v1 and agent paths to strand it. - unique: false, - } - : { ...rest, type: data.newType } - }) + const renamedColumns = schema.columns.map((c, i) => (i === columnIndex ? convertedColumn : c)) + const updatedColumns = renamedColumns.map((c, i) => + i === columnIndex ? applyPendingRename(renamedColumns, columnIndex, data.newName) : c + ) const columnValidation = validateColumnDefinition(updatedColumns[columnIndex]) if (!columnValidation.valid) { @@ -620,10 +861,37 @@ export async function updateColumnType( const updatedSchema: TableSchema = { ...schema, columns: updatedColumns } const now = new Date() - if (convertingAwayFromSelect) { - await migrateSelectCellsToNames(trx, data.tableId, columnKey, column.options ?? []) - } else if (isSelectType) { - await migrateCellsToSelectIds(trx, data.tableId, columnKey, targetOptions, !!targetMultiple) + // Cell rewrites are owned by the column-type registry, keyed by direction. + // Outbound runs first: leaving `select` turns opaque option ids into names, + // which is the form the inbound migration (if any) then reads. + const migrationContext = { + trx, + tableId: data.tableId, + columnKey, + previous: column, + target: updatedColumns[columnIndex], + resolved: coercedByRowId, + } + await migrationFrom(column.type)?.(migrationContext) + if (isSelectType) { + await migrationTo(data.newType)?.(migrationContext) + } else { + await writeBackCoercedCells(trx, data.tableId, columnKey, coercedByRowId) + } + + // A `unique` arriving with this retype is validated HERE, against the values + // the conversion just wrote — not by the separate constraint write that + // follows. The conversion itself manufactures duplicates that no scan of the + // pre-conversion data can see (`"5"` and `"5.0"` both coerce to `5`), and + // that write runs in its own transaction, so discovering it there would + // report an error with the retype already committed and the original text + // irrecoverably rewritten. + if (data.unique === true && !column.unique) { + if (await hasDuplicateValues(trx, data.tableId, columnKey)) { + throw new Error( + `Cannot change column "${column.name}" to type "${data.newType}" and set it as unique: the converted values contain duplicates.` + ) + } } await trx @@ -672,44 +940,10 @@ export async function updateColumnConstraints( const column = schema.columns[columnIndex] const columnKey = getColumnId(column) - if (column.workflowGroupId) { - throw new Error( - `Cannot change constraints on workflow-output column "${column.name}". Constraints aren't applicable to columns whose values come from workflow execution.` - ) - } - if (data.required === true && !column.required) { - const emptyCount = await countEmptyCells(trx, data.tableId, columnKey) - if (emptyCount > 0) { - throw new Error( - `Cannot set column "${column.name}" as required: ${emptyCount} row(s) have null, missing, or empty values` - ) - } - } - - if (data.unique === true && column.type === 'select') { - throw new Error( - `Cannot set column "${column.name}" as unique: select columns compare stored option ids, which would allow only one row per option.` - ) - } - - if (data.unique === true && !column.unique) { - const duplicates = (await trx.execute( - sql`SELECT ${userTableRows.data}->>${columnKey}::text AS val, count(*) AS cnt FROM ${userTableRows} WHERE table_id = ${data.tableId} AND ${userTableRows.data} ? ${columnKey} AND ${userTableRows.data}->>${columnKey}::text IS NOT NULL GROUP BY val HAVING count(*) > 1 LIMIT 1` - )) as { val: string; cnt: number }[] - - if (duplicates.length > 0) { - throw new Error(`Cannot set column "${column.name}" as unique: duplicate values exist`) - } - } - - const updatedColumns = schema.columns.map((c, i) => - i === columnIndex - ? { - ...c, - ...(data.required !== undefined ? { required: data.required } : {}), - ...(data.unique !== undefined ? { unique: data.unique } : {}), - } - : c + const constrained = await applyConstraints(trx, data.tableId, column, columnKey, data) + const withConstraints = schema.columns.map((c, i) => (i === columnIndex ? constrained : c)) + const updatedColumns = withConstraints.map((c, i) => + i === columnIndex ? applyPendingRename(withConstraints, columnIndex, data.newName) : c ) const updatedSchema: TableSchema = { ...schema, columns: updatedColumns } const now = new Date() @@ -762,29 +996,16 @@ export async function updateColumnOptions( throw new Error(`Invalid column: ${columnValidation.errors.join('; ')}`) } - const updatedColumns = schema.columns.map((c, i) => (i === columnIndex ? updatedColumn : c)) - const updatedSchema: TableSchema = { ...schema, columns: updatedColumns } - const now = new Date() - const nextMultiple = !!(data.multiple ?? column.multiple) const wasMultiple = !!column.multiple const keptIds = new Set(data.options.map((o) => o.id)) const removedAny = (column.options ?? []).some((o) => !keptIds.has(o.id)) const togglingCardinality = nextMultiple !== wasMultiple + // The constraint the column ENDS UP with, which may be arriving in this same + // request. `applyConstraints` validates and applies it below, after the cell + // migrations; the checks in between need to read the target value. const targetRequired = !!(data.required ?? column.required) - // Newly imposing `required` in the same request: rows that are ALREADY empty - // would fail the separate constraint write after this one commits, so they - // have to be caught here, through the same predicate that write will use. - if (targetRequired && !column.required) { - const emptyCount = await countEmptyCells(trx, data.tableId, columnKey) - if (emptyCount > 0) { - throw new Error( - `Cannot make column "${column.name}" required: ${emptyCount} row(s) have null, missing, or empty values. Fill them first.` - ) - } - } - if (togglingCardinality || removedAny) { const timeoutMs = scaledStatementTimeoutMs(table.rowCount ?? 0, { baseMs: 60_000, @@ -859,157 +1080,120 @@ export async function updateColumnOptions( // scalar, so leaving cells un-normalized would silently drop every // pre-toggle row out of its own column's filters. if (togglingCardinality) { - await migrateCellsToSelectIds(trx, data.tableId, columnKey, data.options, nextMultiple) + // Same registry migration the retype path uses — `updatedColumn` already + // carries the post-toggle `options`/`multiple`, which is all it reads. + await migrationTo('select')?.({ + trx, + tableId: data.tableId, + columnKey, + previous: column, + target: updatedColumn, + resolved: new Map(), + }) } - await trx - .update(userTableDefinitions) - .set({ schema: updatedSchema, updatedAt: now }) - .where(eq(userTableDefinitions.id, data.tableId)) + // Constraints are validated and applied AFTER the migrations above, because + // those migrations rewrite stored values — a `unique` scan run before them + // would read the pre-migration shape and pass, and the migration could then + // produce the duplicates it was meant to prevent. + const constrainedColumn = await applyConstraints( + trx, + data.tableId, + updatedColumn, + columnKey, + data + ) + const withOptions = schema.columns.map((c, i) => (i === columnIndex ? constrainedColumn : c)) + const updatedColumns = withOptions.map((c, i) => + i === columnIndex ? applyPendingRename(withOptions, columnIndex, data.newName) : c + ) + + const updated = await persistColumns(trx, table, updatedColumns) logger.info( `[${requestId}] Updated options for column "${column.name}" in table ${data.tableId}` ) - return { ...table, schema: updatedSchema, updatedAt: now } + return updated }) } /** - * Rewrites a column's cells from stored option **ids** to option **names**, for - * a column that is ceasing to be a `select`. A multi cell joins comma-separated - * — the same shape it exports as. + * Changes the currency a `currency` column renders in. * - * An id whose option no longer exists becomes null, matching - * {@link selectValueForConversion}, which is what the compatibility check ran - * on. Passing it through instead would leave an opaque `opt_…` in a typed cell - * the check had already accounted as empty. + * Deliberately the cheapest column mutation in this module: cells store a bare + * number, so re-denominating a column touches only the schema — no row rewrite, + * no compatibility scan, no scaled timeouts. It notably does **not** convert + * amounts between currencies; `1000` stays `1000`, now labelled in the new code. * - * Set-based: one statement per stored shape, driven by a jsonb id→name map, so - * cost is independent of row count. + * @param data - Column + target ISO 4217 code + * @param requestId - Request ID for logging + * @returns Updated table definition + * @throws Error if the table or column is missing, or the column is not a currency column */ -async function migrateSelectCellsToNames( - trx: DbTransaction, - tableId: string, - columnKey: string, - options: SelectOption[] -): Promise { - const nameById = JSON.stringify(Object.fromEntries(options.map((o) => [o.id, o.name]))) - await trx.execute( - sql`UPDATE ${userTableRows} - SET data = jsonb_set(data, ARRAY[${columnKey}::text], - COALESCE(${nameById}::jsonb -> (data->>${columnKey}::text), 'null'::jsonb)) - WHERE table_id = ${tableId} - AND jsonb_typeof(data->${columnKey}::text) = 'string'` - ) - await trx.execute( - sql`UPDATE ${userTableRows} - SET data = jsonb_set(data, ARRAY[${columnKey}::text], COALESCE(to_jsonb(( - SELECT string_agg(${nameById}::jsonb ->> e.v, ', ' ORDER BY e.ord) - FROM jsonb_array_elements_text(data->${columnKey}::text) WITH ORDINALITY AS e(v, ord) - WHERE ${nameById}::jsonb ? e.v - )), 'null'::jsonb)) - WHERE table_id = ${tableId} - AND jsonb_typeof(data->${columnKey}::text) = 'array'` - ) -} +export async function updateColumnCurrency( + data: UpdateColumnCurrencyData, + requestId: string +): Promise { + return withLockedTable(data.tableId, async (table, trx) => { + assertSchemaMutable(table) -/** - * Rewrites a column's cells into the canonical `select` storage shape: the - * option **id**, wrapped in an array when the column is `multiple`. - * - * Needed in both directions of a select change. Converting *to* select, cells - * hold option names (that is what made them compatible) but every reader — - * pills, filters, exports — resolves by id. Toggling single→multi, cells hold a - * scalar id while multi filters compile to array containment, which a scalar - * never matches. Either way the cell silently drops out until it is re-edited. - * - * The map keys ids, names, and lower-cased names — ids so re-running is a no-op, - * lower-cased names because `resolveSelectOptionId` accepts a case-mismatched - * name and a cell that passed that check must actually migrate. Duplicate option - * names are rejected case-insensitively at validation, so the folded key is - * unambiguous; lookups still try the exact form first to preserve its precedence. - */ -async function migrateCellsToSelectIds( - trx: DbTransaction, - tableId: string, - columnKey: string, - options: SelectOption[], - multiple: boolean -): Promise { - // Ids are written last so an id always outranks any option's name, matching - // `resolveSelectOptionId`'s id-before-name precedence. A single flat pass - // would let a later option whose *name* equals an earlier option's *id* - // overwrite that id entry and repoint its cells at the wrong option. - // A Map, not plain-object assignment: an option named `__proto__` would set - // the prototype instead of an own key and drop out of the serialized map. - const refs = new Map() - for (const o of options) { - refs.set(o.name.toLowerCase(), o.id) - refs.set(o.name, o.id) - } - for (const o of options) { - refs.set(o.id, o.id) - } - const idByRef = JSON.stringify(Object.fromEntries(refs)) + const schema = table.schema + const columnIndex = schema.columns.findIndex((c) => columnMatchesRef(c, data.columnName)) + if (columnIndex === -1) { + throw new Error(`Column "${data.columnName}" not found`) + } - if (multiple) { - // A string cell reaching a multi target is either one option name or the - // comma-joined form a multiselect converts to text as. Try the whole string - // first so an option whose own name contains a comma still wins, then split - // — mirroring `splitMultiSelectInput` on the write path, including its - // first-occurrence dedup. - await trx.execute( - sql`UPDATE ${userTableRows} - SET data = jsonb_set(data, ARRAY[${columnKey}::text], - CASE WHEN data->>${columnKey}::text = '' THEN '[]'::jsonb - WHEN COALESCE(${idByRef}::jsonb -> (data->>${columnKey}::text), ${idByRef}::jsonb -> lower(data->>${columnKey}::text)) IS NOT NULL - THEN jsonb_build_array(COALESCE(${idByRef}::jsonb -> (data->>${columnKey}::text), ${idByRef}::jsonb -> lower(data->>${columnKey}::text))) - ELSE COALESCE(( - SELECT jsonb_agg(v ORDER BY ord) FROM ( - SELECT COALESCE(${idByRef}::jsonb -> btrim(part), ${idByRef}::jsonb -> lower(btrim(part)), to_jsonb(btrim(part))) AS v, - min(o) AS ord - FROM unnest(string_to_array(data->>${columnKey}::text, ',')) WITH ORDINALITY AS u(part, o) - WHERE btrim(part) <> '' - GROUP BY 1 - ) d), '[]'::jsonb) - END) - WHERE table_id = ${tableId} - AND jsonb_typeof(data->${columnKey}::text) = 'string'` + const column = schema.columns[columnIndex] + if (column.type !== 'currency') { + throw new Error(`Cannot set currency on column "${column.name}" of type "${column.type}"`) + } + + const updatedColumn: ColumnDefinition = { + ...column, + currencyCode: resolveCurrencyCode(data.currencyCode), + } + const columnValidation = validateColumnDefinition(updatedColumn) + if (!columnValidation.valid) { + throw new Error(`Invalid column: ${columnValidation.errors.join('; ')}`) + } + + const constrained = await applyConstraints( + trx, + data.tableId, + updatedColumn, + getColumnId(column), + data ) - await trx.execute( - sql`UPDATE ${userTableRows} - SET data = jsonb_set(data, ARRAY[${columnKey}::text], COALESCE(( - SELECT jsonb_agg(COALESCE(${idByRef}::jsonb -> e.v, ${idByRef}::jsonb -> lower(e.v), to_jsonb(e.v)) ORDER BY e.ord) - FROM jsonb_array_elements_text(data->${columnKey}::text) WITH ORDINALITY AS e(v, ord) - ), '[]'::jsonb)) - WHERE table_id = ${tableId} - AND jsonb_typeof(data->${columnKey}::text) = 'array'` + + // Only a no-op when nothing at all changed — currency, constraints, name. + const renamePending = data.newName !== undefined && data.newName !== column.name + if ( + constrained === updatedColumn && + updatedColumn.currencyCode === column.currencyCode && + !renamePending + ) { + return table + } + + const withCurrency = schema.columns.map((c, i) => (i === columnIndex ? constrained : c)) + const updatedColumns = withCurrency.map((c, i) => + i === columnIndex ? applyPendingRename(withCurrency, columnIndex, data.newName) : c ) - return - } + const updatedSchema: TableSchema = { ...schema, columns: updatedColumns } + const now = new Date() - // A cleared cell is stored as '' — compatibility lets it through, so it has - // to land as null rather than an '' that fails option membership on the next - // write. - await trx.execute( - sql`UPDATE ${userTableRows} - SET data = jsonb_set(data, ARRAY[${columnKey}::text], - CASE WHEN data->>${columnKey}::text = '' THEN 'null'::jsonb - ELSE COALESCE(${idByRef}::jsonb -> (data->>${columnKey}::text), ${idByRef}::jsonb -> lower(data->>${columnKey}::text), data->${columnKey}::text) - END) - WHERE table_id = ${tableId} - AND jsonb_typeof(data->${columnKey}::text) = 'string'` - ) - // Compatibility already rejected multi-valued cells for a single target, so - // any array here holds at most one option. - await trx.execute( - sql`UPDATE ${userTableRows} - SET data = jsonb_set(data, ARRAY[${columnKey}::text], - COALESCE(${idByRef}::jsonb -> (data->${columnKey}::text->>0), ${idByRef}::jsonb -> lower(data->${columnKey}::text->>0), data->${columnKey}::text->0, 'null'::jsonb)) - WHERE table_id = ${tableId} - AND jsonb_typeof(data->${columnKey}::text) = 'array'` - ) + await trx + .update(userTableDefinitions) + .set({ schema: updatedSchema, updatedAt: now }) + .where(eq(userTableDefinitions.id, data.tableId)) + + logger.info( + `[${requestId}] Set currency for column "${column.name}" to "${updatedColumn.currencyCode}" in table ${data.tableId}` + ) + + return { ...table, schema: updatedSchema, updatedAt: now } + }) } /** @@ -1164,58 +1348,12 @@ export function isValueCompatibleWithType( targetRequired = false ): boolean { if (value === null || value === undefined) return true - - switch (targetType) { - case 'string': - // Arrays and objects can't become text — the write-path coercion rejects - // them and would null the cell. Multi-select values are flattened before - // this check, so anything still structured here is genuinely lossy. - return typeof value !== 'object' - case 'select': { - // A cleared select cell is written as '' — still convertible, unless the - // target is required. Required only rejects null/undefined on a write, so - // a required string column legitimately holds ''; the migration turns that - // into null (or [] for a multi), and every later update of that row would - // then fail its own required check. - if (value === '') return !targetRequired - // Read the value exactly as the write-path coercion will. A multi target - // splits a comma-delimited string, so a multiselect → text → multiselect - // round-trip (text holding this feature's own `Bug, Docs` export shape) - // stays convertible instead of being rejected as one unknown option. - const parts = targetMultiple - ? splitMultiSelectInput(value as JsonValue) - : Array.isArray(value) - ? value - : [value] - // A single-select target can't hold several options. `updateColumnOptions` - // blocks the same transition; without this the next coerce would silently - // keep only the first id. - if (!targetMultiple && parts.length > 1) return false - return parts.every((v) => resolveSelectOptionId(v as JsonValue, targetOptions) !== null) - } - case 'number': { - if (typeof value === 'number') return Number.isFinite(value) - if (typeof value === 'string') { - const num = Number(value) - return Number.isFinite(num) && value.trim() !== '' - } - return false - } - case 'boolean': { - if (typeof value === 'boolean') return true - if (typeof value === 'string') - return ['true', 'false', '1', '0'].includes(value.toLowerCase()) - if (typeof value === 'number') return value === 0 || value === 1 - return false - } - case 'date': { - if (value instanceof Date) return !Number.isNaN(value.getTime()) - if (typeof value === 'string') return !Number.isNaN(Date.parse(value)) - return false - } - case 'json': - return true - default: - return false - } + // Each type reads only the metadata it owns. + return isValueCompatible(value, { + name: '', + type: targetType, + options: targetOptions, + multiple: targetMultiple, + required: targetRequired, + }) } diff --git a/apps/sim/lib/table/constants.ts b/apps/sim/lib/table/constants.ts index 1384d3dc181..216e1e826e5 100644 --- a/apps/sim/lib/table/constants.ts +++ b/apps/sim/lib/table/constants.ts @@ -163,7 +163,17 @@ export function getTablePlanLimits(): TablePlanLimitsByPlan { } } -export const COLUMN_TYPES = ['string', 'number', 'boolean', 'date', 'json', 'select'] as const +/** + * Re-exported from the column-type module, which is the single source of truth. + * Kept here because this is where callers already import it from — restating + * the list would let the two drift, which is the class of bug the registry + * exists to remove. + * + * Points at `types` rather than `registry` on purpose: this module is re- + * exported by the `@/lib/table` barrel that 44 server modules import, and the + * registry pulls in `@sim/emcn/icons`. + */ +export { COLUMN_TYPES } from '@/lib/table/column-types/types' /** Maximum number of options a `select`/`multiselect` column may declare. */ export const MAX_SELECT_OPTIONS = 100 diff --git a/apps/sim/lib/table/currency.ts b/apps/sim/lib/table/currency.ts new file mode 100644 index 00000000000..c4aeb97e09b --- /dev/null +++ b/apps/sim/lib/table/currency.ts @@ -0,0 +1,326 @@ +/** + * Helpers for the `currency` column type. + * + * A currency cell stores a **plain JSON number** — the same storage shape as a + * `number` column — and the column carries a `currencyCode` (ISO 4217) as pure + * display metadata. That split is deliberate: filtering, sorting, uniqueness, + * and CSV export all reuse the numeric paths unchanged, changing a column's + * currency never rewrites a single cell, and the public row output stays a + * number rather than a locale-formatted string consumers would have to reparse. + */ + +/** Currency assumed when a column declares none. */ +export const DEFAULT_CURRENCY_CODE = 'USD' + +/** A bare numeric literal in exponent form, e.g. `1e+21` or `-1.5e-3`. */ +const EXPONENT_LITERAL = /^[+-]?\d+(?:\.\d+)?[eE][+-]?\d+$/ + +/** + * Invisible bidi control marks. `Intl` wraps RTL-locale output in them, so a + * pasted `‏1,234.56 ‏₪` carries characters that are not part of the amount. + */ +const BIDI_MARKS = /[\u200e\u200f\u061c\u202a-\u202e\u2066-\u2069]/g + +/** Minus-sign characters `Intl` emits in place of the ASCII hyphen. */ +const UNICODE_MINUS = /[\u2212\u2012\u2013\uFE63\uFF0D]/g + +/** A letter directly adjacent to a digit: an identifier, not an amount. */ +const LETTER_TOUCHING_DIGIT = /\p{L}\d|\d\p{L}/u + +/** + * A leading currency marker: up to three letters and/or a symbol, optionally + * behind a sign. The sign is captured so `-$12.50` keeps it. + */ +const CURRENCY_MARKER_PREFIX = + /^[\s\u00a0\u202f]*([+-]?)[\s\u00a0\u202f]*(?:\p{Sc}\p{L}{0,3}|\p{L}{1,3}\p{Sc}?)[\s\u00a0\u202f]*/u + +/** The same, trailing. */ +const CURRENCY_MARKER_SUFFIX = + /[\s\u00a0\u202f]*(?:\p{Sc}\p{L}{0,3}|\p{L}{1,3}\p{Sc}?)\.?[\s\u00a0\u202f]*$/u + +/** Only digits and separators, with at least one digit. */ +const AMOUNT_SHAPE = /^[+-]?[\d.,]*\d[\d.,]*$/ + +/** + * Whether `text` is validly grouped by `separator`: a first group of 1-3 digits + * and every later group exactly 3. Rejects `0.1.2` and `1,000,00`, which the + * plain "strip the separator" reading would silently turn into 12 and 100000. + */ +function hasValidGrouping(text: string, separator: string): boolean { + const groups = text.split(separator) + if (groups.length === 1) return true + if (!/^\d{1,3}$/.test(groups[0])) return false + const rest = groups.slice(1) + // Western: every later group is exactly three. + if (rest.every((group) => /^\d{3}$/.test(group))) return true + // Indian: the final group is three and the ones before it are two — + // `12,34,567`. Still rejects `1,000,00`, whose final group is two. + return ( + rest.length > 1 && + /^\d{3}$/.test(rest[rest.length - 1]) && + rest.slice(0, -1).every((group) => /^\d{2}$/.test(group)) + ) +} + +/** A decimal/grouping separator followed by whitespace — a list, not an amount. */ +const SEPARATOR_THEN_SPACE = /[.,]\s/ + +/** An `e` with a digit on both sides — an exponent marker, however spaced. */ +const INTERIOR_EXPONENT = /\d\s*[eE][+-]?\s*\d/ + +/** ISO 4217 alphabetic code: exactly three letters. */ +const CURRENCY_CODE_PATTERN = /^[A-Za-z]{3}$/ + +/** + * Codes offered first in the picker. The rest of ICU's set is still selectable + * (and any valid code is accepted over the API) — these are just the ones worth + * reaching without typing. + */ +const PINNED_CURRENCY_CODES = [ + 'USD', + 'EUR', + 'GBP', + 'JPY', + 'CAD', + 'AUD', + 'CHF', + 'CNY', + 'INR', + 'BRL', +] as const + +/** + * Every ISO 4217 code the runtime knows, or `null` when the runtime predates + * `Intl.supportedValuesOf` — in which case validation falls back to the shape + * check alone rather than rejecting codes it cannot enumerate. + */ +const supportedCurrencyCodes: ReadonlySet | null = (() => { + const supportedValuesOf = ( + Intl as typeof Intl & { + supportedValuesOf?: (key: string) => string[] + } + ).supportedValuesOf + if (typeof supportedValuesOf !== 'function') return null + try { + return new Set(supportedValuesOf('currency')) + } catch { + return null + } +})() + +/** Whether `code` is a well-formed ISO 4217 code this runtime can format. */ +export function isSupportedCurrencyCode(code: string): boolean { + if (!CURRENCY_CODE_PATTERN.test(code)) return false + const upper = code.toUpperCase() + return supportedCurrencyCodes === null || supportedCurrencyCodes.has(upper) +} + +/** A column's effective currency code, upper-cased, defaulting to {@link DEFAULT_CURRENCY_CODE}. */ +export function resolveCurrencyCode(currencyCode: string | undefined): string { + return currencyCode ? currencyCode.toUpperCase() : DEFAULT_CURRENCY_CODE +} + +export interface CurrencyOption { + code: string + /** Localized currency name, e.g. `US Dollar`. Falls back to the code. */ + name: string +} + +/** + * Codes for the column-config picker: the pinned set first, then every other + * code the runtime supports, alphabetically. + * + * Built on first call, not at module load: constructing `Intl.DisplayNames` and + * naming ~160 currencies costs several milliseconds of ICU work, and the only + * caller is the column-config sidebar — every table API route imports this + * module and would otherwise pay for a list it never reads. + */ +let currencyOptions: readonly CurrencyOption[] | null = null + +function currencyDisplayNames(): Intl.DisplayNames | null { + try { + return new Intl.DisplayNames(['en'], { type: 'currency' }) + } catch { + return null + } +} + +export function getCurrencyOptions(): readonly CurrencyOption[] { + if (currencyOptions) return currencyOptions + const pinned = new Set(PINNED_CURRENCY_CODES) + const rest = supportedCurrencyCodes + ? [...supportedCurrencyCodes].filter((code) => !pinned.has(code)).sort() + : [] + const displayNames = currencyDisplayNames() + currencyOptions = [...PINNED_CURRENCY_CODES, ...rest].map((code) => ({ + code, + name: displayNames?.of(code) ?? code, + })) + return currencyOptions +} + +/** + * Parses a user-entered or imported amount into a number, tolerating the shapes + * a currency value arrives in: symbols and ISO codes (`$1,234.56`, `1 234,56 €`, + * `USD 12`), grouping separators (including the non-breaking spaces several + * locales use), and accounting negatives (`(1,234.56)` → `-1234.56`). + * + * Separator disambiguation, when only commas are present: a single comma + * followed by exactly three digits is grouping (`1,500` → `1500`); anything + * else is a decimal comma (`1,50` → `1.5`). `1,500` meaning one-and-a-half is + * therefore read as fifteen hundred — a known ambiguity that resolves in favor + * of the far more common reading. + * + * Reads ASCII digits only. Locales that format with their own numeral systems + * (Arabic-Indic `١٢٣`, for instance) are rejected rather than misread — + * supporting them is a wider decision than this type, since it would also + * touch `number`, display, and sorting. + * + * Returns `null` when no amount can be read, so callers can distinguish + * "unparseable" from a legitimate `0`. + */ +export function parseCurrencyInput(raw: unknown): number | null { + if (typeof raw === 'number') return Number.isFinite(raw) ? raw : null + if (typeof raw !== 'string') return null + + // `Intl` emits U+2212 MINUS SIGN (and locale-specific dashes) rather than the + // ASCII hyphen for negatives in several locales, so a pasted `−12,50 kr` + // would otherwise fail to read as negative. + const trimmed = raw.trim().replace(BIDI_MARKS, '').replace(UNICODE_MINUS, '-') + if (trimmed === '') return null + + const parenthesized = /^\((.*)\)$/.exec(trimmed) + const body = parenthesized ? parenthesized[1] : trimmed + + // No real amount puts whitespace after a separator, but a delimited LIST + // does — and a multi-select column flattens to exactly that when it converts. + // Without this, `12, 34` reads as 12.34 and `100, 200` as 100200, so a + // multi-select column of numeric option names would convert to nonsense. + if (SEPARATOR_THEN_SPACE.test(body)) return null + + // Exponent form is taken at face value. `String()` emits it for any magnitude + // past 1e21, so a stored amount round-trips through the editor as `1e+21` — + // and stripping the `e` as decoration would read that back as 121, silently + // losing 19 orders of magnitude. + const exponentCandidate = body.replace(/[^\d.,\-+eE]/g, '') + if (EXPONENT_LITERAL.test(exponentCandidate)) { + const parsed = Number(exponentCandidate) + if (Number.isFinite(parsed)) return parenthesized ? -Math.abs(parsed) : parsed + } + // An `e` sitting between digits is an exponent marker, not decoration. If the + // string is not a clean literal we cannot read it unambiguously, and dropping + // the `e` would join the digit groups and change the magnitude (`1e5 EUR` + // would become 15) — so refuse instead of guessing. The digit on BOTH sides + // is what distinguishes this from the `E` inside an ISO code like `12 EUR`. + if (INTERIOR_EXPONENT.test(body)) return null + + // A letter touching a digit means this is an identifier, not an amount — + // `SKU400`, `ABC1234`. Currency markers are always separated from the number + // by a space or a symbol, so this distinguishes them without a symbol list. + if (LETTER_TOUCHING_DIGIT.test(body)) return null + + // Strip the currency marker: up to three letters (an ISO code, `kr`, `zł`) + // optionally joined to a symbol (`R$`, `CHF`), at either end. Bounded at + // three so prose does not qualify — `Revenue 5` keeps its letters and is + // rejected below. + const cleaned = body + .replace(CURRENCY_MARKER_PREFIX, '$1') + .replace(CURRENCY_MARKER_SUFFIX, '') + .replace(/[\s\u00a0\u202f\u2019']/gu, '') + // What remains must be ONLY digits and separators. Anything else — a + // US-format date, leftover prose — is not an amount. + if (!AMOUNT_SHAPE.test(cleaned)) return null + + const signed = /^[+-]/.test(cleaned) + const digitsAndSeps = signed ? cleaned.slice(1) : cleaned + const negative = parenthesized !== null || (signed && cleaned.startsWith('-')) + + const lastComma = digitsAndSeps.lastIndexOf(',') + const lastDot = digitsAndSeps.lastIndexOf('.') + let normalized: string + if (lastComma !== -1 && lastDot !== -1) { + // Both present: whichever comes last is the decimal separator. + const decimalSeparator = lastComma > lastDot ? ',' : '.' + const groupSeparator = decimalSeparator === ',' ? '.' : ',' + const [integerPart, ...decimalParts] = digitsAndSeps.split(decimalSeparator) + if (decimalParts.length > 1 || !hasValidGrouping(integerPart, groupSeparator)) return null + normalized = `${integerPart.split(groupSeparator).join('')}.${decimalParts[0]}` + } else if (lastComma !== -1) { + // A single comma followed by exactly three digits is grouping (`1,500`); + // anything else is a decimal comma (`1,50`). + const grouping = digitsAndSeps.indexOf(',') !== lastComma || /,\d{3}$/.test(digitsAndSeps) + if (grouping) { + if (!hasValidGrouping(digitsAndSeps, ',')) return null + normalized = digitsAndSeps.split(',').join('') + } else { + normalized = digitsAndSeps.replace(',', '.') + } + } else if (lastDot !== -1 && digitsAndSeps.indexOf('.') !== lastDot) { + // More than one dot can only be grouping: `1.234.567`. + if (!hasValidGrouping(digitsAndSeps, '.')) return null + normalized = digitsAndSeps.split('.').join('') + } else { + normalized = digitsAndSeps + } + + const parsed = Number(normalized) + if (!Number.isFinite(parsed)) return null + return negative ? -parsed : parsed +} + +/** + * Formatters are cached by locale + code: a grid paints thousands of currency + * cells per scroll, and constructing an `Intl.NumberFormat` per cell is orders + * of magnitude more expensive than the format call itself. + */ +const formatterCache = new Map() + +function currencyFormatter( + currencyCode: string, + locale: string | undefined +): Intl.NumberFormat | null { + const key = `${locale ?? ''}:${currencyCode}` + const cached = formatterCache.get(key) + if (cached !== undefined) return cached + let formatter: Intl.NumberFormat | null + try { + formatter = new Intl.NumberFormat(locale, { style: 'currency', currency: currencyCode }) + } catch { + formatter = null + } + formatterCache.set(key, formatter) + return formatter +} + +/** + * Formats a stored cell for display — symbol placement and fraction digits come + * from the currency itself, so `JPY` renders `¥1,235` while `USD` renders + * `$1,234.56`. Values that carry no readable amount (a string left behind by a + * `string` → `currency` conversion, say) render verbatim rather than blanking, + * and an unformattable code degrades to `CODE amount`. + * + * `locale` is left to the caller's runtime by default, which means the viewer's + * own grouping/decimal conventions in the browser. + */ +export function formatCurrencyDisplay( + value: unknown, + currencyCode: string | undefined, + locale?: string +): string { + const amount = parseCurrencyInput(value) + if (amount === null) return typeof value === 'string' ? value : '' + const code = resolveCurrencyCode(currencyCode) + const formatter = currencyFormatter(code, locale) + return formatter ? formatter.format(amount) : `${code} ${amount}` +} + +/** + * Renders a stored cell for a text input: the bare amount, with no symbol or + * grouping, so editing round-trips through {@link parseCurrencyInput} exactly. + */ +export function formatCurrencyForInput(value: unknown): string { + if (value === null || value === undefined) return '' + const amount = parseCurrencyInput(value) + if (amount !== null) return String(amount) + return typeof value === 'string' ? value : '' +} diff --git a/apps/sim/lib/table/export-format.ts b/apps/sim/lib/table/export-format.ts index c24c5863932..60f3d79eae4 100644 --- a/apps/sim/lib/table/export-format.ts +++ b/apps/sim/lib/table/export-format.ts @@ -4,6 +4,7 @@ * byte-identical files. */ +import { columnTypeOf } from '@/lib/table/column-types' import { selectValueToNames } from '@/lib/table/select-values' import type { ColumnDefinition } from '@/lib/table/types' @@ -43,10 +44,10 @@ export function formatCsvValue(value: unknown): string { * (comma-joined for multi) so the file shows the enum label, not the id. */ export function formatCsvCell(column: ColumnDefinition, value: unknown): string { - if (column.type === 'select') { - const resolved = resolveSelectExportValue(column, value) - const text = Array.isArray(resolved) ? resolved.join(', ') : (resolved ?? '') - return neutralizeCsvFormula(text) + // Every other type writes its stored value verbatim so the file re-imports + // byte-identically. + if (columnTypeOf(column).storesOpaqueIds) { + return neutralizeCsvFormula(columnTypeOf(column).formatForDisplay(value, column)) } return formatCsvValue(value) } diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index 91baff2b94e..2943870e899 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -136,6 +136,15 @@ describe('import', () => { expect(coerceValue('not a number', 'number')).toBeNull() }) + it('coerces a formatted amount into a currency column', () => { + // Importing into an EXISTING currency column — inference never picks + // currency, so this is the only way the branch is reached. + expect(coerceValue('$1,234.56', 'currency')).toBe(1234.56) + expect(coerceValue('1.234,56', 'currency')).toBe(1234.56) + expect(coerceValue(12, 'currency')).toBe(12) + expect(coerceValue('ask sales', 'currency')).toBeNull() + }) + it('coerces booleans strictly', () => { expect(coerceValue('true', 'boolean')).toBe(true) expect(coerceValue('FALSE', 'boolean')).toBe(false) diff --git a/apps/sim/lib/table/import.ts b/apps/sim/lib/table/import.ts index 94b4dc2272d..e01147c36f9 100644 --- a/apps/sim/lib/table/import.ts +++ b/apps/sim/lib/table/import.ts @@ -13,6 +13,8 @@ import { type Options as CsvParseOptions, type Parser, parse as parseCsvStream } from 'csv-parse' import { getColumnId } from '@/lib/table/column-keys' +import type { ColumnType } from '@/lib/table/column-types' +import { parseCurrencyInput } from '@/lib/table/currency' import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates' import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types' @@ -193,8 +195,18 @@ export async function detectCsvDelimiter( return best?.delimiter ?? fallback } -/** Narrower type than `COLUMN_TYPES` used internally for coercion. */ -export type CsvColumnType = 'string' | 'number' | 'boolean' | 'date' | 'json' +/** + * Column types the CSV path coerces. Derived from the registry rather than + * restated, so a new type is covered automatically. + */ +export type CsvColumnType = ColumnType + +/** + * The subset {@link inferColumnType} can return. Every other type either needs + * configuration inference cannot supply (`select`'s options, `currency`'s + * code) or would swallow ordinary text (`json`). + */ +type InferredCsvColumnType = Extract /** Number of CSV rows sampled when inferring column types for a new table. */ export const CSV_SCHEMA_SAMPLE_SIZE = 100 @@ -275,9 +287,11 @@ export async function parseCsvBuffer( /** * Infers a column type from a sample of non-empty values. Order matters: we * prefer narrower types (number > boolean > ISO date) and fall back to string. - * JSON is never inferred automatically. + * JSON and currency are never inferred automatically — currency because the + * column would also have to guess an ISO code from a symbol, and guessing wrong + * mislabels every amount in the column. */ -export function inferColumnType(values: unknown[]): Exclude { +export function inferColumnType(values: unknown[]): InferredCsvColumnType { const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '') if (nonEmpty.length === 0) return 'string' @@ -360,6 +374,13 @@ export function inferSchemaFromCsv( * empty inputs or values that cannot be parsed (numbers/booleans). Dates fall * back to the original string when unparseable so that schema validation can * reject it with context rather than silently inserting `null`. + * + * Deliberately NOT routed through the column-type registry's `coerce`, despite + * covering the same types. The registry's contract is "coerced or rejected", + * which the write path turns into `null`; an import instead wants an + * unparseable date or JSON blob to survive as its raw string so the row-level + * validation error names the offending value. Unifying the two would silently + * swap a descriptive import error for a blanked cell. */ export function coerceValue( value: unknown, @@ -372,6 +393,10 @@ export function coerceValue( const n = Number(value) return Number.isNaN(n) ? null : n } + // Importing into an existing currency column: the file carries the + // formatted amount (`$1,234.56`) but the cell stores a bare number. + case 'currency': + return parseCurrencyInput(value) case 'boolean': { const s = String(value).toLowerCase() if (s === 'true') return true diff --git a/apps/sim/lib/table/index.ts b/apps/sim/lib/table/index.ts index fad60f65370..72585f3c07a 100644 --- a/apps/sim/lib/table/index.ts +++ b/apps/sim/lib/table/index.ts @@ -9,6 +9,7 @@ export * from '@/lib/table/billing' export * from '@/lib/table/column-keys' export * from '@/lib/table/columns/service' export * from '@/lib/table/constants' +export * from '@/lib/table/currency' export * from '@/lib/table/dates' export * from '@/lib/table/errors' export * from '@/lib/table/import' diff --git a/apps/sim/lib/table/llm/enrichment.ts b/apps/sim/lib/table/llm/enrichment.ts index 2d9cb7d5d67..3225fd8e5bc 100644 --- a/apps/sim/lib/table/llm/enrichment.ts +++ b/apps/sim/lib/table/llm/enrichment.ts @@ -5,6 +5,7 @@ * with table-specific information so LLMs can construct proper queries. */ +import { columnTypeById } from '@/lib/table/column-types' import type { TableSummary } from '@/lib/table/types' /** @@ -91,7 +92,7 @@ ${filterExample}${sortExample}` const exampleCols = table.columns.slice(0, 3) const dataExample = exampleCols.reduce( (obj, col) => { - obj[col.name] = col.type === 'number' ? 123 : col.type === 'boolean' ? true : 'example' + obj[col.name] = columnTypeById(col.type).sampleValue return obj }, {} as Record @@ -168,7 +169,7 @@ export function enrichTableToolParameters( const exampleCols = table.columns.slice(0, 2) const exampleData = exampleCols.reduce( (obj: Record, col: { name: string; type: string }) => { - obj[col.name] = col.type === 'number' ? 123 : col.type === 'boolean' ? true : 'value' + obj[col.name] = columnTypeById(col.type).sampleValue return obj }, {} as Record diff --git a/apps/sim/lib/table/rows/service.ts b/apps/sim/lib/table/rows/service.ts index dfcc3a4c282..2cf9e357b63 100644 --- a/apps/sim/lib/table/rows/service.ts +++ b/apps/sim/lib/table/rows/service.ts @@ -24,6 +24,7 @@ import { wouldExceedRowLimit, } from '@/lib/table/billing' import { getColumnId } from '@/lib/table/column-keys' +import { columnTypeOf } from '@/lib/table/column-types' import { getMaxPageBytes, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import { @@ -814,7 +815,7 @@ const FIND_MATCH_LIMIT = 1000 * are trusted schema data, escaped and embedded literally; the row alias is `o`. */ export function buildSelectFindNameExpr(columns: ColumnDefinition[]): string | null { - const selectColumns = columns.filter((c) => c.type === 'select') + const selectColumns = columns.filter((c) => columnTypeOf(c).storesOpaqueIds) if (selectColumns.length === 0) return null const esc = (s: string) => s.replace(/'/g, "''") const whens = selectColumns diff --git a/apps/sim/lib/table/select-options.ts b/apps/sim/lib/table/select-options.ts new file mode 100644 index 00000000000..aa62cfa0bfd --- /dev/null +++ b/apps/sim/lib/table/select-options.ts @@ -0,0 +1,87 @@ +/** + * Pure `select`-option helpers, with no database dependency. + * + * These live in their own leaf module rather than in `validation.ts` on + * purpose. `validation.ts` imports `@sim/db`, `drizzle-orm`, and `next/server`, + * so anything importing it — `select-values.ts`, and transitively + * `cell-format.ts` and `export-format.ts` — becomes server-only. That taint is + * the sole reason the tables grid used to hand-roll its own copy of the + * id-resolution logic client-side, and why a select cell's stored ids could + * drift from the names the client resolved them to. + * + * Keeping the option primitives here lets both sides share one implementation. + */ + +import type { ColumnDefinition, JsonValue, SelectOption } from '@/lib/table/types' + +/** Set of valid option ids for a `select`/`multiselect` column. */ +export function optionIds(column: ColumnDefinition): Set { + return new Set((column.options ?? []).map((o) => o.id)) +} + +/** + * Resolves a raw cell value to a declared option id, accepting either the + * stable id or (tolerant for tool/import writes) the option's display name. + * Returns null when no option matches. + * + * Id wins over name, and an exact name wins over a case-folded one, so an + * option whose *name* equals another option's *id* can never repoint a cell. + */ +export function resolveSelectOptionId(value: JsonValue, options: SelectOption[]): string | null { + // The block builder serializes without schema access, so an option NAME that + // looks numeric or boolean ("123", "true") arrives scalar-coerced. Stringify + // scalars so the name still resolves; arrays/objects stay unresolvable. + const text = + typeof value === 'string' + ? value + : typeof value === 'number' || typeof value === 'boolean' + ? String(value) + : null + if (text === null) return null + const byId = options.find((o) => o.id === text) + if (byId) return byId.id + const byName = + options.find((o) => o.name === text) ?? + options.find((o) => o.name.toLowerCase() === text.toLowerCase()) + return byName ? byName.id : null +} + +/** + * Splits a raw value into the parts a multi-select cell should resolve. A cell + * may arrive as an array (canonical) or as a single comma-delimited string — + * the shape a multi cell exports, copies, and converts to text as — so both the + * write-path coercion and the column-conversion compatibility check read it + * through here rather than each deciding for itself. Option names that + * themselves contain commas are an accepted ambiguity. + */ +export function splitMultiSelectInput(value: JsonValue): JsonValue[] { + if (Array.isArray(value)) return value + if (typeof value !== 'string') return [value] + return value + .split(',') + .map((part) => part.trim()) + .filter((part) => part !== '') +} + +/** + * Resolves a raw value to the canonical stored shape for a select column: an + * array of option ids when `multiple`, otherwise a single id (or `null`). + * + * This is the one place the multi/single split is decided. Unresolvable parts + * are dropped and duplicates collapse to their first occurrence. + */ +export function resolveSelectCellValue(value: JsonValue, column: ColumnDefinition): JsonValue { + const options = column.options ?? [] + if (column.multiple) { + const ids: string[] = [] + for (const entry of splitMultiSelectInput(value)) { + const id = resolveSelectOptionId(entry, options) + if (id !== null && !ids.includes(id)) ids.push(id) + } + return ids + } + // Tolerate an array left behind by a multiple→single toggle by resolving its + // first element, rather than dropping the cell wholesale. + const single = Array.isArray(value) ? value[0] : value + return single === undefined ? null : resolveSelectOptionId(single, options) +} diff --git a/apps/sim/lib/table/select-values.ts b/apps/sim/lib/table/select-values.ts index c120a989ed7..5c1c2ae0f60 100644 --- a/apps/sim/lib/table/select-values.ts +++ b/apps/sim/lib/table/select-values.ts @@ -10,6 +10,7 @@ */ import { buildIdByName, getColumnId, predicateNamesToIds } from '@/lib/table/column-keys' +import { resolveSelectOptionId } from '@/lib/table/select-options' import type { ColumnDefinition, ConditionOperators, @@ -21,7 +22,6 @@ import type { TablePredicate, TableSchema, } from '@/lib/table/types' -import { resolveSelectOptionId } from '@/lib/table/validation' /** * Resolves a `select` cell's stored option id(s) to their display name(s). A diff --git a/apps/sim/lib/table/sql.ts b/apps/sim/lib/table/sql.ts index 7ea5a6de3f0..97dff495b3b 100644 --- a/apps/sim/lib/table/sql.ts +++ b/apps/sim/lib/table/sql.ts @@ -9,6 +9,12 @@ import { isRecordLike } from '@sim/utils/object' import type { SQL } from 'drizzle-orm' import { sql } from 'drizzle-orm' import { getColumnId } from '@/lib/table/column-keys' +import { + columnTypeById, + filterOperatorsFor, + MULTI_SELECT_OPERATORS, + SINGLE_SELECT_OPERATORS, +} from '@/lib/table/column-types' import { NAME_PATTERN } from '@/lib/table/constants' import { TableQueryValidationError } from '@/lib/table/errors' import type { @@ -23,20 +29,16 @@ import type { TablePredicate, } from '@/lib/table/types' +/** + * Re-exported: the `$`-prefixed wire whitelists now live with the `select` type + * definition, but this module is where callers and tests already look for them. + */ +export { MULTI_SELECT_OPERATORS, SINGLE_SELECT_OPERATORS } + type ColumnType = ColumnDefinition['type'] type ColumnMap = ReadonlyMap /** - * Operators that make sense on a `select` column (whose values are opaque option - * ids), split by cardinality. A single-select cell holds one id, so it compares - * for equality; a multi-select cell holds an array of ids, so the question is - * membership — hence contains / does-not-contain. `$eq` against an array cell - * can never be true (`{"t":["a"]} @> {"t":"a"}` is false in Postgres), so - * allowing it would silently match nothing. - */ -export const SINGLE_SELECT_OPERATORS = new Set(['$eq', '$ne', '$in', '$nin', '$empty']) -export const MULTI_SELECT_OPERATORS = new Set(['$contains', '$ncontains', '$empty']) - /** * The same allowlists in the v2 bare-operator grammar, applied inside * `fieldPredicate` so both wire formats gate identically. Not derived from the @@ -71,14 +73,7 @@ const MULTI_SELECT_OPS = new Set([ * paths from drifting apart. */ function jsonbCastForType(type: ColumnType | undefined): 'numeric' | 'timestamptz' | null { - switch (type) { - case 'number': - return 'numeric' - case 'date': - return 'timestamptz' - default: - return null - } + return columnTypeById(type).jsonbCast } /** @@ -405,6 +400,9 @@ function buildFieldCondition( const columnType = column?.type const isSelect = columnType === 'select' const isMultiSelect = isSelect && column?.multiple === true + // Types whose stored value is opaque (a select's option ids) restrict which + // operators mean anything; `null` means the type accepts them all. + const allowedOperators = column ? filterOperatorsFor(column) : null const conditions: SQL[] = [] if (isRecordLike(condition)) { @@ -412,14 +410,10 @@ function buildFieldCondition( // Validate against the legacy `$`-whitelist, then normalize onto the shared // `FilterOp` so v1 and v2 emit byte-identical leaf SQL. validateOperator(op) - // Select values are opaque option ids — range/pattern operators are meaningless. - if (isSelect) { - const allowed = isMultiSelect ? MULTI_SELECT_OPERATORS : SINGLE_SELECT_OPERATORS - if (!allowed.has(op)) { - throw new TableQueryValidationError( - `Operator "${op}" is not supported on ${isMultiSelect ? 'multi-select' : 'select'} column "${field}". Allowed: ${Array.from(allowed).join(', ')}` - ) - } + if (allowedOperators && !allowedOperators.has(op)) { + throw new TableQueryValidationError( + `Operator "${op}" is not supported on ${isMultiSelect ? 'multi-select' : columnType} column "${field}". Allowed: ${Array.from(allowedOperators).join(', ')}` + ) } if (op === '$empty') { diff --git a/apps/sim/lib/table/types.ts b/apps/sim/lib/table/types.ts index 1c6b5e39913..40231c77900 100644 --- a/apps/sim/lib/table/types.ts +++ b/apps/sim/lib/table/types.ts @@ -61,6 +61,12 @@ export interface ColumnDefinition { options?: SelectOption[] /** When true, a `select` column accepts several options per cell (string[]). */ multiple?: boolean + /** + * ISO 4217 code for a `currency` column, e.g. `USD`. Display metadata only — + * cells store a plain number, so changing this reformats without touching a + * single row. Absent means {@link DEFAULT_CURRENCY_CODE}. + */ + currencyCode?: string } /** The column `type` discriminator, named so callers don't index into the interface. */ @@ -775,16 +781,27 @@ export interface RenameColumnData { export interface UpdateColumnTypeData { tableId: string columnName: string + /** + * A rename to apply in the SAME transaction as this write. Folding it in is + * what stops a combined request from committing one half and then failing. + */ + newName?: string newType: (typeof COLUMN_TYPES)[number] /** Options to set when changing to a `select` type. */ options?: SelectOption[] /** Whether the `select` column accepts multiple options per cell. */ multiple?: boolean + /** Currency to set when changing to the `currency` type. */ + currencyCode?: string /** - * The `required` value the same request is about to set, when it changes type - * and constraints together. Those are separate transactions, so the - * conversion has to validate against the constraint the column will END UP - * with — otherwise it commits and the constraint write then fails. + * The `unique` value the same request is about to set. Validated inside the + * retype against the post-conversion values, because the conversion is what + * can create the duplicates. + */ + unique?: boolean + /** + * The `required` value the same request is about to set. Applied by this + * write, in the same transaction as the change it accompanies. */ required?: boolean } @@ -792,21 +809,50 @@ export interface UpdateColumnTypeData { export interface UpdateColumnOptionsData { tableId: string columnName: string + /** + * A rename to apply in the SAME transaction as this write. Folding it in is + * what stops a combined request from committing one half and then failing. + */ + newName?: string + /** Constraints to apply in the SAME transaction as this write. */ + unique?: boolean options: SelectOption[] /** Toggle single/multi selection alongside the options update. */ multiple?: boolean /** - * The `required` value the same request is about to set. The constraint write - * is a separate transaction, so the options update has to validate against - * the constraint the column will END UP with — otherwise it clears cells and - * the constraint write then fails, leaving the removal committed. + * The `required` value the same request is about to set. Applied by this + * write, in the same transaction as the options change. + */ + required?: boolean +} + +/** + * Payload for `updateColumnCurrency`. Unlike an options update this rewrites no + * cells — a currency cell stores a plain number, and `currencyCode` only + * changes how it is rendered. + */ +export interface UpdateColumnCurrencyData { + tableId: string + columnName: string + /** + * A rename to apply in the SAME transaction as this write. Folding it in is + * what stops a combined request from committing one half and then failing. */ + newName?: string + /** Constraints to apply in the SAME transaction as this write. */ + unique?: boolean required?: boolean + currencyCode: string } export interface UpdateColumnConstraintsData { tableId: string columnName: string + /** + * A rename to apply in the SAME transaction as this write. Folding it in is + * what stops a combined request from committing one half and then failing. + */ + newName?: string required?: boolean unique?: boolean } diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index 14275ea74eb..270ee5e3ee4 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -7,28 +7,51 @@ import { userTableRows } from '@sim/db/schema' import { and, eq, or, type SQL, sql } from 'drizzle-orm' import { NextResponse } from 'next/server' import { getColumnId } from '@/lib/table/column-keys' +import type { CoerceResult, TypeSpecificColumnKey } from '@/lib/table/column-types' import { + COLUMN_TYPE_REGISTRY, COLUMN_TYPES, + columnTypeOf, + isColumnType, + TYPE_SPECIFIC_COLUMN_KEYS, + validateTypeMetadata, +} from '@/lib/table/column-types' +import { getMaxRowSizeBytes, - MAX_SELECT_OPTIONS, NAME_PATTERN, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME, } from '@/lib/table/constants' -import { normalizeDateCellValue } from '@/lib/table/dates' import { withSeqscanOff } from '@/lib/table/planner' +import { resolveSelectOptionId, splitMultiSelectInput } from '@/lib/table/select-options' import { fieldPredicate } from '@/lib/table/sql' import type { ColumnDefinition, JsonValue, RowData, - SelectOption, TableSchema, ValidationResult, } from '@/lib/table/types' export type { ColumnDefinition, TableSchema, ValidationResult } +/** + * Re-exported so existing importers keep working; the implementations moved to + * `select-options.ts` to break this module's drizzle dependency for clients. + */ +export { resolveSelectOptionId, splitMultiSelectInput } + +/** + * How each type-specific key is named when it appears on a type that doesn't + * own it. `Record` keeps this exhaustive — a new + * key cannot be added without giving it a message. + */ +const FOREIGN_METADATA_VERB: Record = { + options: 'define options', + multiple: 'be multiple', + currencyCode: 'define a currency', +} + type ValidationSuccess = { valid: true } type ValidationFailure = { valid: false; response: NextResponse } @@ -234,174 +257,21 @@ export function validateRowAgainstSchema(data: RowData, schema: TableSchema): Va if (value === null || value === undefined) continue - switch (column.type) { - case 'string': - if (typeof value !== 'string') { - errors.push(`${column.name} must be string, got ${typeof value}`) - } - break - case 'number': - if (typeof value !== 'number' || Number.isNaN(value)) { - errors.push(`${column.name} must be number`) - } - break - case 'boolean': - if (typeof value !== 'boolean') { - errors.push(`${column.name} must be boolean`) - } - break - case 'date': - if ( - !(value instanceof Date) && - (typeof value !== 'string' || Number.isNaN(Date.parse(value))) - ) { - errors.push(`${column.name} must be valid date`) - } - break - case 'json': - try { - JSON.stringify(value) - } catch { - errors.push(`${column.name} must be valid JSON`) - } - break - case 'select': { - const ids = optionIds(column) - if (column.multiple) { - if (!Array.isArray(value)) { - errors.push(`${column.name} must be a list of options`) - } else if (!value.every((v) => typeof v === 'string' && ids.has(v))) { - errors.push(`${column.name} must only contain defined options`) - } else if (column.required && value.length === 0) { - errors.push(`Missing required field: ${column.name}`) - } - } else if (typeof value !== 'string' || !ids.has(value)) { - errors.push(`${column.name} must be one of the defined options`) - } - break - } - } + const error = columnTypeOf(column).validateCell(value, column) + if (error !== null) errors.push(error) } return { valid: errors.length === 0, errors } } -/** Set of valid option ids for a `select`/`multiselect` column. */ -function optionIds(column: ColumnDefinition): Set { - return new Set((column.options ?? []).map((o) => o.id)) -} - -/** - * Resolves a raw cell value to a declared option id, accepting either the - * stable id or (tolerant for tool/import writes) the option's display name. - * Returns null when no option matches. Exported so the column-type-conversion - * path can gate a `select`/`multiselect` change on whether existing values - * actually fit the target option set. - */ -export function resolveSelectOptionId(value: JsonValue, options: SelectOption[]): string | null { - // The block builder serializes without schema access, so an option NAME that - // looks numeric or boolean ("123", "true") arrives scalar-coerced. Stringify - // scalars so the name still resolves; arrays/objects stay unresolvable. - const text = - typeof value === 'string' - ? value - : typeof value === 'number' || typeof value === 'boolean' - ? String(value) - : null - if (text === null) return null - const byId = options.find((o) => o.id === text) - if (byId) return byId.id - const byName = - options.find((o) => o.name === text) ?? - options.find((o) => o.name.toLowerCase() === text.toLowerCase()) - return byName ? byName.id : null -} - -/** - * Splits a raw value into the parts a multi-select cell should resolve. A cell - * may arrive as an array (canonical) or as a single comma-delimited string — - * the shape a multi cell exports, copies, and converts to text as — so both the - * write-path coercion and the column-conversion compatibility check read it - * through here rather than each deciding for itself. Option names that - * themselves contain commas are an accepted ambiguity. - */ -export function splitMultiSelectInput(value: JsonValue): JsonValue[] { - if (Array.isArray(value)) return value - if (typeof value !== 'string') return [value] - return value - .split(',') - .map((part) => part.trim()) - .filter((part) => part !== '') -} - /** * Attempts to coerce a non-null value to a column's declared type. Returns the * coerced value when the value already matches or can be converted without * ambiguity (e.g. the string `"1999"` to the number `1999`), and `ok: false` * when no safe conversion exists. */ -function coerceValueToColumnType( - value: JsonValue, - column: ColumnDefinition -): { ok: true; value: JsonValue } | { ok: false } { - switch (column.type) { - case 'string': - if (typeof value === 'string') return { ok: true, value } - if (typeof value === 'number' || typeof value === 'boolean') { - return { ok: true, value: String(value) } - } - return { ok: false } - case 'number': - if (typeof value === 'number') { - return Number.isFinite(value) ? { ok: true, value } : { ok: false } - } - if (typeof value === 'string' && value.trim() !== '') { - const parsed = Number(value) - return Number.isFinite(parsed) ? { ok: true, value: parsed } : { ok: false } - } - return { ok: false } - case 'boolean': - if (typeof value === 'boolean') return { ok: true, value } - if (typeof value === 'string') { - const normalized = value.trim().toLowerCase() - if (normalized === 'true') return { ok: true, value: true } - if (normalized === 'false') return { ok: true, value: false } - } - return { ok: false } - case 'date': { - if (typeof value === 'string') { - const normalized = normalizeDateCellValue(value) - return normalized === null ? { ok: false } : { ok: true, value: normalized } - } - // Date instances and epoch numbers may still be out of the representable - // range (>±8.64e15ms) — guard `toISOString()`, which throws RangeError on - // an Invalid Date, so an over-range value degrades to `{ ok: false }` - // rather than crashing the write. - const date = - value instanceof Date ? value : typeof value === 'number' ? new Date(value) : null - if (date && !Number.isNaN(date.getTime())) return { ok: true, value: date.toISOString() } - return { ok: false } - } - case 'select': { - const options = column.options ?? [] - if (column.multiple) { - const raw = splitMultiSelectInput(value) - const ids: string[] = [] - for (const entry of raw) { - const id = resolveSelectOptionId(entry, options) - if (id !== null && !ids.includes(id)) ids.push(id) - } - return { ok: true, value: ids } - } - // Single: tolerate an array (e.g. right after a multiple→single toggle) by - // resolving its first element so the value isn't dropped wholesale. - const single = Array.isArray(value) ? value[0] : value - const id = single === undefined ? null : resolveSelectOptionId(single, options) - return id !== null ? { ok: true, value: id } : { ok: false } - } - default: - return { ok: true, value } - } +function coerceValueToColumnType(value: JsonValue, column: ColumnDefinition): CoerceResult { + return columnTypeOf(column).coerce(value, column) } /** @@ -628,7 +498,7 @@ export async function checkBatchUniqueConstraintsDb( const value = rowData[key] if (value === null || value === undefined) continue - const normalizedValue = typeof value === 'string' ? value : JSON.stringify(value) + const normalizedValue = JSON.stringify(value) // Check for duplicate within batch const columnValueMap = batchValueMap.get(key)! @@ -664,10 +534,13 @@ export async function checkBatchUniqueConstraintsDb( const valueArray = Array.from(values) const valueConditions = valueArray.map((normalizedValue) => { - // Reconstruct the original typed value from its normalized key: string - // columns store the raw string; others store JSON.stringify(value). - const originalValue: JsonValue = - column.type === 'string' ? normalizedValue : JSON.parse(normalizedValue) + // Reconstruct the original typed value from its normalized key. Both + // directions go through JSON unconditionally: keying the write on the + // value's RUNTIME type while keying the read on the column's DECLARED + // type made them disagree for any non-`string` type that stores a + // string — a unique `date` column normalized to a bare `2024-01-01` + // and then threw `SyntaxError` trying to parse it back. + const originalValue: JsonValue = JSON.parse(normalizedValue) // Same case-sensitive containment leaf as every other matcher. const clause = fieldPredicate( USER_TABLE_ROWS_SQL_NAME, @@ -759,67 +632,34 @@ export function validateColumnDefinition(column: ColumnDefinition): ValidationRe ) } - if (!COLUMN_TYPES.includes(column.type)) { + if (!isColumnType(column.type)) { errors.push( `Column "${column.name}" has invalid type "${column.type}". Valid types: ${COLUMN_TYPES.join(', ')}` ) + // Every check below reads the type's own rules; without a known type there + // are none to apply. + return { valid: false, errors } } - if (column.type === 'select') { - errors.push(...validateSelectOptions(column)) - // Uniqueness on a select compares the stored option id, so it caps each - // option at one row for the whole table — and the UI hides the toggle, so a - // constraint set through the API or an agent could never be cleared. - if (column.unique) { - errors.push(`Column "${column.name}" of type "select" cannot be unique`) - } - } else { - if (column.options !== undefined) { - errors.push(`Column "${column.name}" cannot define options for type "${column.type}"`) - } - // A stored `multiple` on a non-select column is inert until the column is - // converted, at which point `updateColumnType` inherits it — silently - // turning an intended single-select into a multiselect and rewriting every - // cell as an array. - if (column.multiple) { - errors.push(`Column "${column.name}" cannot be multiple for type "${column.type}"`) - } - } + const definition = COLUMN_TYPE_REGISTRY[column.type] + errors.push(...validateTypeMetadata(column)) - return { valid: errors.length === 0, errors } -} + // Uniqueness compares the stored value, which is meaningless for a type whose + // storage is an opaque id — it would cap each option at one row for the whole + // table, and the UI hides the toggle so it could never be cleared again. + if (column.unique && !definition.supportsUnique) { + errors.push(`Column "${column.name}" of type "${column.type}" cannot be unique`) + } -/** Validates the option set declared on a `select` column. */ -function validateSelectOptions(column: ColumnDefinition): string[] { - const errors: string[] = [] - const options = column.options - if (!Array.isArray(options) || options.length === 0) { - errors.push(`Column "${column.name}" of type "${column.type}" must define at least one option`) - return errors - } - if (options.length > MAX_SELECT_OPTIONS) { - errors.push(`Column "${column.name}" cannot have more than ${MAX_SELECT_OPTIONS} options`) - } - const ids = new Set() - const names = new Set() - for (const opt of options) { - if (!opt.id || typeof opt.id !== 'string') { - errors.push(`Column "${column.name}" has an option missing an id`) - } else if (ids.has(opt.id)) { - errors.push(`Column "${column.name}" has duplicate option id "${opt.id}"`) - } else { - ids.add(opt.id) - } - if (!opt.name || typeof opt.name !== 'string') { - errors.push(`Column "${column.name}" has an option missing a name`) - } else { - const key = opt.name.toLowerCase() - if (names.has(key)) { - errors.push(`Column "${column.name}" has duplicate option name "${opt.name}"`) - } else { - names.add(key) - } - } + // Type-specific metadata stored on the wrong type is inert until a later + // conversion inherits it — silently overriding what that request asked for. + const owned = new Set(definition.ownedMetadata) + for (const key of TYPE_SPECIFIC_COLUMN_KEYS) { + if (column[key] === undefined || owned.has(key)) continue + errors.push( + `Column "${column.name}" cannot ${FOREIGN_METADATA_VERB[key]} for type "${column.type}"` + ) } - return errors + + return { valid: errors.length === 0, errors } } diff --git a/apps/sim/stores/table/types.ts b/apps/sim/stores/table/types.ts index 030377b775e..1da15ace218 100644 --- a/apps/sim/stores/table/types.ts +++ b/apps/sim/stores/table/types.ts @@ -60,6 +60,9 @@ export type TableUndoAction = // holds option ids, would have nothing to attach to. columnOptions?: ColumnDefinition['options'] columnMultiple?: boolean + // Likewise for a `currency` column: without its code the restore would + // silently re-denominate every cell to the default currency. + columnCurrencyCode?: string cellData: Array<{ rowId: string; value: unknown }> previousOrder: string[] | null previousWidth: number | null diff --git a/packages/emcn/src/icons/index.ts b/packages/emcn/src/icons/index.ts index 5bbd98df1b5..1f871b6f680 100644 --- a/packages/emcn/src/icons/index.ts +++ b/packages/emcn/src/icons/index.ts @@ -103,6 +103,7 @@ export { TrashOutline } from './trash-outline' export { Trash2 } from './trash2' export { TriangleAlert } from './triangle-alert' export { TypeBoolean } from './type-boolean' +export { TypeCurrency } from './type-currency' export { TypeJson } from './type-json' export { TypeNumber } from './type-number' export { TypeText } from './type-text' diff --git a/packages/emcn/src/icons/type-currency.tsx b/packages/emcn/src/icons/type-currency.tsx new file mode 100644 index 00000000000..ca0bbd0d761 --- /dev/null +++ b/packages/emcn/src/icons/type-currency.tsx @@ -0,0 +1,26 @@ +import type { SVGProps } from 'react' + +/** + * Type currency icon component - dollar sign for currency columns + * @param props - SVG properties including className, fill, etc. + */ +export function TypeCurrency(props: SVGProps) { + return ( + + ) +} From 64bcfead343e42a65249c5986b4839070ae205e7 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 30 Jul 2026 17:55:16 -0700 Subject: [PATCH 11/21] fix(desktop): clear the traffic lights on every full-viewport surface, and enumerate them in CI (#6109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(desktop): clear the traffic lights on every auth-shell surface Only /login reserved the macOS traffic-light lane, so signup drew its logo underneath the lights — and so did reset-password, sso, verify, the CLI auth handoff, and the invite pages. The pre-paint script marks the lane on every desktop route, so any surface that did not reserve it overlapped. Ownership moves to `AuthShell`, which is the single source of truth for the frame all of these wear, and it now reserves unconditionally. Per-route gating was the wrong shape rather than merely incomplete: `/invite/[id]` is a dynamic segment, so no route list could have covered it. `supportsDesktopTitleBar` therefore drops its pathname argument — the caller mounting the controller is the signal, and only `AuthShell` mounts it. Workspace routes never render it and keep their existing `WorkspaceChrome`-owned listener, so the two never contend for the attribute. Off the desktop shell `--desktop-title-bar-height` is `0px`, so the reservation and the drag strip collapse to nothing and `.desktop-title-bar-page` is exactly the `min-h-screen` these surfaces had before — web is unchanged. With the prop gone the client auth layout was a bare passthrough, so the route layout renders the shell directly and the passthrough is deleted. Measured in the Electron renderer over CDP across /signup, /login, /reset-password and /cli/auth: lane 40px, logo top 56px, zero overflow on each. /invite/[id] redirects to login when signed out and was not measured directly. The surface audit gains a `stripComments` helper that every negative assertion runs through. These files document the shapes they avoid, so a bare `not.toContain` was matching the prose explaining the fix and failing on correct code. * fix(desktop): cover the remaining traffic-light overlaps, and enumerate them in CI Fixing signup by hand would have been the fourth time this bug was found by a person hitting it. The audit now enumerates instead of listing what to inspect: it walks every `.tsx` outside workspace chrome, flags each full-viewport root, and fails unless that root either composes `.desktop-title-bar-page` or appears in an allowlist with a written reason. A brand-new page that fills the viewport fails on arrival — verified by adding one, and by reverting each fix below. Running it found three more surfaces already overlapping: - `/oauth-error` is Better Auth's `onAPIError.errorURL`, which is precisely where desktop OAuth failures land, so the one page a user sees when sign-in breaks drew its content under the lights. - `/f/[token]` public file view, same shell family, same origin, reachable in the window. - The signup and reset-password Suspense fallbacks are viewport-tall *inside* the lane-reserving shell, so the page overflowed by the lane while the split chunk loaded. A placeholder needs no viewport height. Four surfaces are allowlisted with reasons: the two landing shells (the desktop shell boots to /login or a workspace and has no path to marketing routes), the dev-only playground, and the embedded resume interface. Measured over CDP: /oauth-error reserves 40px with zero overflow. * fix(desktop): cover the shells behind the allowlist, and make the guard fail Greptile was right on both counts, and the first one is worse than reported. `LogoShell` was allowlisted as "marketing chrome, not reachable in the desktop shell". That claim was simply false: it is the frame for `not-found`, the interfaces shell, the desktop handoff shell, and the public-file access gates — so the password, email, and SSO gates for `/f/[token]` all still drew under the traffic lights. The allowlist existed to make risk visible and instead hid four surfaces behind one unverified sentence. It now carries two entries, both checked: the landing shell (every consumer lives under `app/(landing)/`) and the playground (calls `notFound()` unless `NEXT_PUBLIC_ENABLE_PLAYGROUND` is set). The lane's two halves also travelled separately, so `/oauth-error` and the public-file view reserved the space without the drag strip — clearing the lights but leaving the window with no title bar on those pages. `DesktopTitleBarLane` now ships both together and the audit enforces the pairing. Both new checks were unfailable when first written, and mutation testing is the only reason that surfaced: - the pairing check matched `DesktopTitleBarLane` anywhere in the file, so the import line satisfied it after the JSX was deleted; - the coverage check matched `LogoShell` anywhere, so a shell's own definition file self-certified as covered. Both now match JSX usage (`/`, which already owns the workspace lane and its drag region, so padding that root would double it. Their content was centred, so the lights were never covering text — the real gap was that none of them rendered a drag strip, leaving the window immovable on those screens. The guard's granularity is per file, not per JSX root: `workspace/page.tsx` holds two full-viewport roots and still passes if only one reserves the lane. Verified by mutation and documented rather than papered over — catching it needs an AST pass, and the check's job is to stop a whole surface being forgotten, which is how every instance of this bug has actually shipped. * fix(desktop): teach the audit about nesting, and stop the resume skeleton double-reserving Four findings, all correct, and the first is a bug this PR introduced. The resume loading skeleton reserved the lane while already rendering inside `(interfaces)/layout.tsx` -> `InterfacesShell` -> `LogoShell`, which reserves it too. Two lots of padding, two drag strips, two controllers. It came from adding the lane there before `LogoShell` became lane-aware and never reconciling the two. The skeleton now reserves nothing and is no longer viewport-tall either — nesting a viewport-tall root inside a viewport-tall shell overflowed even before this PR. The public-file header pinned `sticky top-0`, which parks it inside the reserved lane and under the lights. It now sticks below the lane, inert on web where the variable is `0px`. Both audit gaps were real: - The check was file-local, so it could not see the doubling above. It now resolves ancestor layouts: a root counts as covered when it reserves OR sits inside a layout that does, and reserving on both levels is its own failure. That also stops the check demanding a second reservation from chat and the workspace overlays, which correctly inherit theirs. - Detection only matched `min-h-screen`/`h-screen`, so `fixed inset-0` roots never entered it. Now included. With nesting understood, that addition resolved to a single genuinely uncovered file rather than the ten it flagged beforehand. Two allowlist entries added, both reasoned rather than assumed: the landing prefix (dozens of files, one justification), and the desktop update gate — it centres its content, and under `hiddenInset` macOS draws the lights above the web contents, so web UI cannot cover them. This bug class is app chrome sitting under the lights, never the reverse. Verified by mutation: reintroducing the double reservation fails the new check. * test(desktop): do not credit inherited coverage across a layout's early return Ancestor resolution is static, so it credits any file under a layout that mentions a lane-aware shell. That is wrong when the layout returns the surface *instead of* its chrome: `workspace/[workspaceId]/layout.tsx` returns `` at the top and only reaches `` far below, so at runtime the denied page has no chrome at all. The page does reserve the lane today, but a regression would have read as inherited and passed. `SessionExpired` is deliberately not listed: it renders as a sibling within the chrome tree, so its inherited coverage is real. The distinction is which side of the early return the surface sits on, not which directory it lives in. Verified by reverting the access-denied page exactly as described — it now fails. * test(desktop): count the lane class itself as a viewport claim Cursor caught the audit failing to watch exactly the files this PR converted. Detection keyed on `min-h-screen`/`h-screen`/`fixed inset-0`, but converting a surface to `.desktop-title-bar-page` removes those tokens — the class supplies `min-height: 100vh` itself. So `/oauth-error`, the public-file view and `AuthShell` dropped out of the check entirely, and a nested class-only reservation could ship green. That also means the doubled-reservation check had never actually fired. The mutation I used to "verify" it removed the lane component as well, so the pairing check caught it and the doubled check was never exercised. It now fires on Cursor's exact scenario: a nested class-only reservation, correctly paired, inside a lane-aware shell. Pulling those files back in exposed a second-order bug: a shell's own definition file sits under the layout that renders it, so ancestor resolution called `AuthShell` nested inside itself. Shell definitions are excluded from inheritance. One limit stays, documented rather than papered over: a root is in scope because of how it claims the viewport, so deleting the reservation outright drops the file from the check. That regression is loud, not silent — the surface stops being full height. Closing it properly means treating every route entry point as a window root, which pulls in seven account/organization/selfhost pages needing individual assessment. Worth doing separately; allowlisting them on assumptions is the mistake that produced the `LogoShell` hole. * fix(desktop): a fixed root escapes ancestor padding — chat drew under the lights Cursor's sharpest catch, and the audit was actively hiding the bug rather than missing it. `position: fixed` resolves against the viewport, not the parent, so a lane-aware shell's `padding-top` never moves it. The chat surfaces sit inside `LogoShell` and still painted at viewport top, under the traffic lights, while the check reported them covered — and adding the correct reservation would then have tripped the nested-reservation check, so the audit pushed toward the wrong answer. Roots matching `fixed inset-0` no longer inherit coverage and are exempt from the doubled check. That reclassified seven surfaces, each decided on evidence: - chat, its loading boundary, the loading state and the voice interface are full-window roots at `z-[100]` with their own top chrome — all now reserve; - the file viewer wraps a full-bleed `