feat: add options for custom bigint and timestamp conversions in fetchRow and fetchAll

Author: bitofsky

README.md

@@ -209,6 +209,8 @@ function fetchRow(
 ```
 - Streams each row to `options.onEachRow`.
 - Use `format: 'JSON_OBJECT'` to map rows into schema-based objects.
+- Use `encodeBigInt` to customize BIGINT/LONG conversions when using `JSON_OBJECT`.
+- Use `encodeTimestamp` to customize TIMESTAMP* conversions when using `JSON_OBJECT`.
 - Supports `INLINE` results or `JSON_ARRAY` formatted `EXTERNAL_LINKS` only.
 - If only a subset of external links is returned, missing chunk metadata is fetched by index.
 
@@ -274,15 +276,19 @@ type ExecuteStatementOptions = {
 
 type FetchRowsOptions = {
   signal?: AbortSignal
-  onEachRow?: (row: RowArray | RowObject) => void
   format?: 'JSON_ARRAY' | 'JSON_OBJECT'
   logger?: Logger
+  onEachRow?: (row: RowArray | RowObject) => void
+  encodeBigInt?: (value: bigint) => unknown
+  encodeTimestamp?: (value: string) => unknown
 }
 
 type FetchAllOptions = {
   signal?: AbortSignal
   format?: 'JSON_ARRAY' | 'JSON_OBJECT'
   logger?: Logger
+  encodeBigInt?: (value: bigint) => unknown
+  encodeTimestamp?: (value: string) => unknown
 }
 
 type FetchStreamOptions = {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bitofsky/databricks-sql",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "Databricks SQL client for Node.js - Direct REST API without SDK",
   "main": "dist/index.cjs",
   "module": "dist/index.js",

src/api/fetchAll.ts

@@ -41,6 +41,12 @@ export async function fetchAll(
   if (options.logger)
     fetchOptions.logger = options.logger
 
+  if (options.encodeBigInt)
+    fetchOptions.encodeBigInt = options.encodeBigInt
+
+  if (options.encodeTimestamp)
+    fetchOptions.encodeTimestamp = options.encodeTimestamp
+
   await fetchRow(statementResult, auth, fetchOptions)
   logger?.info?.(`fetchAll fetched ${rows.length} rows for statement ${statementId}.`, {
     ...logContext,

src/api/fetchRow.ts

@@ -30,7 +30,10 @@ export async function fetchRow(
   const statementId = statementResult.statement_id
   const logContext = { statementId, manifest, requestedFormat: format }
   // Map JSON_ARRAY rows to JSON_OBJECT when requested.
-  const mapRow = createRowMapper(manifest, format)
+  const mapRow = createRowMapper(manifest, format, {
+    ...options.encodeBigInt ? { encodeBigInt: options.encodeBigInt } : {},
+    ...options.encodeTimestamp ? { encodeTimestamp: options.encodeTimestamp } : {},
+  })
 
   logger?.info?.(`fetchRow fetching rows for statement ${statementId}.`, {
     ...logContext,

src/createRowMapper.ts

@@ -4,6 +4,7 @@ import type {
   FetchRowsOptions,
   RowArray,
   RowObject,
+  RowMapperOptions,
   StatementManifest,
 } from './types.js'
 
@@ -30,12 +31,10 @@ const INTEGER_TYPES = new Set(['TINYINT', 'SMALLINT', 'INT'])
 const BIGINT_TYPES = new Set(['BIGINT', 'LONG'])
 const FLOAT_TYPES = new Set(['FLOAT', 'DOUBLE'])
 const BOOLEAN_TYPES = new Set(['BOOLEAN'])
+const TIMESTAMP_TYPES = new Set(['TIMESTAMP', 'TIMESTAMP_NTZ', 'TIMESTAMP_LTZ'])
 const STRING_TYPES = new Set([
   'STRING',
   'DATE',
-  'TIMESTAMP',
-  'TIMESTAMP_NTZ',
-  'TIMESTAMP_LTZ',
   'TIME',
 ])
 
@@ -46,15 +45,16 @@ const STRING_TYPES = new Set([
  */
 export function createRowMapper(
   manifest: StatementManifest,
-  format: FetchRowsOptions['format']
+  format: FetchRowsOptions['format'],
+  options: RowMapperOptions = {}
 ): RowMapper {
   if (format !== 'JSON_OBJECT')
     return (row) => row
 
   // Precompute per-column converters for fast row mapping.
   const columnConverters = manifest.schema.columns.map((column: ColumnInfo) => ({
     name: column.name,
-    convert: createColumnConverter(column),
+    convert: createColumnConverter(column, options),
   }))
 
   return (row) => {
@@ -72,9 +72,12 @@ export function createRowMapper(
   }
 }
 
-function createColumnConverter(column: ColumnInfo): (value: unknown) => unknown {
+function createColumnConverter(
+  column: ColumnInfo,
+  options: RowMapperOptions
+): (value: unknown) => unknown {
   const descriptor = parseColumnType(column)
-  return (value) => convertValue(descriptor, value)
+  return (value) => convertValue(descriptor, value, options)
 }
 
 function parseColumnType(column: ColumnInfo): TypeDescriptor {
@@ -255,19 +258,23 @@ function stripNotNull(typeText: string): string {
   return trimmed
 }
 
-function convertValue(descriptor: TypeDescriptor, value: unknown): unknown {
+function convertValue(
+  descriptor: TypeDescriptor,
+  value: unknown,
+  options: RowMapperOptions
+): unknown {
   if (value === null || value === undefined)
     return value
 
   if (descriptor.typeName === 'STRUCT' && descriptor.fields)
     // STRUCT values are JSON strings in JSON_ARRAY format.
-    return convertStructValue(descriptor.fields, value)
+    return convertStructValue(descriptor.fields, value, options)
 
   if (descriptor.typeName === 'ARRAY' && descriptor.elementType)
-    return convertArrayValue(descriptor.elementType, value)
+    return convertArrayValue(descriptor.elementType, value, options)
 
   if (descriptor.typeName === 'MAP' && descriptor.keyType && descriptor.valueType)
-    return convertMapValue(descriptor.keyType, descriptor.valueType, value)
+    return convertMapValue(descriptor.keyType, descriptor.valueType, value, options)
 
   if (descriptor.typeName === 'DECIMAL')
     return convertNumber(value)
@@ -276,45 +283,57 @@ function convertValue(descriptor: TypeDescriptor, value: unknown): unknown {
     return convertNumber(value)
 
   if (BIGINT_TYPES.has(descriptor.typeName))
-    return convertInteger(value)
+    return convertInteger(value, options.encodeBigInt)
 
   if (FLOAT_TYPES.has(descriptor.typeName))
     return convertNumber(value)
 
   if (BOOLEAN_TYPES.has(descriptor.typeName))
     return convertBoolean(value)
 
+  if (TIMESTAMP_TYPES.has(descriptor.typeName))
+    return convertTimestamp(value, options.encodeTimestamp)
+
   if (STRING_TYPES.has(descriptor.typeName))
     return value
 
   return value
 }
 
-function convertStructValue(fields: StructField[], value: unknown): unknown {
+function convertStructValue(
+  fields: StructField[],
+  value: unknown,
+  options: RowMapperOptions
+): unknown {
   const raw = parseStructValue(value)
   if (!raw || typeof raw !== 'object' || Array.isArray(raw))
     return value
 
   // Apply nested field conversions based on the parsed STRUCT schema.
   const mapped: RowObject = {}
   for (const field of fields)
-    mapped[field.name] = convertValue(field.type, (raw as RowObject)[field.name])
+    mapped[field.name] = convertValue(field.type, (raw as RowObject)[field.name], options)
 
   return mapped
 }
 
-function convertArrayValue(elementType: TypeDescriptor, value: unknown): unknown {
+function convertArrayValue(
+  elementType: TypeDescriptor,
+  value: unknown,
+  options: RowMapperOptions
+): unknown {
   const raw = parseJsonValue(value)
   if (!Array.isArray(raw))
     return value
 
-  return raw.map((entry) => convertValue(elementType, entry))
+  return raw.map((entry) => convertValue(elementType, entry, options))
 }
 
 function convertMapValue(
   keyType: TypeDescriptor,
   valueType: TypeDescriptor,
-  value: unknown
+  value: unknown,
+  options: RowMapperOptions
 ): unknown {
   const raw = parseJsonValue(value)
   if (!raw || typeof raw !== 'object')
@@ -325,16 +344,16 @@ function convertMapValue(
     for (const entry of raw) {
       if (!Array.isArray(entry) || entry.length < 2)
         continue
-      const convertedKey = convertValue(keyType, entry[0])
-      mapped[String(convertedKey)] = convertValue(valueType, entry[1])
+      const convertedKey = convertValue(keyType, entry[0], options)
+      mapped[String(convertedKey)] = convertValue(valueType, entry[1], options)
     }
     return mapped
   }
 
   const mapped: RowObject = {}
   for (const [key, entryValue] of Object.entries(raw)) {
-    const convertedKey = convertValue(keyType, key)
-    mapped[String(convertedKey)] = convertValue(valueType, entryValue)
+    const convertedKey = convertValue(keyType, key, options)
+    mapped[String(convertedKey)] = convertValue(valueType, entryValue, options)
   }
 
   return mapped
@@ -372,20 +391,23 @@ function convertNumber(value: unknown): unknown {
   return value
 }
 
-function convertInteger(value: unknown): unknown {
+function convertInteger(value: unknown, encodeBigInt?: (value: bigint) => unknown): unknown {
   if (typeof value === 'bigint')
-    return value
+    return encodeBigInt ? encodeBigInt(value) : value
 
   if (typeof value === 'number') {
-    if (Number.isInteger(value))
-      return BigInt(value)
+    if (Number.isInteger(value)) {
+      const bigintValue = BigInt(value)
+      return encodeBigInt ? encodeBigInt(bigintValue) : bigintValue
+    }
     return value
   }
 
   if (typeof value === 'string') {
     try {
       // Preserve integer semantics for BIGINT/DECIMAL(scale=0) by returning bigint.
-      return BigInt(value)
+      const bigintValue = BigInt(value)
+      return encodeBigInt ? encodeBigInt(bigintValue) : bigintValue
     } catch {
       return value
     }
@@ -394,6 +416,16 @@ function convertInteger(value: unknown): unknown {
   return value
 }
 
+function convertTimestamp(
+  value: unknown,
+  encodeTimestamp?: (value: string) => unknown
+): unknown {
+  if (typeof value !== 'string')
+    return value
+
+  return encodeTimestamp ? encodeTimestamp(value) : value
+}
+
 function convertBoolean(value: unknown): unknown {
   if (typeof value === 'boolean')
     return value

src/types.ts

@@ -235,6 +235,14 @@ export type RowArray = unknown[]
 /** Row data as JSON object */
 export type RowObject = Record<string, unknown>
 
+/** Options for row mapping */
+export type RowMapperOptions = {
+  /** Hook to transform bigint values (e.g., to string or number) */
+  encodeBigInt?: (value: bigint) => unknown
+  /** Hook to transform timestamp-like values (e.g., to Date) */
+  encodeTimestamp?: (value: string) => unknown
+}
+
 /** Format for fetchRow/fetchAll */
 export type FetchRowFormat = 'JSON_ARRAY' | 'JSON_OBJECT'
 
@@ -248,12 +256,16 @@ export type FetchStreamOptions = SignalOptions & {
 
 /** Options for fetchRow */
 export type FetchRowsOptions = SignalOptions & {
-  /** Callback for each row */
-  onEachRow?: (row: RowArray | RowObject) => void
   /** Row format (default: JSON_ARRAY) */
   format?: FetchRowFormat
   /** Optional logger for lifecycle events */
   logger?: Logger
+  /** Callback for each row */
+  onEachRow?: (row: RowArray | RowObject) => void
+  /** Customize bigint conversion for JSON_OBJECT rows */
+  encodeBigInt?: RowMapperOptions['encodeBigInt']
+  /** Customize TIMESTAMP* conversion for JSON_OBJECT rows */
+  encodeTimestamp?: RowMapperOptions['encodeTimestamp']
 }
 
 /** Options for fetchAll */
@@ -262,6 +274,10 @@ export type FetchAllOptions = SignalOptions & {
   format?: FetchRowFormat
   /** Optional logger for lifecycle events */
   logger?: Logger
+  /** Customize bigint conversion for JSON_OBJECT rows */
+  encodeBigInt?: RowMapperOptions['encodeBigInt']
+  /** Customize TIMESTAMP* conversion for JSON_OBJECT rows */
+  encodeTimestamp?: RowMapperOptions['encodeTimestamp']
 }
 
 /** Result from mergeStreamToExternalLink callback */

test/createRowMapper.spec.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect } from 'vitest'
 import { createRowMapper } from '../src/createRowMapper.js'
-import type { FetchRowsOptions, StatementManifest } from '../src/types.js'
+import type { FetchRowsOptions, RowMapperOptions, StatementManifest } from '../src/types.js'
 
 type Column = StatementManifest['schema']['columns'][number]
 
@@ -23,10 +23,11 @@ function buildManifest(columns: Column[]): StatementManifest {
 function mapRow(
   columns: Column[],
   row: unknown[],
-  format: FetchRowsOptions['format'] = 'JSON_OBJECT'
+  format: FetchRowsOptions['format'] = 'JSON_OBJECT',
+  options?: RowMapperOptions
 ) {
   const manifest = buildManifest(columns)
-  const mapper = createRowMapper(manifest, format)
+  const mapper = createRowMapper(manifest, format, options)
   return mapper(row)
 }
 
@@ -53,6 +54,21 @@ describe('createRowMapper', () => {
     })
   })
 
+  it('applies encodeBigInt to bigint values', () => {
+    const columns: Column[] = [
+      { name: 'big_col', type_text: 'BIGINT', type_name: 'BIGINT', position: 0 },
+      { name: 'nested', type_text: 'STRUCT<inner: BIGINT>', type_name: 'STRUCT', position: 1 },
+    ]
+    const row = ['9007199254740993', '{"inner":"42"}']
+    const options: RowMapperOptions = {
+      encodeBigInt: (value) => value.toString(),
+    }
+    expect(mapRow(columns, row, 'JSON_OBJECT', options)).toEqual({
+      big_col: '9007199254740993',
+      nested: { inner: '42' },
+    })
+  })
+
   it('converts float and double values to numbers', () => {
     const columns: Column[] = [
       { name: 'float_col', type_text: 'FLOAT', type_name: 'FLOAT', position: 0 },
@@ -91,6 +107,30 @@ describe('createRowMapper', () => {
     })
   })
 
+  it('applies encodeTimestamp to TIMESTAMP types only', () => {
+    const columns: Column[] = [
+      { name: 'd', type_text: 'DATE', type_name: 'DATE', position: 0 },
+      { name: 't', type_text: 'TIME', type_name: 'TIME', position: 1 },
+      { name: 'ts', type_text: 'TIMESTAMP', type_name: 'TIMESTAMP', position: 2 },
+      { name: 'ts_ltz', type_text: 'TIMESTAMP_LTZ', type_name: 'TIMESTAMP_LTZ', position: 3 },
+    ]
+    const row = [
+      '2024-01-02',
+      '03:04:05',
+      '2024-01-02T03:04:05.123Z',
+      '2024-01-02T03:04:05.123Z',
+    ]
+    const options: RowMapperOptions = {
+      encodeTimestamp: (value) => `ts:${value}`,
+    }
+    expect(mapRow(columns, row, 'JSON_OBJECT', options)).toEqual({
+      d: '2024-01-02',
+      t: '03:04:05',
+      ts: 'ts:2024-01-02T03:04:05.123Z',
+      ts_ltz: 'ts:2024-01-02T03:04:05.123Z',
+    })
+  })
+
   it('converts boolean string values', () => {
     const columns: Column[] = [
       { name: 'flag', type_text: 'BOOLEAN', type_name: 'BOOLEAN', position: 0 },