diff --git a/Date Format Handling for ClickHouse Events API.md b/Date Format Handling for ClickHouse Events API.md new file mode 100644 index 0000000..6a667f7 --- /dev/null +++ b/Date Format Handling for ClickHouse Events API.md @@ -0,0 +1,47 @@ + +# Date Format Handling for ClickHouse Events API + +## Problem Description + +The event tracking API was experiencing issues with date format compatibility when inserting records into the ClickHouse database. ClickHouse has specific requirements for datetime formats, particularly for its `DateTime64` type fields, which weren't being properly addressed in the original implementation. + +## Root Cause + +- JavaScript's default date serialization (`toISOString()`) produces formats like `2023-08-24T12:34:56.789Z`, which include `T` as a separator and `Z` as the UTC timezone indicator +- ClickHouse prefers datetime values in the format `YYYY-MM-DD HH:MM:SS.SSS` for seamless parsing +- The mismatch between these formats was causing insertion errors in the database + +## Solution Implemented + +We created a `formatDateTime` utility function that properly formats JavaScript Date objects for ClickHouse compatibility: + +```typescript +const formatDateTime = (date: Date) => { + return date.toISOString().replace('T', ' ').replace('Z', ''); +}; +``` + +This function: +1. Takes a JavaScript Date object as input +2. Converts it to ISO format string +3. Replaces the 'T' separator with a space +4. Removes the trailing 'Z' UTC indicator + +The solution was applied to all date fields in the event payload: +- `event_time` +- `link_created_at` +- `link_expires_at` + +## Additional Improvements + +- We standardized date handling by using a consistent `currentTime` variable +- Added type checking for JSON fields to ensure proper serialization +- Improved error handling for date parsing failures + +## Best Practices for ClickHouse Date Handling + +1. Always format dates as `YYYY-MM-DD HH:MM:SS.SSS` when inserting into ClickHouse +2. Use consistent date handling utilities across your application +3. Consider timezone handling explicitly when needed +4. For query parameters, use ClickHouse's `parseDateTimeBestEffort` function when possible +5. Test with various date formats and edge cases to ensure robustness diff --git a/app/api/events/track/route.ts b/app/api/events/track/route.ts index 45918e8..fd321f9 100644 --- a/app/api/events/track/route.ts +++ b/app/api/events/track/route.ts @@ -3,6 +3,11 @@ import { Event } from '../../types'; import { v4 as uuid } from 'uuid'; import clickhouse from '@/lib/clickhouse'; +// 将时间格式化为ClickHouse兼容的格式:YYYY-MM-DD HH:MM:SS.SSS +const formatDateTime = (date: Date) => { + return date.toISOString().replace('T', ' ').replace('Z', ''); +}; + // Handler for POST request to track events export async function POST(req: NextRequest) { try { @@ -17,11 +22,14 @@ export async function POST(req: NextRequest) { ); } + // 获取当前时间并格式化 + const currentTime = formatDateTime(new Date()); + // Set default values for required fields if missing const event: Event = { // Core event fields event_id: eventData.event_id || uuid(), - event_time: eventData.event_time || new Date().toISOString(), + event_time: eventData.event_time ? formatDateTime(new Date(eventData.event_time)) : currentTime, event_type: eventData.event_type, event_attributes: eventData.event_attributes || '{}', @@ -32,8 +40,8 @@ export async function POST(req: NextRequest) { link_title: eventData.link_title || '', link_original_url: eventData.link_original_url || '', link_attributes: eventData.link_attributes || '{}', - link_created_at: eventData.link_created_at || new Date().toISOString(), - link_expires_at: eventData.link_expires_at || null, + link_created_at: eventData.link_created_at ? formatDateTime(new Date(eventData.link_created_at)) : currentTime, + link_expires_at: eventData.link_expires_at ? formatDateTime(new Date(eventData.link_expires_at)) : null, link_tags: eventData.link_tags || '[]', // User information @@ -82,6 +90,29 @@ export async function POST(req: NextRequest) { conversion_value: eventData.conversion_value || 0, }; + // 确保JSON字符串字段的正确处理 + if (typeof event.event_attributes === 'object') { + event.event_attributes = JSON.stringify(event.event_attributes); + } + if (typeof event.link_attributes === 'object') { + event.link_attributes = JSON.stringify(event.link_attributes); + } + if (typeof event.user_attributes === 'object') { + event.user_attributes = JSON.stringify(event.user_attributes); + } + if (typeof event.team_attributes === 'object') { + event.team_attributes = JSON.stringify(event.team_attributes); + } + if (typeof event.project_attributes === 'object') { + event.project_attributes = JSON.stringify(event.project_attributes); + } + if (typeof event.qr_code_attributes === 'object') { + event.qr_code_attributes = JSON.stringify(event.qr_code_attributes); + } + if (typeof event.link_tags === 'object') { + event.link_tags = JSON.stringify(event.link_tags); + } + // Insert event into ClickHouse await clickhouse.insert({ table: 'events',