eventid

.env

@@ -0,0 +1,31 @@
+PORT=3007
+
+MONGO_URL="mongodb://10.0.1.41:27017"
+
+# ClickHouse Configuration
+CLICKHOUSE_HOST=10.0.1.60
+CLICKHOUSE_PORT=8123
+CLICKHOUSE_USER=admin
+CLICKHOUSE_PASSWORD=your_secure_password
+CLICKHOUSE_DATABASE=shorturl_analytics
+CLICKHOUSE_URL=http://10.0.1.60:8123
+
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_PASSWORD=""
+
+# Supabase Configuration
+SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
+SUPABASE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc0NDM0NTY0MywiZXhwIjoyMDU5OTIxNjQzfQ.ZenTsEAdGiDu1DCCOT7G8xxvgFXKLl4qhHB-AhSVf6w"
+SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
+DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
+
+
+# Next.js Public Environment Variables (accessible in browser)
+NEXT_PUBLIC_SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
+NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
+DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
+
+NEXT_PUBLIC_LIMQ_API="https://app.upj.to"
+# Application URL for redirects (replace with your production URL)
+NEXT_PUBLIC_SITE_URL="https://main.upj.to"

.gitignore

@@ -31,7 +31,7 @@ yarn-error.log*
 .pnpm-debug.log*
 
 # env files (can opt-in for committing if needed)
-.env*
+# .env
 
 # vercel
 .vercel

README-auth-setup.md

@@ -0,0 +1,42 @@
+# 身份验证重定向 URL 配置指南
+
+## 问题：注册后确认邮件链接指向 localhost
+
+如果您在生产环境中使用此应用，并且发现用户注册后收到的确认邮件中的链接指向 `localhost` 而非您的实际网站域名，请按照以下步骤解决：
+
+## 解决方案
+
+### 1. 设置环境变量
+
+在项目根目录的 `.env.production` 文件中，确保 `NEXT_PUBLIC_SITE_URL` 变量设置为您的实际生产域名：
+
+```
+NEXT_PUBLIC_SITE_URL="https://您的真实域名.com"
+```
+
+### 2. 在 Supabase 控制台中配置
+
+登录 [Supabase 控制台](https://app.supabase.com/)，然后：
+
+1. 选择您的项目
+2. 导航到 **Authentication** > **URL Configuration**
+3. 在 **Site URL** 字段中输入您的实际网站 URL
+4. 在 **Redirect URLs** 部分添加：
+   - `https://您的真实域名.com/auth/callback`
+
+### 3. 本地开发与生产环境
+
+- **开发环境**：使用 `.env.development` 文件中的设置，通常为 `http://localhost:3007`
+- **生产环境**：使用 `.env.production` 文件中的设置，应为您的实际域名
+
+### 4. 部署后验证
+
+项目重新部署后：
+1. 尝试注册一个新账户
+2. 检查收到的确认邮件，确认链接现在指向您的实际域名而非 localhost
+
+## 技术说明
+
+身份验证流程中，应用使用环境变量 `NEXT_PUBLIC_SITE_URL` 构建重定向 URL。如果未设置此变量，它会回退到使用 `window.location.origin`，这在本地开发时会是 `localhost`。
+
+通过正确设置此变量，您可以确保无论在何处运行应用，邮件中的链接都能正确指向应用的实际位置。 

README-google-auth.md

@@ -0,0 +1,50 @@
+# 配置 Google 登录功能
+
+为了启用 Google 登录功能，您需要在 Supabase 和 Google Cloud Platform 进行配置。
+
+## 步骤 1: 创建 Google OAuth 客户端
+
+1. 访问 [Google Cloud Console](https://console.cloud.google.com/)
+2. 创建一个新项目或选择现有项目
+3. 在左侧菜单中导航到 "API 和服务" > "OAuth 同意屏幕"
+4. 选择用户类型（外部或内部），然后点击"创建"
+5. 填写必要的信息（应用名称、用户支持电子邮件等）并保存
+6. 导航到 "API 和服务" > "凭据"
+7. 点击"创建凭据" > "OAuth 客户端 ID"
+8. 应用类型选择 "Web 应用"
+9. 名称中输入您的应用名称
+10. 添加以下已获授权的重定向 URI:
+    - `https://mwwvqwevplndzvmqmrxa.supabase.co/auth/v1/callback`
+11. 点击"创建"
+12. 复制生成的 "客户端 ID" 和 "客户端密钥"
+
+## 步骤 2: 在 Supabase 中配置 Google 提供商
+
+1. 登录 [Supabase 仪表板](https://app.supabase.com)
+2. 选择您的项目
+3. 导航到 "身份验证" > "提供商"
+4. 找到 Google 提供商并启用它
+5. 粘贴您刚才获取的 "客户端 ID" 和 "客户端密钥"
+6. 保存配置
+
+## 步骤 3: 更新重定向 URL（如有需要）
+
+如果您的应用需要在登录后重定向到特定页面，请确保在 Google Cloud Console 和 Supabase 中配置了正确的重定向 URL。
+
+在 Supabase 中:
+1. 导航到 "身份验证" > "URL 配置"
+2. 添加您的前端 URL 到站点 URL 字段中
+3. 设置重定向 URL（通常是您的前端 URL）
+
+## 测试
+
+1. 在您的应用中，尝试使用 Google 登录
+2. 验证认证流程，确保可以成功登录并重定向到应用
+3. 检查 Supabase 中的用户数据，确认新用户已创建
+
+## 故障排除
+
+- 确保重定向 URI 完全匹配
+- 确保 OAuth 同意屏幕已正确配置
+- 查看 Supabase 和应用程序中的日志以获取详细的错误信息
+- 如果遇到 CORS 错误，检查您的站点 URL 配置 

app/api/activities/readme.md

@@ -0,0 +1,126 @@
+# Activities API Documentation
+
+## Overview
+The Activities API provides event tracking data for short URLs. It allows retrieving visitor activity information based on various filters such as URL slug, domain, and date ranges.
+
+## Endpoint
+```
+GET /api/activities
+```
+
+## Request Parameters
+
+| Parameter  | Type    | Required | Description |
+|------------|---------|----------|-------------|
+| `slug`     | string  | No*      | The short URL slug to filter events by |
+| `domain`   | string  | No*      | The domain to filter events by |
+| `startTime`| string  | No*      | Start time for date range filter (ISO format) |
+| `endTime`  | string  | No*      | End time for date range filter (ISO format) |
+| `page`     | integer | No       | Page number for pagination (default: 1) |
+| `pageSize` | integer | No       | Number of records per page (default: 50) |
+| `format`   | string  | No       | Response format, set to 'csv' for CSV output (default: JSON) |
+
+\* Either `slug`+`domain` combination OR at least one of `startTime`/`endTime` must be provided.
+
+## Response Formats
+
+### JSON Format (Default)
+JSON responses include the following structure:
+
+```json
+{
+  "success": true,
+  "data": [
+    {
+      "id": "event-id",
+      "type": "event-type",
+      "time": "timestamp",
+      "visitor": {
+        "id": "visitor-id",
+        "ipAddress": "ip-address",
+        "userAgent": "user-agent-string",
+        "referrer": "referrer-url"
+      },
+      "device": {
+        "type": "device-type",
+        "browser": "browser-name",
+        "os": "operating-system"
+      },
+      "location": {
+        "country": "country-code",
+        "city": "city-name"
+      },
+      "link": {
+        "id": "link-id",
+        "slug": "link-slug",
+        "originalUrl": "original-url",
+        "label": "link-label",
+        "tags": ["tag1", "tag2"]
+      },
+      "utm": {
+        "source": "utm-source",
+        "medium": "utm-medium",
+        "campaign": "utm-campaign",
+        "term": "utm-term",
+        "content": "utm-content"
+      }
+    }
+  ],
+  "meta": {
+    "total": 100,
+    "page": 1,
+    "pageSize": 50
+  }
+}
+```
+
+In case of an error:
+```json
+{
+  "success": false,
+  "data": null,
+  "error": "Error message description"
+}
+```
+
+### CSV Format
+When `format=csv` is specified, the response is returned as plain text in CSV format with the following columns:
+- `time`: Timestamp of the event
+- `activity`: Type of activity/event
+- `campaign`: UTM campaign value (defaults to "demo" if not found)
+- `clientId`: Visitor ID
+- `originPath`: Original request path or referrer URL
+
+## Examples
+
+### Get activities for a specific short URL
+```
+GET /api/activities?slug=promo123&domain=googleads.link
+```
+
+### Get activities within a date range
+```
+GET /api/activities?startTime=2023-06-01T00:00:00Z&endTime=2023-06-30T23:59:59Z
+```
+
+### Get events as CSV
+```
+GET /api/activities?slug=promo123&domain=googleads.link&format=csv
+```
+
+### Pagination example
+```
+GET /api/activities?slug=promo123&domain=googleads.link&page=2&pageSize=20
+```
+
+## Error Codes
+
+| Status Code | Description |
+|-------------|-------------|
+| 400         | Missing required parameters |
+| 500         | Server error while processing the request |
+
+## Notes
+- For privacy and security reasons, some fields may be omitted or anonymized based on user settings.
+- The CSV format is optimized for easy import into spreadsheet applications.
+- When using the CSV format, the response is returned as plain text rather than a downloadable file.

app/api/activities/route.ts

@@ -0,0 +1,250 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { getEvents } from '@/lib/analytics';
+import { ApiResponse } from '@/lib/types';
+
+// Extended Event type with required fields
+interface EventWithFullPath {
+  event_id?: string;
+  event_time?: string;
+  event_type?: string;
+  visitor_id?: string;
+  ip_address?: string;
+  req_full_path?: string;
+  referrer?: string;
+  event_attributes?: string | Record<string, unknown>;
+  link_tags?: string | string[];
+  link_id?: string;
+  link_slug?: string;
+  link_original_url?: string;
+  link_label?: string;
+  device_type?: string;
+  browser?: string;
+  os?: string;
+  country?: string;
+  city?: string;
+  [key: string]: unknown;
+}
+
+export async function GET(request: NextRequest) {
+  try {
+    const { searchParams } = new URL(request.url);
+    
+    // Get parameters
+    const slug = searchParams.get('slug');
+    const domain = searchParams.get('domain');
+    const format = searchParams.get('format');
+    
+    // Optional date range parameters
+    const startTime = searchParams.get('startTime') || undefined;
+    const endTime = searchParams.get('endTime') || undefined;
+    
+    // Check if either slug or domain is provided without the other
+    if ((slug && !domain) || (!slug && domain)) {
+      return NextResponse.json({
+        success: false,
+        error: 'Both slug and domain parameters must be provided together'
+      }, { status: 400 });
+    }
+    
+    // Ensure either slug+domain or date range is provided
+    if ((!slug && !domain) && (!startTime && !endTime)) {
+      return NextResponse.json({
+        success: false,
+        error: 'Missing filter parameters: provide either slug+domain or date range'
+      }, { status: 400 });
+    }
+    
+    // Construct the shortUrl from domain and slug if both are provided
+    let shortUrl = undefined;
+    if (slug && domain) {
+      shortUrl = `https://${domain}/${slug}`;
+      
+      // Log the request for debugging
+      console.log('Activities API received parameters:', {
+        slug,
+        domain,
+        shortUrl,
+        startTime,
+        endTime
+      });
+    } else {
+      console.log('Activities API using time range filter:', {
+        startTime,
+        endTime
+      });
+    }
+    
+    // Set default page size and page
+    const page = parseInt(searchParams.get('page') || '1');
+    const pageSize = parseInt(searchParams.get('pageSize') || '50');
+    
+    // Get events for the specified filters
+    const { events, total } = await getEvents({
+      linkSlug: slug || undefined,
+      page,
+      pageSize,
+      startTime,
+      endTime,
+      sortBy: 'event_time',
+      sortOrder: 'desc'
+    });
+    
+    // If format=csv, return CSV format data
+    if (format === 'csv') {
+      // CSV header line
+      let csvContent = 'time,activity,campaign,clientId,originPath\n';
+      
+      // Helper function to extract utm_campaign from URL
+      const extractUtmCampaign = (url: string | null | undefined): string => {
+        if (!url) return 'demo';
+        
+        try {
+          // Try to parse URL and extract utm_campaign parameter
+          const urlObj = new URL(url.startsWith('http') ? url : `https://example.com${url}`);
+          const campaign = urlObj.searchParams.get('utm_campaign');
+          if (campaign) return campaign;
+          
+          // If utm_campaign is not found or URL parsing fails, use regex as fallback
+          const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
+          if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
+        } catch {
+          // If URL parsing fails, try regex directly
+          const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
+          if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
+        }
+        
+        return 'demo'; // Default value
+      };
+      
+      // Process each event record
+      events.forEach(event => {
+        // 使用类型断言处理扩展字段
+        const eventWithFullPath = event as unknown as EventWithFullPath;
+        
+        // Get the full URL from appropriate field
+        // Try different possible fields that might contain the URL
+        const fullUrl = eventWithFullPath.req_full_path || eventWithFullPath.referrer || '';
+        
+        // Extract campaign from URL
+        const campaign = extractUtmCampaign(fullUrl);
+        
+        // Format time
+        const time = eventWithFullPath.event_time ? 
+          new Date(eventWithFullPath.event_time).toISOString().replace('T', ' ').slice(0, 19) : 
+          '';
+        
+        // Determine activity (event_type)
+        const activity = eventWithFullPath.event_type || '';
+        
+        // 修改：使用link_label替代visitor_id作为clientId
+        const clientId = eventWithFullPath.link_label || 'undefined';
+        
+        // Original path - 修正：使用link_original_url作为原始URL来源
+        const originPath = eventWithFullPath.link_original_url || 'undefined';
+        
+        // Add to CSV content
+        csvContent += `${time},${activity},${campaign},${clientId},${originPath}\n`;
+      });
+      
+      // No need to generate filename since we're not using Content-Disposition header
+      
+      // Return CSV response without forcing download
+      return new NextResponse(csvContent, {
+        headers: {
+          'Content-Type': 'text/plain'
+        }
+      });
+    }
+    
+    // Process the events to extract useful information
+    const processedEvents = events.map(event => {
+      // Parse JSON strings to objects safely
+      let eventAttributes: Record<string, unknown> = {};
+      
+      try {
+        if (typeof event.event_attributes === 'string') {
+          eventAttributes = JSON.parse(event.event_attributes);
+        } else if (typeof event.event_attributes === 'object') {
+          eventAttributes = event.event_attributes;
+        }
+      } catch {
+        // Keep default empty object if parsing fails
+      }
+      
+      // Extract tags
+      let tags: string[] = [];
+      
+      try {
+        if (typeof event.link_tags === 'string') {
+          const parsedTags = JSON.parse(event.link_tags);
+          if (Array.isArray(parsedTags)) {
+            tags = parsedTags;
+          }
+        } else if (Array.isArray(event.link_tags)) {
+          tags = event.link_tags;
+        }
+      } catch {
+        // If parsing fails, keep tags as empty array
+      }
+      
+      // Return a simplified event object
+      return {
+        id: event.event_id,
+        type: event.event_type,
+        time: event.event_time,
+        visitor: {
+          id: event.visitor_id,
+          ipAddress: event.ip_address,
+          userAgent: eventAttributes.user_agent as string || null,
+          referrer: eventAttributes.referrer as string || null
+        },
+        device: {
+          type: event.device_type,
+          browser: event.browser,
+          os: event.os
+        },
+        location: {
+          country: event.country,
+          city: event.city
+        },
+        link: {
+          id: event.link_id,
+          slug: event.link_slug,
+          originalUrl: event.link_original_url,
+          label: event.link_label,
+          tags
+        },
+        utm: {
+          source: eventAttributes.utm_source as string || null,
+          medium: eventAttributes.utm_medium as string || null,
+          campaign: eventAttributes.utm_campaign as string || null,
+          term: eventAttributes.utm_term as string || null,
+          content: eventAttributes.utm_content as string || null
+        }
+      };
+    });
+    
+    // Return processed events
+    const response: ApiResponse<typeof processedEvents> = {
+      success: true,
+      data: processedEvents,
+      meta: {
+        total,
+        page,
+        pageSize
+      }
+    };
+    
+    return NextResponse.json(response);
+  } catch (error) {
+    console.error('Error retrieving activities:', error);
+    
+    const response: ApiResponse<null> = {
+      success: false,
+      data: null,
+      error: error instanceof Error ? error.message : 'An error occurred while retrieving activities'
+    };
+    
+    return NextResponse.json(response, { status: 500 });
+  }
+} 

app/api/events/devices/route.ts

@@ -18,7 +18,9 @@ export async function GET(request: NextRequest) {
       // 添加团队、项目和标签筛选
       teamIds: teamIds.length > 0 ? teamIds : undefined,
       projectIds: projectIds.length > 0 ? projectIds : undefined,
-      tagIds: tagIds.length > 0 ? tagIds : undefined
+      tagIds: tagIds.length > 0 ? tagIds : undefined,
+      // 添加子路径筛选
+      subpath: searchParams.get('subpath') || undefined
     });
 
     const response: ApiResponse<typeof data> = {

app/api/events/geo/route.ts

@@ -22,7 +22,9 @@ export async function GET(request: NextRequest) {
       // 添加团队、项目和标签筛选
       teamIds: teamIds.length > 0 ? teamIds : undefined,
       projectIds: projectIds.length > 0 ? projectIds : undefined,
-      tagIds: tagIds.length > 0 ? tagIds : undefined
+      tagIds: tagIds.length > 0 ? tagIds : undefined,
+      // 添加子路径筛选
+      subpath: searchParams.get('subpath') || undefined
     });
 
     const response: ApiResponse<typeof data> = {

app/api/events/path-analytics/route.ts

@@ -0,0 +1,80 @@
+import { NextRequest, NextResponse } from 'next/server';
+import type { ApiResponse } from '@/lib/types';
+import { executeQuery } from '@/lib/clickhouse';
+
+export async function GET(request: NextRequest) {
+  try {
+    // 获取查询参数
+    const searchParams = request.nextUrl.searchParams;
+    const startTime = searchParams.get('startTime');
+    const endTime = searchParams.get('endTime');
+    const linkId = searchParams.get('linkId');
+    
+    if (!startTime || !endTime || !linkId) {
+      return NextResponse.json({
+        success: false,
+        error: 'Missing required parameters'
+      }, { status: 400 });
+    }
+
+    // 查询链接的点击事件
+    const query = `
+      SELECT event_attributes
+      FROM events
+      WHERE link_id = '${linkId}'
+      AND event_time >= parseDateTimeBestEffort('${startTime}')
+      AND event_time <= parseDateTimeBestEffort('${endTime}')
+      AND event_type = 'click'
+    `;
+    
+    const events = await executeQuery(query);
+    
+    // 处理事件数据，按路径分组
+    const pathMap = new Map<string, number>();
+    let totalClicks = 0;
+    
+    events.forEach((event: any) => {
+      try {
+        if (event.event_attributes) {
+          const attrs = JSON.parse(event.event_attributes);
+          if (attrs.full_url) {
+            // 提取URL的路径和参数部分
+            const url = new URL(attrs.full_url);
+            const pathWithParams = url.pathname + (url.search || '');
+            
+            // 更新路径计数
+            const currentCount = pathMap.get(pathWithParams) || 0;
+            pathMap.set(pathWithParams, currentCount + 1);
+            totalClicks++;
+          }
+        }
+      } catch (error) {
+        // 忽略解析错误
+      }
+    });
+    
+    // 转换为数组并按点击数排序
+    const pathData = Array.from(pathMap.entries())
+      .map(([path, count]) => ({
+        path,
+        count,
+        percentage: totalClicks > 0 ? count / totalClicks : 0,
+      }))
+      .sort((a, b) => b.count - a.count);
+    
+    const response: ApiResponse<typeof pathData> = {
+      success: true, 
+      data: pathData,
+      meta: { total: totalClicks }
+    };
+    
+    return NextResponse.json(response);
+  } catch (error) {
+    console.error('Error fetching path analytics data:', error);
+    const response: ApiResponse<null> = {
+      success: false,
+      error: error instanceof Error ? error.message : 'Internal server error'
+    };
+    return NextResponse.json(response, { status: 500 });
+  }
+} 

app/api/events/route.ts

@@ -14,6 +14,7 @@ export async function GET(request: NextRequest) {
     const linkId = searchParams.get('linkId') || undefined;
     const linkSlug = searchParams.get('linkSlug') || undefined;
     const userId = searchParams.get('userId') || undefined;
+    const subpath = searchParams.get('subpath') || undefined;
     
     // 获取可能存在的多个团队、项目和标签ID
     const teamIds = searchParams.getAll('teamId');
@@ -26,6 +27,7 @@ export async function GET(request: NextRequest) {
     const sortOrder = (searchParams.get('sortOrder') as 'asc' | 'desc') || undefined;
     
     console.log("API接收到的tagIds:", tagIds); // 添加日志便于调试
+    console.log("API接收到的subpath:", subpath); // 添加日志便于调试
     
     // 获取事件列表
     const params: EventsQueryParams = {
@@ -35,6 +37,7 @@ export async function GET(request: NextRequest) {
       linkId,
       linkSlug,
       userId,
+      subpath,
       teamIds: teamIds.length > 0 ? teamIds : undefined,
       projectIds: projectIds.length > 0 ? projectIds : undefined,
       tagIds: tagIds.length > 0 ? tagIds : undefined,
@@ -44,6 +47,9 @@ export async function GET(request: NextRequest) {
       sortOrder
     };
     
+    // 记录完整的参数用于调试
+    console.log("完整请求参数:", JSON.stringify(params));
+    
     const result = await getEvents(params);
     
     const response: ApiResponse<typeof result.events> = {

app/api/events/summary/route.ts

@@ -11,13 +11,22 @@ export async function GET(request: NextRequest) {
     const projectIds = searchParams.getAll('projectId');
     const tagIds = searchParams.getAll('tagId');
     
+    // Add debug log to check if linkId is being received
+    const linkId = searchParams.get('linkId');
+    const subpath = searchParams.get('subpath');
+    console.log('Summary API received linkId:', linkId);
+    console.log('Summary API received subpath:', subpath);
+    console.log('Summary API full parameters:', Object.fromEntries(searchParams.entries()));
+    console.log('Summary API URL:', request.url);
+    
     const summary = await getEventsSummary({
       startTime: searchParams.get('startTime') || undefined,
       endTime: searchParams.get('endTime') || undefined,
       linkId: searchParams.get('linkId') || undefined,
       teamIds: teamIds.length > 0 ? teamIds : undefined,
       projectIds: projectIds.length > 0 ? projectIds : undefined,
-      tagIds: tagIds.length > 0 ? tagIds : undefined
+      tagIds: tagIds.length > 0 ? tagIds : undefined,
+      subpath: searchParams.get('subpath') || undefined
     });
 
     const response: ApiResponse<typeof summary> = {

app/api/events/time-series/route.ts

@@ -28,7 +28,9 @@ export async function GET(request: NextRequest) {
       // 添加团队、项目和标签筛选
       teamIds: teamIds.length > 0 ? teamIds : undefined,
       projectIds: projectIds.length > 0 ? projectIds : undefined,
-      tagIds: tagIds.length > 0 ? tagIds : undefined
+      tagIds: tagIds.length > 0 ? tagIds : undefined,
+      // 添加子路径筛选
+      subpath: searchParams.get('subpath') || undefined
     });
 
     const response: ApiResponse<typeof data> = {

app/api/events/track/readme.md

@@ -0,0 +1,208 @@
+
+# 事件跟踪接口说明
+
+## 概述
+该接口用于跟踪用户交互事件并将数据存储到 ClickHouse 数据库中。支持记录各种类型的事件，并可包含与链接、用户、团队、项目等相关的详细信息。
+
+## 接口信息
+- **URL**: `/api/events/track`
+- **方法**: `POST`
+- **Content-Type**: `application/json`
+
+## 请求参数
+
+### 必填字段
+| 参数 | 类型 | 描述 |
+|------|------|------|
+| `event_type` | string | 事件类型，如 'click', 'view', 'conversion' |
+
+### 核心事件字段
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `event_id` | string | 否 | 事件唯一标识符，不提供时自动生成UUID |
+| `event_time` | string/Date | 否 | 事件发生时间，格式为ISO日期字符串，默认为当前时间 |
+| `event_attributes` | object/string | 否 | 事件相关的其他属性，可以是JSON对象或JSON字符串 |
+
+### 链接信息
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `link_id` | string | 否 | 短链接的唯一ID |
+| `link_slug` | string | 否 | 短链接的slug部分 |
+| `link_label` | string | 否 | 短链接的显示名称 |
+| `link_title` | string | 否 | 短链接的标题 |
+| `link_original_url` | string | 否 | 原始目标URL |
+| `link_attributes` | object/string | 否 | 链接相关的额外属性 |
+| `link_created_at` | string/Date | 否 | 链接创建时间 |
+| `link_expires_at` | string/Date | 否 | 链接过期时间 |
+| `link_tags` | array/string | 否 | 链接标签，可以是数组或JSON字符串 |
+
+### 用户信息
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `user_id` | string | 否 | 用户ID |
+| `user_name` | string | 否 | 用户名称 |
+| `user_email` | string | 否 | 用户邮箱 |
+| `user_attributes` | object/string | 否 | 用户相关的其他属性 |
+
+### 团队和项目信息
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `team_id` | string | 否 | 团队ID |
+| `team_name` | string | 否 | 团队名称 |
+| `team_attributes` | object/string | 否 | 团队相关的其他属性 |
+| `project_id` | string | 否 | 项目ID |
+| `project_name` | string | 否 | 项目名称 |
+| `project_attributes` | object/string | 否 | 项目相关的其他属性 |
+
+### 二维码信息
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `qr_code_id` | string | 否 | 二维码ID |
+| `qr_code_name` | string | 否 | 二维码名称 |
+| `qr_code_attributes` | object/string | 否 | 二维码相关的其他属性 |
+
+### 访问者信息
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `visitor_id` | string | 否 | 访问者唯一标识符，不提供时自动生成 |
+| `session_id` | string | 否 | 会话ID，不提供时自动生成 |
+| `ip_address` | string | 否 | 访问者IP地址，默认从请求头获取 |
+| `country` | string | 否 | 访问者所在国家 |
+| `city` | string | 否 | 访问者所在城市 |
+| `device_type` | string | 否 | 设备类型 (如 desktop, mobile, tablet) |
+| `browser` | string | 否 | 浏览器名称 |
+| `os` | string | 否 | 操作系统 |
+| `user_agent` | string | 否 | 用户代理字符串，默认从请求头获取 |
+
+### 引荐来源信息
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `referrer` | string | 否 | 引荐URL，默认从请求头获取 |
+| `utm_source` | string | 否 | UTM来源参数 |
+| `utm_medium` | string | 否 | UTM媒介参数 |
+| `utm_campaign` | string | 否 | UTM活动参数 |
+| `utm_term` | string | 否 | UTM术语参数 |
+| `utm_content` | string | 否 | UTM内容参数 |
+
+### 交互信息
+| 参数 | 类型 | 必填 | 描述 |
+|------|------|------|------|
+| `time_spent_sec` | number | 否 | 用户在页面上停留的时间（秒），默认0 |
+| `is_bounce` | boolean | 否 | 是否是跳出（只访问一个页面），默认true |
+| `is_qr_scan` | boolean | 否 | 是否来自二维码扫描，默认false |
+| `conversion_type` | string | 否 | 转化类型 |
+| `conversion_value` | number | 否 | 转化价值，默认0 |
+
+## 响应格式
+
+### 成功响应 (201 Created)
+```json
+{
+  "success": true,
+  "message": "Event tracked successfully",
+  "event_id": "uuid-of-tracked-event"
+}
+```
+
+### 错误响应
+
+#### 缺少必填字段 (400 Bad Request)
+```json
+{
+  "error": "Missing required field: event_type"
+}
+```
+
+#### 服务器错误 (500 Internal Server Error)
+```json
+{
+  "error": "Failed to track event",
+  "details": "具体错误信息"
+}
+```
+
+## 使用示例
+
+### 基本事件跟踪请求
+```javascript
+fetch('/api/events/track', {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json'
+  },
+  body: JSON.stringify({
+    event_type: 'click',
+    link_id: 'abc123',
+    link_slug: 'promo-summer',
+    link_original_url: 'https://example.com/summer-promotion'
+  })
+})
+```
+
+### 详细事件跟踪请求
+```javascript
+fetch('/api/events/track', {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json'
+  },
+  body: JSON.stringify({
+    event_type: 'conversion',
+    link_id: 'abc123',
+    link_slug: 'promo-summer',
+    link_original_url: 'https://example.com/summer-promotion',
+    event_attributes: {
+      page: '/checkout',
+      product_id: 'xyz789'
+    },
+    user_id: 'user123',
+    team_id: 'team456',
+    project_id: 'proj789',
+    visitor_id: 'vis987',
+    is_bounce: false,
+    time_spent_sec: 120,
+    conversion_type: 'purchase',
+    conversion_value: 99.99,
+    utm_source: 'email',
+    utm_campaign: 'summer_sale'
+  })
+})
+```
+
+## 注意事项
+- 所有对象类型的字段（如 `event_attributes`）可以作为对象或预先格式化的JSON字符串传递
+- 如果不提供 `event_id`、`visitor_id` 或 `session_id`，系统将自动生成
+- 时间戳字段接受ISO格式的日期字符串，并会被转换为ClickHouse兼容的格式
+
+
+UTM 测试示例。1. 电子邮件营销链接
+https://short.domain.com/summer?utm_source=newsletter&utm_medium=email&utm_campaign=summer_promo&utm_term=discount&utm_content=header
+说明: 用于电子邮件营销活动，跟踪用户从邮件头部横幅点击的流量。
+
+2. 社交媒体广告链接
+https://short.domain.com/product?utm_source=instagram&utm_medium=social&utm_campaign=fall_collection&utm_content=story
+说明: 用于 Instagram Story 广告，跟踪用户从社交媒体故事广告点击的情况。
+
+3. 搜索引擎广告链接
+https://short.domain.com/service?utm_source=google&utm_medium=cpc&utm_campaign=brand_terms&utm_term=service+name
+说明: 用于 Google Ads 广告，跟踪用户从搜索引擎付费广告点击的流量，特别是针对特定搜索词。
+
+4. QR 码链接
+https://short.domain.com/event?utm_source=flyer&utm_medium=print&utm_campaign=local_event&utm_content=qr_code&source=qr
+说明: 用于打印材料上的 QR 码，跟踪用户扫描实体宣传资料的情况。
+
+5. 合作伙伴引荐链接
+https://short.domain.com/partner?utm_source=affiliate&utm_medium=referral&utm_campaign=partner_program&utm_content=banner
+说明: 用于合作伙伴网站上的推广横幅，跟踪来自联盟营销的转化率。
+
+
+https://upj.to/5seaii?utm_source=newsletter&utm_medium=email&utm_campaign=summer_promo&utm_term=discount&utm_content=header
+
+https://upj.to/5seaii?utm_source=instagram&utm_medium=social&utm_campaign=fall_collection&utm_content=story
+
+https://upj.to/5seaii?utm_source=google&utm_medium=cpc&utm_campaign=brand_terms&utm_term=service+name
+
+
+https://upj.to/5seaii?utm_source=flyer&utm_medium=print&utm_campaign=local_event&utm_content=qr_code&source=qr
+
+https://upj.to/5seaii?utm_source=affiliate&utm_medium=referral&utm_campaign=partner_program&utm_content=banner

app/api/events/utm/route.ts

@@ -11,10 +11,11 @@ interface UtmData {
   conversions: number;
 }
 
-// 格式化日期时间字符串为ClickHouse支持的格式
-const formatDateTime = (dateStr: string): string => {
-  return dateStr.replace('T', ' ').replace('Z', '');
-};
+// 辅助函数，将日期格式化为标准格式
+function formatDateTime(dateString: string): string {
+  const date = new Date(dateString);
+  return date.toISOString().split('.')[0];
+}
 
 export async function GET(request: NextRequest) {
   try {
@@ -24,6 +25,7 @@ export async function GET(request: NextRequest) {
     const startTime = searchParams.get('startTime');
     const endTime = searchParams.get('endTime');
     const linkId = searchParams.get('linkId');
+    const subpath = searchParams.get('subpath');
     
     // 获取团队、项目和标签筛选参数
     const teamIds = searchParams.getAll('teamId');
@@ -34,6 +36,20 @@ export async function GET(request: NextRequest) {
     // 获取UTM类型参数
     const utmType = searchParams.get('utmType') || 'source';
     
+    // 添加调试日志
+    console.log('UTM API received parameters:', {
+      startTime,
+      endTime,
+      linkId,
+      subpath,
+      teamIds,
+      projectIds,
+      tagIds,
+      tagNames,
+      utmType,
+      url: request.url
+    });
+    
     // 构建WHERE子句
     let whereClause = '';
     const conditions = [];
@@ -50,6 +66,34 @@ export async function GET(request: NextRequest) {
       conditions.push(`link_id = '${linkId}'`);
     }
     
+    // 添加子路径筛选 - 使用更精确的匹配方式
+    if (subpath && subpath.trim() !== '') {
+      console.log('====== UTM API SUBPATH DEBUG ======');
+      console.log('Raw subpath param:', subpath);
+      
+      // 清理并准备subpath值
+      let cleanSubpath = subpath.trim();
+      // 移除开头的斜杠以便匹配
+      if (cleanSubpath.startsWith('/')) {
+        cleanSubpath = cleanSubpath.substring(1);
+      }
+      // 移除结尾的斜杠以便匹配
+      if (cleanSubpath.endsWith('/')) {
+        cleanSubpath = cleanSubpath.substring(0, cleanSubpath.length - 1);
+      }
+      
+      console.log('Cleaned subpath:', cleanSubpath);
+      
+      // 使用正则表达式匹配URL中的第二个路径部分
+      // 示例: 在 "https://abc.com/slug/subpath/" 中匹配 "subpath"
+      const condition = `match(JSONExtractString(event_attributes, 'full_url'), '/[^/]+/${cleanSubpath}(/|\\\\?|$)')`;
+      
+      console.log('Final SQL condition:', condition);
+      console.log('==================================');
+      
+      conditions.push(condition);
+    }
+    
     // 添加团队筛选
     if (teamIds && teamIds.length > 0) {
       // 如果只有一个团队ID

app/api/shortlinks/[id]/route.ts

@@ -0,0 +1,141 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { executeQuery } from '@/lib/clickhouse';
+import type { ApiResponse } from '@/lib/types';
+
+export async function GET(
+  request: NextRequest,
+  { params }: { params: { id: string } }
+) {
+  try {
+    // Get the id from the URL parameters
+    const { id } = params;
+    
+    if (!id) {
+      return NextResponse.json({
+        success: false,
+        error: 'ID parameter is required'
+      }, { status: 400 });
+    }
+    
+    console.log('Fetching shortlink by ID:', id);
+    
+    // Query to fetch a single shortlink by id
+    const query = `
+      SELECT 
+        id,
+        external_id,
+        type,
+        slug,
+        original_url,
+        title,
+        description,
+        attributes,
+        schema_version,
+        creator_id,
+        creator_email,
+        creator_name,
+        created_at,
+        updated_at,
+        deleted_at,
+        projects,
+        teams,
+        tags,
+        qr_codes AS qr_codes,
+        channels,
+        favorites,
+        expires_at,
+        click_count,
+        unique_visitors,
+        domain
+      FROM shorturl_analytics.shorturl
+      WHERE id = '${id}' AND deleted_at IS NULL
+      LIMIT 1
+    `;
+    
+    console.log('Executing query:', query);
+    
+    // Execute the query
+    const result = await executeQuery(query);
+    
+    // If no shortlink found with the specified ID
+    if (!Array.isArray(result) || result.length === 0) {
+      return NextResponse.json({
+        success: false,
+        error: 'Shortlink not found'
+      }, { status: 404 });
+    }
+    
+    // Process the shortlink data
+    const shortlink = result[0] as any;
+    
+    // Extract shortUrl from attributes
+    let shortUrl = '';
+    try {
+      if (shortlink.attributes && typeof shortlink.attributes === 'string') {
+        const attributes = JSON.parse(shortlink.attributes) as { shortUrl?: string };
+        shortUrl = attributes.shortUrl || '';
+      }
+    } catch (e) {
+      console.error('Error parsing shortlink attributes:', e);
+    }
+    
+    // Process teams
+    let teams: any[] = [];
+    try {
+      if (shortlink.teams && typeof shortlink.teams === 'string') {
+        teams = JSON.parse(shortlink.teams);
+      }
+    } catch (e) {
+      console.error('Error parsing teams:', e);
+    }
+    
+    // Process tags
+    let tags: any[] = [];
+    try {
+      if (shortlink.tags && typeof shortlink.tags === 'string') {
+        tags = JSON.parse(shortlink.tags);
+      }
+    } catch (e) {
+      console.error('Error parsing tags:', e);
+    }
+    
+    // Process projects
+    let projects: any[] = [];
+    try {
+      if (shortlink.projects && typeof shortlink.projects === 'string') {
+        projects = JSON.parse(shortlink.projects);
+      }
+    } catch (e) {
+      console.error('Error parsing projects:', e);
+    }
+    
+    // Format the data to match what our store expects
+    const formattedShortlink = {
+      id: shortlink.id || '',
+      externalId: shortlink.external_id || '',
+      slug: shortlink.slug || '',
+      originalUrl: shortlink.original_url || '',
+      title: shortlink.title || '',
+      shortUrl: shortUrl,
+      teams: teams,
+      projects: projects,
+      tags: tags.map((tag: any) => tag.tag_name || ''),
+      createdAt: shortlink.created_at,
+      domain: shortlink.domain || (shortUrl ? new URL(shortUrl).hostname : '')
+    };
+    
+    const response: ApiResponse<typeof formattedShortlink> = {
+      success: true,
+      data: formattedShortlink
+    };
+    
+    return NextResponse.json(response);
+  } catch (error) {
+    console.error('Error fetching shortlink by ID:', error);
+    const response: ApiResponse<null> = {
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown error occurred'
+    };
+    return NextResponse.json(response, { status: 500 });
+  }
+} 

app/api/shortlinks/byUrl/route.ts

@@ -0,0 +1,143 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { executeQuery } from '@/lib/clickhouse';
+import type { ApiResponse } from '@/lib/types';
+
+export async function GET(request: NextRequest) {
+  try {
+    // Get the url from query parameters
+    const searchParams = request.nextUrl.searchParams;
+    const url = searchParams.get('url');
+    
+    if (!url) {
+      return NextResponse.json({
+        success: false,
+        error: 'URL parameter is required'
+      }, { status: 400 });
+    }
+    
+    console.log('Fetching shortlink by URL:', url);
+    
+    // Query to fetch a single shortlink by shortUrl in attributes
+    const query = `
+      SELECT 
+        id,
+        external_id,
+        type,
+        slug,
+        original_url,
+        title,
+        description,
+        attributes,
+        schema_version,
+        creator_id,
+        creator_email,
+        creator_name,
+        created_at,
+        updated_at,
+        deleted_at,
+        projects,
+        teams,
+        tags,
+        qr_codes AS qr_codes,
+        channels,
+        favorites,
+        expires_at,
+        click_count,
+        unique_visitors,
+        domain
+      FROM shorturl_analytics.shorturl
+      WHERE JSONHas(attributes, 'shortUrl') 
+        AND JSONExtractString(attributes, 'shortUrl') = '${url}'
+        AND deleted_at IS NULL
+      LIMIT 1
+    `;
+    
+    console.log('Executing query:', query);
+    
+    // Execute the query
+    const result = await executeQuery(query);
+    
+    // If no shortlink found with the specified URL
+    if (!Array.isArray(result) || result.length === 0) {
+      return NextResponse.json({
+        success: false,
+        error: 'Shortlink not found'
+      }, { status: 404 });
+    }
+    
+    // Process the shortlink data
+    const shortlink = result[0];
+    
+    // Extract shortUrl from attributes
+    let shortUrl = '';
+    try {
+      if (shortlink.attributes && typeof shortlink.attributes === 'string') {
+        const attributes = JSON.parse(shortlink.attributes);
+        shortUrl = attributes.shortUrl || '';
+      }
+    } catch (e) {
+      console.error('Error parsing shortlink attributes:', e);
+    }
+    
+    // Process teams
+    let teams = [];
+    try {
+      if (shortlink.teams && typeof shortlink.teams === 'string') {
+        teams = JSON.parse(shortlink.teams);
+      }
+    } catch (e) {
+      console.error('Error parsing teams:', e);
+    }
+    
+    // Process tags
+    let tags = [];
+    try {
+      if (shortlink.tags && typeof shortlink.tags === 'string') {
+        tags = JSON.parse(shortlink.tags);
+      }
+    } catch (e) {
+      console.error('Error parsing tags:', e);
+    }
+    
+    // Process projects
+    let projects = [];
+    try {
+      if (shortlink.projects && typeof shortlink.projects === 'string') {
+        projects = JSON.parse(shortlink.projects);
+      }
+    } catch (e) {
+      console.error('Error parsing projects:', e);
+    }
+    
+    // Format the data to match what our store expects
+    const formattedShortlink = {
+      id: shortlink.id || '',
+      externalId: shortlink.external_id || '',
+      slug: shortlink.slug || '',
+      originalUrl: shortlink.original_url || '',
+      title: shortlink.title || '',
+      shortUrl: shortUrl,
+      teams: teams,
+      projects: projects,
+      tags: tags.map((tag) => tag.tag_name || ''),
+      createdAt: shortlink.created_at,
+      domain: shortlink.domain || (shortUrl ? new URL(shortUrl).hostname : '')
+    };
+    
+    console.log('Shortlink data formatted with externalId:', shortlink.external_id, 'Final object:', formattedShortlink);
+    
+    const response: ApiResponse<typeof formattedShortlink> = {
+      success: true,
+      data: formattedShortlink
+    };
+    
+    return NextResponse.json(response);
+  } catch (error) {
+    console.error('Error fetching shortlink by URL:', error);
+    const response: ApiResponse<null> = {
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown error occurred'
+    };
+    return NextResponse.json(response, { status: 500 });
+  }
+} 

app/api/shortlinks/exact/route.ts

@@ -0,0 +1,143 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { executeQuery } from '@/lib/clickhouse';
+import type { ApiResponse } from '@/lib/types';
+
+export async function GET(request: NextRequest) {
+  try {
+    // Get the url from query parameters
+    const searchParams = request.nextUrl.searchParams;
+    const shortUrl = searchParams.get('shortUrl');
+    
+    if (!shortUrl) {
+      return NextResponse.json({
+        success: false,
+        error: 'shortUrl parameter is required'
+      }, { status: 400 });
+    }
+    
+    console.log('Fetching shortlink by exact shortUrl:', shortUrl);
+    
+    // Query to fetch a single shortlink by shortUrl in attributes
+    const query = `
+      SELECT 
+        id,
+        external_id,
+        type,
+        slug,
+        original_url,
+        title,
+        description,
+        attributes,
+        schema_version,
+        creator_id,
+        creator_email,
+        creator_name,
+        created_at,
+        updated_at,
+        deleted_at,
+        projects,
+        teams,
+        tags,
+        qr_codes AS qr_codes,
+        channels,
+        favorites,
+        expires_at,
+        click_count,
+        unique_visitors,
+        domain
+      FROM shorturl_analytics.shorturl
+      WHERE JSONHas(attributes, 'shortUrl') 
+        AND JSONExtractString(attributes, 'shortUrl') = '${shortUrl}'
+        AND deleted_at IS NULL
+      LIMIT 1
+    `;
+    
+    console.log('Executing query:', query);
+    
+    // Execute the query
+    const result = await executeQuery(query);
+    
+    // If no shortlink found with the specified URL
+    if (!Array.isArray(result) || result.length === 0) {
+      return NextResponse.json({
+        success: false,
+        error: 'Shortlink not found'
+      }, { status: 404 });
+    }
+    
+    // Process the shortlink data
+    const shortlink = result[0] as Record<string, any>;
+    
+    // Extract shortUrl from attributes
+    let shortUrlValue = '';
+    try {
+      if (shortlink.attributes && typeof shortlink.attributes === 'string') {
+        const attributes = JSON.parse(shortlink.attributes) as { shortUrl?: string };
+        shortUrlValue = attributes.shortUrl || '';
+      }
+    } catch (e) {
+      console.error('Error parsing shortlink attributes:', e);
+    }
+    
+    // Process teams
+    let teams: any[] = [];
+    try {
+      if (shortlink.teams && typeof shortlink.teams === 'string') {
+        teams = JSON.parse(shortlink.teams);
+      }
+    } catch (e) {
+      console.error('Error parsing teams:', e);
+    }
+    
+    // Process tags
+    let tags: any[] = [];
+    try {
+      if (shortlink.tags && typeof shortlink.tags === 'string') {
+        tags = JSON.parse(shortlink.tags);
+      }
+    } catch (e) {
+      console.error('Error parsing tags:', e);
+    }
+    
+    // Process projects
+    let projects: any[] = [];
+    try {
+      if (shortlink.projects && typeof shortlink.projects === 'string') {
+        projects = JSON.parse(shortlink.projects);
+      }
+    } catch (e) {
+      console.error('Error parsing projects:', e);
+    }
+    
+    // Format the data to match what our store expects
+    const formattedShortlink = {
+      id: shortlink.id || '',
+      externalId: shortlink.external_id || '',
+      slug: shortlink.slug || '',
+      originalUrl: shortlink.original_url || '',
+      title: shortlink.title || '',
+      shortUrl: shortUrlValue,
+      teams: teams,
+      projects: projects,
+      tags: tags.map((tag: any) => tag.tag_name || ''),
+      createdAt: shortlink.created_at,
+      domain: shortlink.domain || (shortUrlValue ? new URL(shortUrlValue).hostname : '')
+    };
+    
+    console.log('Formatted shortlink with externalId:', shortlink.external_id);
+    
+    const response: ApiResponse<typeof formattedShortlink> = {
+      success: true,
+      data: formattedShortlink
+    };
+    
+    return NextResponse.json(response);
+  } catch (error) {
+    console.error('Error fetching shortlink by exact URL:', error);
+    const response: ApiResponse<null> = {
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown error occurred'
+    };
+    return NextResponse.json(response, { status: 500 });
+  }
+} 

app/api/shortlinks/route.ts

@@ -75,7 +75,8 @@ export async function GET(request: NextRequest) {
         favorites,
         expires_at,
         click_count,
-        unique_visitors
+        unique_visitors,
+        domain
       FROM shorturl_analytics.shorturl
       WHERE ${whereClause}
       ORDER BY created_at DESC

app/auth/callback/route.ts

@@ -0,0 +1,18 @@
+import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
+import { cookies } from 'next/headers';
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function GET(request: NextRequest) {
+  const requestUrl = new URL(request.url);
+  const code = requestUrl.searchParams.get('code');
+  
+  if (code) {
+    const cookieStore = cookies();
+    const supabase = createRouteHandlerClient({ cookies: () => cookieStore });
+    await supabase.auth.exchangeCodeForSession(code);
+  }
+
+  // URL to redirect to after sign in process completes
+  const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || 'https://main.upj.to';
+  return NextResponse.redirect(new URL('/analytics', siteUrl));
+} 

app/components/ClientRouteGuard.tsx

@@ -0,0 +1,45 @@
+'use client';
+
+import { useEffect, useState } from 'react';
+import { useRouter } from 'next/navigation';
+
+// 这个组件会检查 localStorage 中是否有认证令牌，如果没有则重定向到登录页面
+export default function ClientRouteGuard({ children }: { children: React.ReactNode }) {
+  const router = useRouter();
+  const [isAuthenticated, setIsAuthenticated] = useState(false);
+  const [isLoading, setIsLoading] = useState(true);
+
+  useEffect(() => {
+    // 检查 localStorage 中是否有认证令牌
+    const checkAuth = () => {
+      // 查找 Supabase 认证令牌
+      const hasAuthToken = !!localStorage.getItem('sb-mwwvqwevplndzvmqmrxa-auth-token') || 
+                           !!localStorage.getItem('sb-auth-token');
+      
+      if (!hasAuthToken) {
+        // 如果没有令牌，重定向到登录页面
+        router.push('/login');
+      } else {
+        setIsAuthenticated(true);
+      }
+      setIsLoading(false);
+    };
+
+    checkAuth();
+  }, [router]);
+
+  // 显示加载状态
+  if (isLoading) {
+    return (
+      <div className="flex items-center justify-center min-h-screen">
+        <div className="text-center">
+          <div className="animate-spin rounded-full h-16 w-16 border-t-2 border-b-2 border-blue-500 mx-auto"></div>
+          <p className="mt-4 text-lg text-gray-700 dark:text-gray-300">加载中...</p>
+        </div>
+      </div>
+    );
+  }
+
+  // 只有当用户已认证时才显示子组件
+  return isAuthenticated ? <>{children}</> : null;
+} 

app/components/analytics/PathAnalytics.tsx

@@ -0,0 +1,162 @@
+import React, { useState, useEffect } from 'react';
+
+interface PathAnalyticsProps {
+  startTime: string;
+  endTime: string;
+  linkId?: string;
+  onPathClick?: (path: string) => void;
+}
+
+interface PathData {
+  path: string;
+  count: number;
+  percentage: number;
+}
+
+const PathAnalytics: React.FC<PathAnalyticsProps> = ({ startTime, endTime, linkId, onPathClick }) => {
+  const [loading, setLoading] = useState(true);
+  const [pathData, setPathData] = useState<PathData[]>([]);
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (!linkId) {
+      setLoading(false);
+      return;
+    }
+    
+    const fetchPathData = async () => {
+      try {
+        const params = new URLSearchParams({
+          startTime,
+          endTime,
+          linkId
+        });
+        
+        const response = await fetch(`/api/events/path-analytics?${params.toString()}`);
+        
+        if (!response.ok) {
+          throw new Error('Failed to fetch path analytics data');
+        }
+        
+        const result = await response.json();
+        
+        if (result.success && result.data) {
+          // 自定义处理路径数据，根据是否有子路径来分组
+          const rawData = result.data;
+          const pathMap = new Map<string, number>();
+          let totalClicks = 0;
+          
+          rawData.forEach((item: PathData) => {
+            const urlPath = item.path.split('?')[0];
+            totalClicks += item.count;
+            
+            // 解析路径，检查是否有子路径
+            const pathParts = urlPath.split('/').filter(Boolean);
+            
+            // 基础路径（例如/5seaii）或者带有查询参数但没有子路径的路径视为同一个路径
+            // 子路径（例如/5seaii/bbbbb）单独统计
+            const groupKey = pathParts.length > 1 ? urlPath : `/${pathParts[0]}`;
+            
+            const currentCount = pathMap.get(groupKey) || 0;
+            pathMap.set(groupKey, currentCount + item.count);
+          });
+          
+          // 转换回数组并排序
+          const groupedPathData = Array.from(pathMap.entries())
+            .map(([path, count]) => ({
+              path,
+              count,
+              percentage: totalClicks > 0 ? count / totalClicks : 0,
+            }))
+            .sort((a, b) => b.count - a.count);
+          
+          setPathData(groupedPathData);
+        } else {
+          setError(result.error || 'Failed to load path analytics');
+        }
+      } catch (err) {
+        setError(err instanceof Error ? err.message : 'An error occurred');
+      } finally {
+        setLoading(false);
+      }
+    };
+    
+    fetchPathData();
+  }, [startTime, endTime, linkId]);
+  
+  const handlePathClick = (path: string, e: React.MouseEvent) => {
+    e.preventDefault();
+    console.log('====== PATH CLICK DEBUG ======');
+    console.log('Path value:', path);
+    console.log('Path type:', typeof path);
+    console.log('Path length:', path.length);
+    console.log('Path chars:', Array.from(path).map(c => c.charCodeAt(0)));
+    console.log('==============================');
+    if (onPathClick) {
+      onPathClick(path);
+    }
+  };
+  
+  if (loading) {
+    return <div className="py-8 flex justify-center">
+      <div className="animate-spin rounded-full h-8 w-8 border-t-2 border-b-2 border-blue-500" />
+    </div>;
+  }
+  
+  if (error) {
+    return <div className="py-4 text-red-500">{error}</div>;
+  }
+  
+  if (!linkId) {
+    return <div className="py-4 text-gray-500">Select a specific link to view path analytics.</div>;
+  }
+  
+  if (pathData.length === 0) {
+    return <div className="py-4 text-gray-500">No path data available for this link.</div>;
+  }
+  
+  return (
+    <div>
+      <div className="text-sm text-gray-500 mb-4">
+        Note: Paths are grouped by subpath. URLs with different query parameters but the same base path (without subpath) are grouped together.
+      </div>
+      <div className="overflow-x-auto">
+        <table className="min-w-full divide-y divide-gray-200">
+          <thead>
+            <tr>
+              <th className="px-6 py-3 bg-gray-50 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Path</th>
+              <th className="px-6 py-3 bg-gray-50 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">Clicks</th>
+              <th className="px-6 py-3 bg-gray-50 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">Percentage</th>
+            </tr>
+          </thead>
+          <tbody className="bg-white divide-y divide-gray-200">
+            {pathData.map((item, index) => (
+              <tr key={index} className={index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}>
+                <td className="px-6 py-4 whitespace-nowrap text-sm font-medium text-gray-900">
+                  <a 
+                    href="#" 
+                    className="hover:text-blue-600 hover:underline cursor-pointer"
+                    onClick={(e) => handlePathClick(item.path, e)}
+                  >
+                    {item.path}
+                  </a>
+                </td>
+                <td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500 text-right">{item.count}</td>
+                <td className="px-6 py-4 whitespace-nowrap text-right">
+                  <div className="flex items-center justify-end">
+                    <span className="text-sm text-gray-500 mr-2">{(item.percentage * 100).toFixed(1)}%</span>
+                    <div className="w-32 bg-gray-200 rounded-full h-2.5">
+                      <div className="bg-blue-600 h-2.5 rounded-full" style={{ width: `${item.percentage * 100}%` }}></div>
+                    </div>
+                  </div>
+                </td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </div>
+    </div>
+  );
+};
+
+export default PathAnalytics; 

app/components/analytics/UtmAnalytics.tsx

@@ -18,9 +18,10 @@ interface UtmAnalyticsProps {
   teamIds?: string[];
   projectIds?: string[];
   tagIds?: string[];
+  subpath?: string;
 }
 
-export default function UtmAnalytics({ startTime, endTime, linkId, teamIds, projectIds, tagIds }: UtmAnalyticsProps) {
+export default function UtmAnalytics({ startTime, endTime, linkId, teamIds, projectIds, tagIds, subpath }: UtmAnalyticsProps) {
   const [activeTab, setActiveTab] = useState<string>('source');
   const [utmData, setUtmData] = useState<UtmData[]>([]);
   const [isLoading, setIsLoading] = useState<boolean>(false);
@@ -38,6 +39,7 @@ export default function UtmAnalytics({ startTime, endTime, linkId, teamIds, proj
         if (startTime) params.append('startTime', startTime);
         if (endTime) params.append('endTime', endTime);
         if (linkId) params.append('linkId', linkId);
+        if (subpath) params.append('subpath', subpath);
         params.append('utmType', activeTab);
         
         // 添加团队ID参数
@@ -78,7 +80,7 @@ export default function UtmAnalytics({ startTime, endTime, linkId, teamIds, proj
     };
     
     fetchUtmData();
-  }, [activeTab, startTime, endTime, linkId, teamIds, projectIds, tagIds]);
+  }, [activeTab, startTime, endTime, linkId, teamIds, projectIds, tagIds, subpath]);
   
   // 安全地格式化数字
   const formatNumber = (value: number | undefined | null): string => {

app/components/charts/TimeSeriesChart.tsx

@@ -137,6 +137,11 @@ export default function TimeSeriesChart({ data }: TimeSeriesChartProps) {
                   return date.toLocaleDateString();
                 }
                 return '';
+              },
+              label: (context) => {
+                const label = context.dataset.label || '';
+                const value = context.parsed.y;
+                return `${label}: ${Math.round(value)}`;
               }
             }
           }
@@ -160,9 +165,9 @@ export default function TimeSeriesChart({ data }: TimeSeriesChartProps) {
               callback: (value: number) => {
                 if (!value && value !== 0) return '';
                 if (value >= 1000) {
-                  return `${(value / 1000).toFixed(1)}k`;
+                  return `${Math.round(value / 1000)}k`;
                 }
-                return value;
+                return Math.round(value);
               }
             }
           }

app/components/layout/Header.tsx

@@ -44,6 +44,11 @@ export default function Header() {
                     Short Links
                   </Link>
                 </li>
+                <li>
+                  <Link href="/create-shorturl" className="text-sm text-gray-700 hover:text-blue-500">
+                    Create Short URL
+                  </Link>
+                </li>
               </ul>
             </nav>
           )}

app/components/layout/Navbar.tsx

@@ -1,65 +0,0 @@
-'use client';
-
-import Link from 'next/link';
-
-export default function Navbar() {
-  return (
-    <header className="w-full py-4 border-b border-card-border bg-background">
-      <div className="container flex items-center justify-between px-4 mx-auto">
-        <div className="flex items-center space-x-4">
-          <Link href="/" className="flex items-center space-x-2">
-            <svg
-              className="w-6 h-6 text-accent-blue"
-              xmlns="http://www.w3.org/2000/svg"
-              viewBox="0 0 24 24"
-              fill="none"
-              stroke="currentColor"
-              strokeWidth="2"
-              strokeLinecap="round"
-              strokeLinejoin="round"
-            >
-              <path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
-              <path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
-            </svg>
-            <span className="text-xl font-bold text-foreground">ShortURL</span>
-          </Link>
-          <nav className="hidden space-x-4 md:flex">
-            <Link
-              href="/links"
-              className="text-sm text-foreground hover:text-accent-blue transition-colors"
-            >
-              Links
-            </Link>
-            <Link
-              href="/analytics"
-              className="text-sm text-foreground hover:text-accent-blue transition-colors"
-            >
-              Analytics
-            </Link>
-          </nav>
-        </div>
-        <div className="flex items-center space-x-3">
-          <button className="p-2 text-sm text-foreground rounded-md gradient-border">
-            Upgrade
-          </button>
-          <button className="p-2 text-sm text-foreground hover:text-accent-blue">
-            <svg
-              className="w-5 h-5"
-              xmlns="http://www.w3.org/2000/svg"
-              viewBox="0 0 24 24"
-              fill="none"
-              stroke="currentColor"
-              strokeWidth="2"
-              strokeLinecap="round"
-              strokeLinejoin="round"
-            >
-              <circle cx="12" cy="12" r="10"></circle>
-              <circle cx="12" cy="10" r="3"></circle>
-              <path d="M7 20.662V19a2 2 0 0 1 2-2h6a2 2 0 0 1 2 2v1.662"></path>
-            </svg>
-          </button>
-        </div>
-      </div>
-    </header>
-  );
-} 

app/components/ui/TeamSelector.tsx

@@ -7,6 +7,7 @@ import { getSupabaseClient } from '../../utils/supabase';
 import { AuthChangeEvent, Session } from '@supabase/supabase-js';
 import { Loader2, X, Check } from 'lucide-react';
 import { cn } from '@/lib/utils';
+import { limqRequest } from '@/lib/api';
 
 type Team = Database['limq']['Tables']['teams']['Row'];
 
@@ -69,6 +70,14 @@ export function TeamSelector({
       
       try {
         const supabase = getSupabaseClient();
+
+        // 尝试创建默认团队和项目（如果用户还没有）
+        try {
+          const response = await limqRequest('team/create-default', 'POST');
+          console.log('Default team creation response:', response);
+        } catch (teamError) {
+          console.error('Error creating default team:', teamError);
+        }
         
         const { data: memberships, error: membershipError } = await supabase
           .from('team_membership')

app/create-shorturl/page.tsx

@@ -0,0 +1,403 @@
+'use client';
+
+import { useState, useEffect } from 'react';
+import { useRouter } from 'next/navigation';
+import { useAuth } from '@/lib/auth';
+import { limqRequest } from '@/lib/api';
+import { TeamSelector } from '@/app/components/ui/TeamSelector';
+import { ProjectSelector } from '@/app/components/ui/ProjectSelector';
+import ClientRouteGuard from '@/app/components/ClientRouteGuard';
+
+interface ShortUrlData {
+  originalUrl: string;
+  customSlug?: string;
+  title: string;
+  description?: string;
+  tags?: string[];
+  teamId: string;
+  projectId: string;
+  domain: string;
+}
+
+export default function CreateShortUrlPage() {
+  return (
+    <ClientRouteGuard>
+      <CreateShortUrlForm />
+    </ClientRouteGuard>
+  );
+}
+
+function CreateShortUrlForm() {
+  const router = useRouter();
+  const { user } = useAuth();
+  
+  const [formData, setFormData] = useState<ShortUrlData>({
+    originalUrl: '',
+    customSlug: '',
+    title: '',
+    description: '',
+    tags: [],
+    teamId: '',
+    projectId: '',
+    domain: 'googleads.link',
+  });
+  
+  const [tagInput, setTagInput] = useState('');
+  const [isSubmitting, setIsSubmitting] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [success, setSuccess] = useState(false);
+  
+  // Use useEffect to add user information to form data on load
+  useEffect(() => {
+    if (user) {
+      console.log('Current user:', user.email);
+      // Can add user-related data to the form here
+    }
+  }, [user]);
+  
+  const handleChange = (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
+    const { name, value } = e.target;
+    setFormData(prev => ({
+      ...prev,
+      [name]: value
+    }));
+  };
+  
+  const handleTagKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
+    if (e.key === 'Enter' && tagInput.trim()) {
+      e.preventDefault();
+      addTag();
+    }
+  };
+  
+  const addTag = () => {
+    if (tagInput.trim() && !formData.tags?.includes(tagInput.trim())) {
+      setFormData(prev => ({
+        ...prev,
+        tags: [...(prev.tags || []), tagInput.trim()]
+      }));
+      setTagInput('');
+    }
+  };
+  
+  const removeTag = (tagToRemove: string) => {
+    setFormData(prev => ({
+      ...prev,
+      tags: prev.tags?.filter(tag => tag !== tagToRemove)
+    }));
+  };
+  
+  const handleSubmit = async (e: React.FormEvent) => {
+    e.preventDefault();
+    setIsSubmitting(true);
+    setError(null);
+    
+    try {
+      // Validate required fields
+      if (!formData.originalUrl) {
+        throw new Error('Original URL is required');
+      }
+      
+      if (!formData.title) {
+        throw new Error('Title is required');
+      }
+      
+      if (!formData.teamId) {
+        throw new Error('Team is required');
+      }
+      
+      if (!formData.projectId) {
+        throw new Error('Project is required');
+      }
+      
+      if (!formData.domain) {
+        throw new Error('Domain is required');
+      }
+      
+      // Construct request data according to API requirements
+      const requestData = {
+        type: "shorturl",
+        attributes: {
+          // Can add any additional attributes, but attributes cannot be empty
+          icon: ""
+        },
+        shortUrl: {
+          url: formData.originalUrl,
+          slug: formData.customSlug || undefined,
+          title: formData.title,
+          name: formData.title,
+          description: formData.description || "",
+          domain: formData.domain
+        },
+        teamId: formData.teamId,
+        projectId: formData.projectId,
+        tagIds: formData.tags && formData.tags.length > 0 ? formData.tags : undefined
+      };
+      
+      // Call API to create shorturl resource
+      const response = await limqRequest('resource/shorturl', 'POST', requestData as unknown as Record<string, unknown>);
+      
+      console.log('Created successfully:', response);
+      setSuccess(true);
+      
+      // Redirect to links list page after 2 seconds
+      setTimeout(() => {
+        router.push('/links');
+      }, 2000);
+      
+    } catch (err) {
+      console.error('Failed to create short URL:', err);
+      setError(err instanceof Error ? err.message : 'Failed to create short URL, please try again later');
+    } finally {
+      setIsSubmitting(false);
+    }
+  };
+  
+  return (
+    <div className="container mx-auto px-4 py-8 max-w-3xl">
+      <div className="bg-white rounded-lg shadow-md overflow-hidden">
+        <div className="border-b border-gray-200 bg-blue-50 px-6 py-4">
+          <h1 className="text-xl font-medium text-gray-900">Create Short URL</h1>
+          <p className="mt-1 text-sm text-gray-600">
+            Create a new short URL resource for tracking and analytics
+          </p>
+        </div>
+        
+        {error && (
+          <div className="bg-red-50 border-l-4 border-red-500 p-4 m-6">
+            <div className="flex">
+              <div className="flex-shrink-0">
+                <svg className="h-5 w-5 text-red-500" fill="currentColor" viewBox="0 0 20 20">
+                  <path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
+                </svg>
+              </div>
+              <div className="ml-3">
+                <p className="text-sm text-red-700">{error}</p>
+              </div>
+            </div>
+          </div>
+        )}
+        
+        {success && (
+          <div className="bg-green-50 border-l-4 border-green-500 p-4 m-6">
+            <div className="flex">
+              <div className="flex-shrink-0">
+                <svg className="h-5 w-5 text-green-500" fill="currentColor" viewBox="0 0 20 20">
+                  <path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clipRule="evenodd" />
+                </svg>
+              </div>
+              <div className="ml-3">
+                <p className="text-sm text-green-700">
+                  Short URL created successfully! Redirecting...
+                </p>
+              </div>
+            </div>
+          </div>
+        )}
+        
+        <form onSubmit={handleSubmit} className="p-6 space-y-6">
+          {/* Title */}
+          <div>
+            <label htmlFor="title" className="block text-sm font-medium text-gray-700">
+              Title <span className="text-red-500">*</span>
+            </label>
+            <input
+              type="text"
+              id="title"
+              name="title"
+              value={formData.title}
+              onChange={handleChange}
+              placeholder="e.g., Product Launch Campaign"
+              className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
+              required
+            />
+          </div>
+          
+          {/* Original URL */}
+          <div>
+            <label htmlFor="originalUrl" className="block text-sm font-medium text-gray-700">
+              Original URL <span className="text-red-500">*</span>
+            </label>
+            <input
+              type="url"
+              id="originalUrl"
+              name="originalUrl"
+              value={formData.originalUrl}
+              onChange={handleChange}
+              placeholder="https://example.com/your-long-url"
+              className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
+              required
+            />
+          </div>
+          
+          {/* Custom Short Link */}
+          <div>
+            <label htmlFor="customSlug" className="block text-sm font-medium text-gray-700">
+              Custom Short Link <span className="text-gray-500">(Optional)</span>
+            </label>
+            <div className="flex mt-1 rounded-md shadow-sm">
+              <span className="inline-flex items-center px-3 py-2 text-sm text-gray-500 border border-r-0 border-gray-300 rounded-l-md bg-gray-50">
+                {formData.domain}/
+              </span>
+              <input
+                type="text"
+                id="customSlug"
+                name="customSlug"
+                value={formData.customSlug}
+                onChange={handleChange}
+                placeholder="custom-slug"
+                className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-none rounded-r-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
+              />
+            </div>
+            <p className="mt-1 text-xs text-gray-500">
+              Leave blank to generate a random short link
+            </p>
+          </div>
+          
+          {/* Domain */}
+          <div>
+            <label htmlFor="domain" className="block text-sm font-medium text-gray-700">
+              Domain <span className="text-red-500">*</span>
+            </label>
+            <input
+              type="text"
+              id="domain"
+              name="domain"
+              value={formData.domain}
+              onChange={handleChange}
+              placeholder="e.g., googleads.link"
+              className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
+              required
+            />
+          </div>
+          
+          {/* Team Selection */}
+          <div>
+            <label htmlFor="teamId" className="block text-sm font-medium text-gray-700">
+              Team <span className="text-red-500">*</span>
+            </label>
+            <div className="mt-1">
+              <TeamSelector 
+                value={formData.teamId}
+                onChange={(teamId) => {
+                  setFormData(prev => ({
+                    ...prev,
+                    teamId: teamId as string,
+                    // Clear selected project when team changes
+                    projectId: ''
+                  }));
+                }}
+              />
+            </div>
+          </div>
+
+          {/* Project Selection */}
+          <div>
+            <label htmlFor="projectId" className="block text-sm font-medium text-gray-700">
+              Project <span className="text-red-500">*</span>
+            </label>
+            <div className="mt-1">
+              <ProjectSelector 
+                teamId={formData.teamId}
+                value={formData.projectId}
+                onChange={(projectId) => {
+                  setFormData(prev => ({
+                    ...prev,
+                    projectId: projectId as string
+                  }));
+                }}
+              />
+            </div>
+          </div>
+          
+          {/* Description */}
+          <div>
+            <label htmlFor="description" className="block text-sm font-medium text-gray-700">
+              Description <span className="text-gray-500">(Optional)</span>
+            </label>
+            <textarea
+              id="description"
+              name="description"
+              value={formData.description}
+              onChange={handleChange}
+              rows={3}
+              placeholder="A brief description of this link"
+              className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
+            />
+          </div>
+          
+          {/* Tags */}
+          <div>
+            <label htmlFor="tagInput" className="block text-sm font-medium text-gray-700">
+              Tags <span className="text-gray-500">(Optional)</span>
+            </label>
+            <div className="flex mt-1 rounded-md shadow-sm">
+              <input
+                type="text"
+                id="tagInput"
+                value={tagInput}
+                onChange={(e) => setTagInput(e.target.value)}
+                onKeyDown={handleTagKeyDown}
+                placeholder="Add a tag and press Enter"
+                className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-l-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
+              />
+              <button
+                type="button"
+                onClick={addTag}
+                className="inline-flex items-center px-3 py-2 text-sm font-medium text-white border border-transparent rounded-r-md shadow-sm bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
+              >
+                Add
+              </button>
+            </div>
+            
+            {formData.tags && formData.tags.length > 0 && (
+              <div className="flex flex-wrap gap-2 mt-2">
+                {formData.tags.map(tag => (
+                  <span key={tag} className="inline-flex items-center px-2 py-0.5 text-xs font-medium bg-blue-100 rounded-full text-blue-800">
+                    {tag}
+                    <button
+                      type="button"
+                      onClick={() => removeTag(tag)}
+                      className="flex-shrink-0 ml-1 text-blue-500 rounded-full hover:text-blue-700 focus:outline-none"
+                    >
+                      <span className="sr-only">Remove tag {tag}</span>
+                      <svg className="w-3 h-3" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
+                        <path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
+                      </svg>
+                    </button>
+                  </span>
+                ))}
+              </div>
+            )}
+          </div>
+          
+          {/* Submit Button */}
+          <div className="flex justify-end pt-4 border-t border-gray-200">
+            <button
+              type="button"
+              onClick={() => router.back()}
+              className="px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 mr-3"
+            >
+              Cancel
+            </button>
+            <button
+              type="submit"
+              disabled={isSubmitting}
+              className="inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
+            >
+              {isSubmitting ? (
+                <>
+                  <svg className="w-5 h-5 mr-2 -ml-1 animate-spin" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
+                    <circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
+                    <path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+                  </svg>
+                  Processing...
+                </>
+              ) : 'Create Short URL'}
+            </button>
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+} 

app/links/page.tsx

@@ -7,6 +7,7 @@ import { Loader2, ExternalLink, Search } from 'lucide-react';
 import { TeamSelector } from '@/app/components/ui/TeamSelector';
 import { useRouter } from 'next/navigation';
 import { useShortUrlStore, ShortUrlData } from '@/app/utils/store';
+import ClientRouteGuard from '@/app/components/ClientRouteGuard';
 
 // Define attribute type to avoid using 'any'
 interface LinkAttributes {
@@ -49,6 +50,7 @@ interface ShortLink {
   expires_at?: string | null;
   click_count?: number;
   unique_visitors?: number;
+  domain?: string;
 }
 
 // Define ClickHouse shorturl type
@@ -77,6 +79,7 @@ interface ClickHouseShortUrl {
   expires_at: string | null;
   click_count: number;
   unique_visitors: number;
+  domain?: string; // 添加domain字段
   link_attributes?: string; // Optional JSON string containing link-specific attributes
 }
 
@@ -100,6 +103,14 @@ const convertClickHouseToShortLink = (data: Record<string, unknown>): ShortLink
 };
 
 export default function LinksPage() {
+  return (
+    <ClientRouteGuard>
+      <LinksPageContent />
+    </ClientRouteGuard>
+  );
+}
+
+function LinksPageContent() {
   const [loading, setLoading] = useState(true);
   const [error, setError] = useState<string | null>(null);
   const [links, setLinks] = useState<ShortLink[]>([]);
@@ -115,30 +126,77 @@ export default function LinksPage() {
   // 使用 Zustand store
   const { setSelectedShortUrl } = useShortUrlStore();
 
-  // 处理链接记录点击
-  const handleLinkClick = (shortUrl: string, link: ShortLink, metadata: any) => {
-    // 编码 shortUrl 以确保 URL 安全
-    const encodedShortUrl = encodeURIComponent(shortUrl);
+  // 处理点击链接行
+  const handleRowClick = (link: any) => {
+    // 解析 attributes 字符串为对象
+    let attributes: Record<string, any> = {};
+    try {
+      if (link.attributes && typeof link.attributes === 'string') {
+        attributes = JSON.parse(link.attributes || '{}');
+      }
+    } catch (e) {
+      console.error('Error parsing link attributes:', e);
+    }
     
-    // 创建完整的 ShortUrlData 对象
-    const shortUrlData: ShortUrlData = {
+    // 解析 teams 字符串为数组
+    let teams: any[] = [];
+    try {
+      if (link.teams && typeof link.teams === 'string') {
+        teams = JSON.parse(link.teams || '[]');
+      }
+    } catch (e) {
+      console.error('Error parsing teams:', e);
+    }
+    
+    // 解析 projects 字符串为数组
+    let projects: any[] = [];
+    try {
+      if (link.projects && typeof link.projects === 'string') {
+        projects = JSON.parse(link.projects || '[]');
+      }
+    } catch (e) {
+      console.error('Error parsing projects:', e);
+    }
+    
+    // 解析 tags 字符串为数组
+    let tags: string[] = [];
+    try {
+      if (link.tags && typeof link.tags === 'string') {
+        const parsedTags = JSON.parse(link.tags);
+        if (Array.isArray(parsedTags)) {
+          tags = parsedTags.map((tag: { tag_name?: string }) => tag.tag_name || '');
+        }
+      }
+    } catch (e) {
+      console.error('Error parsing tags:', e);
+    }
+
+    // 确保 shortUrl 存在
+    const shortUrlValue = attributes.shortUrl || '';
+    
+    // 提取用于显示的字段
+    const shortUrlData = {
       id: link.id,
-      slug: metadata.slug,
-      originalUrl: metadata.originalUrl,
-      title: metadata.title,
-      shortUrl: shortUrl,
-      teams: metadata.teamNames,
-      tags: metadata.tagNames,
-      projects: metadata.projectNames,
-      createdAt: metadata.createdAt,
-      domain: metadata.domain
+      externalId: link.external_id, // 明确添加 externalId 字段
+      slug: link.slug,
+      originalUrl: link.original_url,
+      title: link.title,
+      shortUrl: shortUrlValue,
+      teams: teams,
+      projects: projects,
+      tags: tags,
+      createdAt: link.created_at,
+      domain: link.domain || (shortUrlValue ? new URL(shortUrlValue).hostname : '')
     };
     
-    // 使用 Zustand store 保存数据
+    // 打印完整数据，确保 externalId 被包含
+    console.log('Setting shortURL data in store with externalId:', link.external_id);
+    
+    // 将数据保存到 Zustand store
     setSelectedShortUrl(shortUrlData);
     
-    // 导航到 analytics 页面并带上参数
-    router.push(`/analytics?shorturl=${encodedShortUrl}`);
+    // 导航到分析页面，并在 URL 中包含 shortUrl 参数
+    router.push(`/analytics?shorturl=${encodeURIComponent(shortUrlValue)}`);
   };
 
   // Extract link metadata from attributes
@@ -150,19 +208,26 @@ export default function LinksPage() {
         : link.attributes || {};
       
       // Parse attributes to get domain if available
-      let domain = 'shorturl.example.com';
+      let domain = '';
       try {
-        // Extract domain from shortUrl in attributes if available
-        const attributesObj = typeof link.attributes === 'string'
-          ? JSON.parse(link.attributes)
-          : link.attributes || {};
-        
-        if (attributesObj.shortUrl) {
-          try {
-            const urlObj = new URL(attributesObj.shortUrl);
-            domain = urlObj.hostname;
-          } catch (e) {
-            console.error('Error parsing shortUrl:', e);
+        // 首先尝试使用link.domain字段
+        if (link.domain) {
+          domain = link.domain;
+        }
+        // 如果没有domain字段，从shortUrl中提取
+        else {
+          // Extract domain from shortUrl in attributes if available
+          const attributesObj = typeof link.attributes === 'string'
+            ? JSON.parse(link.attributes)
+            : link.attributes || {};
+          
+          if (attributesObj.shortUrl) {
+            try {
+              const urlObj = new URL(attributesObj.shortUrl);
+              domain = urlObj.hostname;
+            } catch (e) {
+              console.error('Error parsing shortUrl:', e);
+            }
           }
         }
       } catch (e) {
@@ -423,7 +488,7 @@ export default function LinksPage() {
               const shortUrl = `https://${metadata.domain}/${metadata.slug}`;
               
               return (
-                <tr key={link.id} className="hover:bg-gray-50 cursor-pointer" onClick={() => handleLinkClick(shortUrl, link, metadata)}>
+                <tr key={link.id} className="hover:bg-gray-50 cursor-pointer" onClick={() => handleRowClick(link)}>
                   <td className="px-6 py-4">
                     <div className="flex flex-col space-y-1">
                       <span className="font-medium text-gray-900">{metadata.title}</span>

app/login/page.tsx

@@ -1,13 +1,27 @@
 'use client';
 
-import { useState, useEffect } from 'react';
-import { useRouter } from 'next/navigation';
+import { useState, useEffect, Suspense } from 'react';
+import { useRouter, useSearchParams } from 'next/navigation';
 import Link from 'next/link';
 import { useAuth } from '@/lib/auth';
 
+// Separate component for message handling to isolate useSearchParams
+function MessageHandler({ setMessage }: { setMessage: (message: { type: string, content: string }) => void }) {
+  const searchParams = useSearchParams();
+  
+  useEffect(() => {
+    const messageParam = searchParams.get('message');
+    if (messageParam) {
+      setMessage({ type: 'info', content: messageParam });
+    }
+  }, [searchParams, setMessage]);
+  
+  return null;
+}
+
 export default function LoginPage() {
   const router = useRouter();
-  const { signIn, signInWithGitHub, signInWithGoogle, user } = useAuth();
+  const { signIn, signInWithGoogle, user } = useAuth();
   
   const [email, setEmail] = useState('');
   const [password, setPassword] = useState('');
@@ -53,28 +67,6 @@ export default function LoginPage() {
     }
   };
   
-  const handleGitHubSignIn = async () => {
-    try {
-      setIsLoading(true);
-      setMessage({ type: '', content: '' });
-      
-      const { error } = await signInWithGitHub();
-      
-      if (error) {
-        throw new Error(error.message);
-      }
-      
-      // 登录成功后，会通过 useEffect 重定向
-    } catch (error) {
-      console.error('GitHub login error:', error);
-      setMessage({ 
-        type: 'error', 
-        content: error instanceof Error ? error.message : 'Failed to sign in with GitHub' 
-      });
-      setIsLoading(false);
-    }
-  };
-  
   const handleGoogleSignIn = async () => {
     try {
       setIsLoading(true);
@@ -86,7 +78,7 @@ export default function LoginPage() {
         throw new Error(error.message);
       }
       
-      // 登录成功后，会通过 useEffect 重定向
+      // Google OAuth will handle the redirect
     } catch (error) {
       console.error('Google login error:', error);
       setMessage({ 
@@ -99,6 +91,11 @@ export default function LoginPage() {
   
   return (
     <div className="flex items-center justify-center min-h-screen bg-gray-100">
+      {/* Wrap the component using useSearchParams in Suspense */}
+      <Suspense fallback={null}>
+        <MessageHandler setMessage={setMessage} />
+      </Suspense>
+      
       <div className="w-full max-w-md p-8 space-y-8 bg-white rounded-lg shadow-md">
         <div className="text-center">
           <h1 className="text-2xl font-bold text-gray-900">Login</h1>
@@ -112,16 +109,45 @@ export default function LoginPage() {
         
         {/* Message display */}
         {message.content && (
-          <div className={`p-4 mb-4 text-sm ${
+          <div className={`p-4 mb-4 text-sm rounded-lg ${
             message.type === 'error' 
-              ? 'text-red-700 bg-red-100 rounded-lg'
-              : 'text-blue-700 bg-blue-100 rounded-lg'
+              ? 'text-red-700 bg-red-100 border border-red-200'
+              : 'text-blue-700 bg-blue-50 border border-blue-200'
           }`}>
+            {message.type === 'error' ? (
+              <span className="font-medium">Error: </span>
+            ) : (
+              <span className="font-medium">Notice: </span>
+            )}
             {message.content}
           </div>
         )}
         
-        <form onSubmit={handleEmailSignIn} className="mt-8 space-y-6">
+        {/* Google Sign In Button */}
+        <button
+          type="button"
+          onClick={handleGoogleSignIn}
+          disabled={isLoading}
+          className="w-full flex items-center justify-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
+        >
+          <svg className="h-5 w-5 mr-2" viewBox="0 0 24 24" width="24" height="24">
+            <g transform="matrix(1, 0, 0, 1, 27.009001, -39.238998)">
+              <path fill="#4285F4" d="M -3.264 51.509 C -3.264 50.719 -3.334 49.969 -3.454 49.239 L -14.754 49.239 L -14.754 53.749 L -8.284 53.749 C -8.574 55.229 -9.424 56.479 -10.684 57.329 L -10.684 60.329 L -6.824 60.329 C -4.564 58.239 -3.264 55.159 -3.264 51.509 Z"/>
+              <path fill="#34A853" d="M -14.754 63.239 C -11.514 63.239 -8.804 62.159 -6.824 60.329 L -10.684 57.329 C -11.764 58.049 -13.134 58.489 -14.754 58.489 C -17.884 58.489 -20.534 56.379 -21.484 53.529 L -25.464 53.529 L -25.464 56.619 C -23.494 60.539 -19.444 63.239 -14.754 63.239 Z"/>
+              <path fill="#FBBC05" d="M -21.484 53.529 C -21.734 52.809 -21.864 52.039 -21.864 51.239 C -21.864 50.439 -21.724 49.669 -21.484 48.949 L -21.484 45.859 L -25.464 45.859 C -26.284 47.479 -26.754 49.299 -26.754 51.239 C -26.754 53.179 -26.284 54.999 -25.464 56.619 L -21.484 53.529 Z"/>
+              <path fill="#EA4335" d="M -14.754 43.989 C -12.984 43.989 -11.404 44.599 -10.154 45.789 L -6.734 42.369 C -8.804 40.429 -11.514 39.239 -14.754 39.239 C -19.444 39.239 -23.494 41.939 -25.464 45.859 L -21.484 48.949 C -20.534 46.099 -17.884 43.989 -14.754 43.989 Z"/>
+            </g>
+          </svg>
+          {isLoading ? 'Signing in...' : 'Sign in with Google'}
+        </button>
+        
+        <div className="mt-6 flex items-center justify-center">
+          <div className="border-t border-gray-300 flex-grow mr-3"></div>
+          <span className="text-sm text-gray-500">or</span>
+          <div className="border-t border-gray-300 flex-grow ml-3"></div>
+        </div>
+        
+        <form onSubmit={handleEmailSignIn} className="mt-6 space-y-6">
           <div>
             <label htmlFor="email" className="block text-sm font-medium text-gray-700">
               Email address
@@ -164,52 +190,12 @@ export default function LoginPage() {
               disabled={isLoading}
               className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
             >
-              {isLoading ? 'Signing in...' : 'Sign in'}
+              {isLoading ? 'Signing in...' : 'Sign in with Email'}
             </button>
           </div>
         </form>
         
-        <div className="mt-6">
-          <div className="relative">
-            <div className="absolute inset-0 flex items-center">
-              <div className="w-full border-t border-gray-300"></div>
-            </div>
-            <div className="relative flex justify-center text-sm">
-              <span className="px-2 bg-white text-gray-500">Or</span>
-            </div>
-          </div>
-          
-          <div className="mt-6 grid grid-cols-2 gap-3">
-            <button
-              type="button"
-              onClick={handleGitHubSignIn}
-              disabled={isLoading}
-              className="flex justify-center items-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
-            >
-              <svg className="h-5 w-5 mr-2" fill="currentColor" viewBox="0 0 24 24">
-                <path fillRule="evenodd" d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z" clipRule="evenodd" />
-              </svg>
-              GitHub
-            </button>
-            
-            <button
-              type="button"
-              onClick={handleGoogleSignIn}
-              disabled={isLoading}
-              className="flex justify-center items-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
-            >
-              <svg className="h-5 w-5 mr-2" viewBox="0 0 24 24">
-                <path d="M12.545 12.151L12.545 12.151L12.545 12.151C12.545 9.85553 14.0905 7.98375 16.088 7.98375C17.0865 7.98375 17.938 8.43025 18.5592 9.0514L21.3404 6.27019C19.7172 4.75612 18.0026 4 16.088 4C12.5405 4 9.5 6.67528 9.5 10.2505C9.5 12.0582 10.1533 13.4581 10.8634 14.4685C12.1453 16.3618 14.4737 18.501 16.088 18.501C19.9265 18.501 22 16.0057 22 12.4071C22 11.4245 21.9318 10.9113 21.7953 10.2505H16.088V12.151H12.545Z" fill="#4285F4" />
-                <path d="M5.90607 10.2197C5.40834 11.1993 5.12343 12.2959 5.12343 13.4564C5.12343 14.6646 5.41958 15.782 5.92853 16.7831L5.92786 16.7818C6.91998 18.6136 8.81431 19.8018 11.0008 19.8018C12.5581 19.8018 13.8262 19.318 14.7997 18.5825L14.7976 18.5845C15.6806 17.9139 16.401 16.9218 16.6662 15.7257L16.6657 15.7276C16.7331 15.3933 16.7688 15.0493 16.7688 14.6895H11.0008C10.3375 14.6895 9.80078 14.1523 9.80078 13.4882V10.2197H5.90607Z" fill="#34A853" />
-                <path d="M5.12207 6.25024C4 7.86024 3.33789 9.81535 3.33789 11.9339C3.33789 12.9995 3.55215 14.0269 3.94853 14.9805L5.90673 10.2197H9.80143V6.25024H5.12207Z" fill="#FBBC05" />
-                <path d="M11.001 3.57764C12.4571 3.57764 13.778 4.11181 14.8023 5.06959L14.8028 5.0692L17.2711 2.60092L17.271 2.60082C15.5041 0.97625 13.3649 0 11.001 0C8.81453 0 6.91994 1.18824 5.92853 3.02125L9.80224 6.25031V6.25031H11.001V3.57764Z" fill="#EA4335" />
-              </svg>
-              Google
-            </button>
-          </div>
-        </div>
-        
-        <p className="text-sm text-gray-600">
+        <p className="text-sm mt-6 text-gray-600">
           Don&apos;t have an account?{' '}
           <Link href="/register" className="font-medium text-blue-600 hover:text-blue-500">
             Register

app/register/page.tsx

@@ -12,44 +12,44 @@ export default function RegisterPage() {
   const [isLoading, setIsLoading] = useState(false);
   const { signUp, signInWithGoogle } = useAuth();
 
-  // 处理注册表单提交
+  // Handle registration form submission
   const handleSubmit = async (e: FormEvent) => {
     e.preventDefault();
     setError(null);
     
-    // 验证密码
+    // Validate passwords
     if (password !== confirmPassword) {
-      setError('两次输入的密码不一致');
+      setError('Passwords do not match');
       return;
     }
     
-    // 密码强度验证
+    // Password strength validation
     if (password.length < 6) {
-      setError('密码长度至少为6个字符');
+      setError('Password must be at least 6 characters');
       return;
     }
     
     setIsLoading(true);
     try {
       await signUp(email, password);
-      // 注册成功后会跳转到登录页面，提示用户验证邮箱
+      // After successful registration, redirect to login page with email verification prompt
     } catch (error) {
       console.error('Registration error:', error);
-      setError('注册失败，请稍后再试或使用其他邮箱');
+      setError('Registration failed. Please try again later or use a different email');
     } finally {
       setIsLoading(false);
     }
   };
 
-  // 处理Google注册/登录
+  // Handle Google registration/login
   const handleGoogleSignIn = async () => {
     setError(null);
     try {
       await signInWithGoogle();
-      // 登录流程会重定向到Google，然后回到应用
+      // Login flow will redirect to Google and then back to the application
     } catch (error) {
       console.error('Google sign in error:', error);
-      setError('Google登录失败，请稍后再试');
+      setError('Google login failed. Please try again later');
     }
   };
 
@@ -57,13 +57,13 @@ export default function RegisterPage() {
     <div className="flex items-center justify-center min-h-screen bg-gray-100 dark:bg-gray-900">
       <div className="w-full max-w-md p-8 space-y-8 bg-white dark:bg-gray-800 rounded-lg shadow-md">
         <div className="text-center">
-          <h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">注册</h1>
+          <h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">Register</h1>
           <p className="mt-2 text-sm text-gray-600 dark:text-gray-400">
-            创建您的帐户以访问分析仪表板
+            Create your account to access the analytics dashboard
           </p>
         </div>
 
-        {/* 错误提示 */}
+        {/* Error message */}
         {error && (
           <div className="p-4 mb-4 text-sm text-red-700 bg-red-100 dark:bg-red-900 dark:text-red-200 rounded-lg">
             {error}
@@ -74,7 +74,7 @@ export default function RegisterPage() {
           <div className="space-y-4">
             <div>
               <label htmlFor="email" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
-                邮箱地址
+                Email Address
               </label>
               <input
                 id="email"
@@ -90,7 +90,7 @@ export default function RegisterPage() {
             </div>
             <div>
               <label htmlFor="password" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
-                密码
+                Password
               </label>
               <input
                 id="password"
@@ -106,7 +106,7 @@ export default function RegisterPage() {
             </div>
             <div>
               <label htmlFor="confirmPassword" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
-                确认密码
+                Confirm Password
               </label>
               <input
                 id="confirmPassword"
@@ -128,7 +128,7 @@ export default function RegisterPage() {
               disabled={isLoading}
               className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
             >
-              {isLoading ? '注册中...' : '注册'}
+              {isLoading ? 'Registering...' : 'Register'}
             </button>
           </div>
 
@@ -137,7 +137,7 @@ export default function RegisterPage() {
               <div className="w-full border-t border-gray-300 dark:border-gray-600"></div>
             </div>
             <div className="relative flex justify-center text-sm">
-              <span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">或</span>
+              <span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">or</span>
             </div>
           </div>
 
@@ -173,19 +173,19 @@ export default function RegisterPage() {
                   />
                 </g>
               </svg>
-              使用Google账号注册
+              Sign up with Google
             </button>
           </div>
         </form>
 
         <div className="mt-6 text-center">
           <p className="text-sm text-gray-600 dark:text-gray-400">
-            已有账号？{' '}
+            Already have an account?{' '}
             <Link
               href="/login"
               className="font-medium text-blue-600 hover:text-blue-500 dark:text-blue-400 dark:hover:text-blue-300"
             >
-              登录
+              Log in
             </Link>
           </p>
         </div>

app/utils/store.ts

@@ -1,15 +1,30 @@
 import { create } from 'zustand';
+import { persist } from 'zustand/middleware';
+
+// Define interface for team, project and tag objects
+interface TeamData {
+  team_id: string;
+  team_name: string;
+  [key: string]: unknown;
+}
+
+interface ProjectData {
+  project_id: string;
+  project_name: string;
+  [key: string]: unknown;
+}
 
 // 定义 ShortUrl 数据类型
 export interface ShortUrlData {
   id: string;
+  externalId: string;
   slug: string;
   originalUrl: string;
   title?: string;
   shortUrl: string;
-  teams?: any[];
-  projects?: any[];
-  tags?: any[];
+  teams?: TeamData[];
+  projects?: ProjectData[];
+  tags?: string[];
   createdAt?: string;
   domain?: string;
 }
@@ -21,9 +36,17 @@ interface ShortUrlStore {
   clearSelectedShortUrl: () => void;
 }
 
-// 创建 store
-export const useShortUrlStore = create<ShortUrlStore>((set) => ({
-  selectedShortUrl: null,
-  setSelectedShortUrl: (shortUrl) => set({ selectedShortUrl: shortUrl }),
-  clearSelectedShortUrl: () => set({ selectedShortUrl: null }),
-}));
+// 创建 store 并使用 persist 中间件保存到 localStorage
+export const useShortUrlStore = create<ShortUrlStore>()(
+  persist(
+    (set) => ({
+      selectedShortUrl: null,
+      setSelectedShortUrl: (shortUrl) => set({ selectedShortUrl: shortUrl }),
+      clearSelectedShortUrl: () => set({ selectedShortUrl: null }),
+    }),
+    {
+      name: 'shorturl-storage', // localStorage 中的 key 名称
+      partialize: (state) => ({ selectedShortUrl: state.selectedShortUrl }), // 只持久化 selectedShortUrl
+    }
+  )
+);

lib/analytics.ts

@@ -22,6 +22,7 @@ export interface EventsQueryParams {
   teamIds?: string[];
   projectIds?: string[];
   tagIds?: string[];
+  subpath?: string;
   page?: number;
   pageSize?: number;
   sortBy?: string;
@@ -66,8 +67,11 @@ export async function getEventsSummary(params: {
   teamIds?: string[];
   projectIds?: string[];
   tagIds?: string[];
+  subpath?: string;
 }): Promise<EventsSummary> {
+  console.log('getEventsSummary received params:', params);
   const filter = buildFilter(params);
+  console.log('getEventsSummary built filter:', filter);
   
   // 获取基本统计数据
   const baseQuery = `
@@ -184,6 +188,7 @@ export async function getTimeSeriesData(params: {
   teamIds?: string[];
   projectIds?: string[];
   tagIds?: string[];
+  subpath?: string;
 }): Promise<TimeSeriesData[]> {
   const filter = buildFilter(params);
   
@@ -219,6 +224,7 @@ export async function getGeoAnalytics(params: {
   teamIds?: string[];
   projectIds?: string[];
   tagIds?: string[];
+  subpath?: string;
 }): Promise<GeoData[]> {
   const filter = buildFilter(params);
   
@@ -255,6 +261,7 @@ export async function getDeviceAnalytics(params: {
   teamIds?: string[];
   projectIds?: string[];
   tagIds?: string[];
+  subpath?: string;
 }): Promise<DeviceAnalytics> {
   const filter = buildFilter(params);
   

lib/api.ts

@@ -0,0 +1,50 @@
+import supabase from './supabase';
+
+// Define response type for API
+export interface ApiResponse<T = unknown> {
+  success: boolean;
+  data?: T;
+  error?: string;
+  message?: string;
+}
+
+// Common function for authenticated API requests to LIMQ
+export async function limqRequest<T = unknown>(
+  endpoint: string,
+  method: 'GET' | 'POST' | 'PUT' | 'DELETE' = 'GET',
+  data?: Record<string, unknown>
+): Promise<ApiResponse<T>> {
+  // Get current session
+  const { data: { session } } = await supabase.auth.getSession();
+  
+  if (!session) {
+    throw new Error('No active session. User must be authenticated.');
+  }
+  
+  const baseUrl = process.env.NEXT_PUBLIC_LIMQ_API;
+  const url = `${baseUrl}${endpoint.startsWith('/') ? endpoint : '/' + endpoint}`;
+  
+  const options: RequestInit = {
+    method,
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${session.access_token}`
+    },
+    mode: 'cors'
+  };
+  
+  if (data && (method === 'POST' || method === 'PUT')) {
+    options.body = JSON.stringify(data);
+  }
+  
+  const response = await fetch(url, options);
+  
+  if (!response.ok) {
+    const errorData = await response.json().catch(() => null);
+    throw new Error(
+      errorData?.error || `Request failed with status ${response.status}`
+    );
+  }
+  
+  return response.json();
+} 

lib/auth.tsx

@@ -4,6 +4,7 @@ import React, { createContext, useContext, useEffect, useState } from 'react';
 import { useRouter } from 'next/navigation';
 import { Session, User } from '@supabase/supabase-js';
 import supabase from './supabase';
+import { limqRequest } from './api';
 
 // 定义用户类型
 export type AuthUser = User | null;
@@ -13,21 +14,16 @@ export type AuthContextType = {
   user: AuthUser;
   session: Session | null;
   isLoading: boolean;
-  signIn: (email: string, password: string) => Promise<{ error?: any }>;
-  signInWithGoogle: () => Promise<{ error?: any }>;
-  signInWithGitHub: () => Promise<{ error?: any }>;
+  signIn: (email: string, password: string) => Promise<{ error?: unknown }>;
+  signInWithGoogle: () => Promise<{ error?: unknown }>;
+  signInWithGitHub: () => Promise<{ error?: unknown }>;
   signUp: (email: string, password: string) => Promise<void>;
   signOut: () => Promise<void>;
-  autoRegisterTestUser: () => Promise<void>; // 添加自动注册测试用户函数
 };
 
 // 创建验证上下文
 const AuthContext = createContext<AuthContextType | undefined>(undefined);
 
-// 测试账户常量 - 使用已验证的账户
-const TEST_EMAIL = 'vitalitymailg@gmail.com';
-const TEST_PASSWORD = 'password123';
-
 // 验证提供者组件
 export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
   const [user, setUser] = useState<AuthUser>(null);
@@ -90,7 +86,8 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
 
       setSession(data.session);
       setUser(data.user);
-      router.push('/dashboard');
+      
+      router.push('/analytics');
       return {};
     } catch (error) {
       console.error('登录过程出错:', error);
@@ -104,11 +101,14 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
   const signInWithGoogle = async () => {
     setIsLoading(true);
     try {
+      // 获取网站 URL，如果环境变量不存在则使用当前来源
+      const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
+      
       // 尝试通过Supabase登录Google
       const { error } = await supabase.auth.signInWithOAuth({
         provider: 'google',
         options: {
-          redirectTo: `${window.location.origin}/auth/callback`,
+          redirectTo: `${siteUrl}/auth/callback`,
         },
       });
 
@@ -130,11 +130,14 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
   const signInWithGitHub = async () => {
     setIsLoading(true);
     try {
+      // 获取网站 URL，如果环境变量不存在则使用当前来源
+      const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
+      
       // 尝试通过Supabase登录GitHub
       const { error } = await supabase.auth.signInWithOAuth({
         provider: 'github',
         options: {
-          redirectTo: `${window.location.origin}/auth/callback`,
+          redirectTo: `${siteUrl}/auth/callback`,
         },
       });
 
@@ -156,10 +159,16 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
   const signUp = async (email: string, password: string) => {
     setIsLoading(true);
     try {
+      // 获取网站 URL，如果环境变量不存在则使用当前来源
+      const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
+      
       // 尝试通过Supabase注册
       const { error } = await supabase.auth.signUp({
         email,
         password,
+        options: {
+          emailRedirectTo: `${siteUrl}/auth/callback`,
+        }
       });
 
       if (error) {
@@ -168,7 +177,7 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
       }
 
       // 注册成功后跳转到登录页面并显示确认消息
-      router.push('/login?message=注册成功，请查看邮箱确认账户');
+      router.push('/login?message=Registration successful! Please check your email to verify your account before logging in.');
     } catch (error) {
       console.error('注册过程出错:', error);
       throw error;
@@ -198,35 +207,6 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
     }
   };
 
-  // 自动注册测试用户函数
-  const autoRegisterTestUser = async () => {
-    setIsLoading(true);
-    try {
-      console.log('正在使用测试账户登录:', TEST_EMAIL);
-      
-      // 使用测试账户直接登录
-      const { data, error } = await supabase.auth.signInWithPassword({
-        email: TEST_EMAIL,
-        password: TEST_PASSWORD,
-      });
-      
-      if (error) {
-        console.error('测试账户登录失败:', error);
-        throw error;
-      }
-      
-      console.log('测试账户登录成功!');
-      setSession(data.session);
-      setUser(data.user);
-      router.push('/dashboard');
-    } catch (error) {
-      console.error('测试账户登录出错:', error);
-      throw error;
-    } finally {
-      setIsLoading(false);
-    }
-  };
-
   const contextValue: AuthContextType = {
     user,
     session,
@@ -236,7 +216,6 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
     signInWithGitHub,
     signUp,
     signOut,
-    autoRegisterTestUser,
   };
 
   return (

lib/clickhouse.ts

@@ -1,5 +1,5 @@
 import { createClient } from '@clickhouse/client';
-import type { EventsQueryParams } from './types';
+import { EventsQueryParams } from './analytics';
 
 // ClickHouse 客户端配置
 const clickhouse = createClient({
@@ -26,6 +26,7 @@ function buildDateFilter(startTime?: string, endTime?: string): string {
 
 // 构建通用过滤条件
 export function buildFilter(params: Partial<EventsQueryParams>): string {
+  console.log('buildFilter received params:', JSON.stringify(params));
   const filters = [];
   
   // 添加日期过滤条件
@@ -43,6 +44,7 @@ export function buildFilter(params: Partial<EventsQueryParams>): string {
   
   // 添加链接ID过滤条件
   if (params.linkId) {
+    console.log('Adding link_id filter:', params.linkId);
     filters.push(`link_id = '${params.linkId}'`);
   }
   
@@ -56,6 +58,34 @@ export function buildFilter(params: Partial<EventsQueryParams>): string {
     filters.push(`user_id = '${params.userId}'`);
   }
   
+  // 添加子路径过滤条件 - 使用更精确的匹配方式
+  if (params.subpath && params.subpath.trim() !== '') {
+    console.log('====== SUBPATH DEBUG ======');
+    console.log('Raw subpath param:', params.subpath);
+    
+    // 清理并准备subpath值
+    let cleanSubpath = params.subpath.trim();
+    // 移除开头的斜杠以便匹配
+    if (cleanSubpath.startsWith('/')) {
+      cleanSubpath = cleanSubpath.substring(1);
+    }
+    // 移除结尾的斜杠以便匹配
+    if (cleanSubpath.endsWith('/')) {
+      cleanSubpath = cleanSubpath.substring(0, cleanSubpath.length - 1);
+    }
+    
+    console.log('Cleaned subpath:', cleanSubpath);
+    
+    // 使用正则表达式匹配URL中的第二个路径部分
+    // 示例: 在 "https://abc.com/slug/subpath/" 中匹配 "subpath"
+    const condition = `match(JSONExtractString(event_attributes, 'full_url'), '/[^/]+/${cleanSubpath}(/|\\\\?|$)')`;
+    
+    console.log('Final SQL condition:', condition);
+    console.log('==========================');
+    
+    filters.push(condition);
+  }
+  
   // 添加团队ID过滤条件
   if (params.teamId) {
     filters.push(`team_id = '${params.teamId}'`);
@@ -100,7 +130,7 @@ export function buildOrderBy(sortBy: string = 'event_time', sortOrder: string =
 
 // 执行查询
 export async function executeQuery(query: string) {
-  console.log('执行查询:', query); // 查询日志
+  console.log('Executing query:', query); // 查询日志
   try {
     const resultSet = await clickhouse.query({
       query,
@@ -117,7 +147,7 @@ export async function executeQuery(query: string) {
 
 // 执行返回单一结果的查询
 export async function executeQuerySingle(query: string) {
-  console.log('执行单一结果查询:', query); // 查询日志
+  console.log('Executing single result query:', query); // 查询日志
   try {
     const resultSet = await clickhouse.query({
       query,

middleware.ts

@@ -0,0 +1,22 @@
+import { createMiddlewareClient } from '@supabase/auth-helpers-nextjs';
+import { NextResponse } from 'next/server';
+import type { NextRequest } from 'next/server';
+
+export async function middleware(req: NextRequest) {
+  const res = NextResponse.next();
+  
+  // Create a Supabase client configured to use cookies
+  const supabase = createMiddlewareClient({ req, res });
+  
+  // Refresh session if expired - required for Server Components
+  await supabase.auth.getSession();
+  
+  return res;
+}
+
+// Specify the paths where this middleware should run
+export const config = {
+  matcher: [
+    '/((?!_next/static|_next/image|favicon.ico).*)',
+  ],
+}; 

package.json

@@ -3,7 +3,7 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "dev": "next dev",
+    "dev": "next dev -p 3007",
     "build": "next build",
     "start": "next start",
     "lint": "next lint",

scripts/db/sql/clickhouse/add_domain_column.sql

@@ -0,0 +1,5 @@
+-- 添加domain列到shorturl_analytics.shorturl表
+ALTER TABLE
+  shorturl_analytics.shorturl
+ADD
+  COLUMN IF NOT EXISTS domain Nullable(String) COMMENT '域名';

scripts/db/sql/clickhouse/truncate_events.sh

@@ -0,0 +1 @@
+./ch-query.sh -q "TRUNCATE TABLE shorturl_analytics.events"

scripts/db/sql/clickhouse/truncate_shorturl.sh

@@ -0,0 +1 @@
+./ch-query.sh -q "TRUNCATE TABLE shorturl_analytics.shorturl"

scripts/db/sync_mongo_to_events.ts

@@ -1,364 +0,0 @@
-// Sync data from MongoDB trace table to ClickHouse events table
-import { getVariable } from "npm:windmill-client@1";
-import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
-
-interface MongoConfig {
-  host: string;
-  port: string;
-  db: string;
-  username: string;
-  password: string;
-}
-
-interface ClickHouseConfig {
-  clickhouse_host: string;
-  clickhouse_port: number;
-  clickhouse_user: string;
-  clickhouse_password: string;
-  clickhouse_database: string;
-  clickhouse_url: string;
-}
-
-interface TraceRecord {
-  _id: ObjectId;
-  slugId: ObjectId;
-  label: string | null;
-  ip: string;
-  type: number;
-  platform: string;
-  platformOS: string;
-  browser: string;
-  browserVersion: string;
-  url: string;
-  createTime: number;
-}
-
-export async function main(
-  batch_size = 1000,
-  max_records = 9999999,
-  timeout_minutes = 60,
-  skip_clickhouse_check = false,
-  force_insert = false
-) {
-  const logWithTimestamp = (message: string) => {
-    const now = new Date();
-    console.log(`[${now.toISOString()}] ${message}`);
-  };
-
-  logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
-  logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
-
-  // Set timeout
-  const startTime = Date.now();
-  const timeoutMs = timeout_minutes * 60 * 1000;
-  
-  const checkTimeout = () => {
-    if (Date.now() - startTime > timeoutMs) {
-      console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
-      return true;
-    }
-    return false;
-  };
-
-  // Get MongoDB and ClickHouse connection info
-  let mongoConfig: MongoConfig;
-  let clickhouseConfig: ClickHouseConfig;
-  
-  try {
-    const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
-    mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
-    
-    const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
-    clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
-  } catch (error) {
-    console.error("Failed to get config:", error);
-    throw error;
-  }
-
-  // Build MongoDB connection URL
-  let mongoUrl = "mongodb://";
-  if (mongoConfig.username && mongoConfig.password) {
-    mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
-  }
-  mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
-
-  // Connect to MongoDB
-  const client = new MongoClient();
-  try {
-    await client.connect(mongoUrl);
-    console.log("MongoDB connected successfully");
-
-    const db = client.database(mongoConfig.db);
-    const traceCollection = db.collection<TraceRecord>("trace");
-    
-    // Build query conditions
-    const query: Record<string, unknown> = {
-      type: 1  // Only sync records with type 1
-    };
-
-    // Count total records
-    const totalRecords = await traceCollection.countDocuments(query);
-    console.log(`Found ${totalRecords} records to sync`);
-
-    const recordsToProcess = Math.min(totalRecords, max_records);
-    console.log(`Will process ${recordsToProcess} records`);
-
-    if (totalRecords === 0) {
-      console.log("No records to sync, task completed");
-      return {
-        success: true,
-        records_synced: 0,
-        message: "No records to sync"
-      };
-    }
-
-    // Check ClickHouse connection
-    const checkClickHouseConnection = async (): Promise<boolean> => {
-      if (skip_clickhouse_check) {
-        logWithTimestamp("Skipping ClickHouse connection check");
-        return true;
-      }
-      
-      try {
-        logWithTimestamp("Testing ClickHouse connection...");
-        const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
-        const response = await fetch(clickhouseUrl, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/x-www-form-urlencoded",
-            "Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
-          },
-          body: "SELECT 1",
-          signal: AbortSignal.timeout(5000)
-        });
-        
-        if (response.ok) {
-          logWithTimestamp("ClickHouse connection test successful");
-          return true;
-        } else {
-          const errorText = await response.text();
-          logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
-          return false;
-        }
-      } catch (err) {
-        logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
-        return false;
-      }
-    };
-
-    // Check if records exist in ClickHouse
-    const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
-      if (records.length === 0) return [];
-      
-      if (skip_clickhouse_check || force_insert) {
-        logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
-        return records;
-      }
-      
-      try {
-        const recordIds = records.map(record => record._id.toString());
-        
-        const query = `
-          SELECT event_id 
-          FROM ${clickhouseConfig.clickhouse_database}.events 
-          WHERE event_attributes LIKE '%"mongo_id":"%' 
-          AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
-          FORMAT JSON
-        `;
-        
-        const response = await fetch(clickhouseConfig.clickhouse_url, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/x-www-form-urlencoded",
-            "Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
-          },
-          body: query,
-          signal: AbortSignal.timeout(10000)
-        });
-        
-        if (!response.ok) {
-          const errorText = await response.text();
-          throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
-        }
-        
-        const result = await response.json();
-        const existingIds = new Set(result.data.map((row: any) => {
-          const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
-          return matches ? matches[1] : null;
-        }).filter(Boolean));
-        
-        return records.filter(record => !existingIds.has(record._id.toString()));
-      } catch (err) {
-        logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
-        return skip_clickhouse_check ? records : [];
-      }
-    };
-
-    // Process records function
-    const processRecords = async (records: TraceRecord[]) => {
-      if (records.length === 0) return 0;
-      
-      const newRecords = await checkExistingRecords(records);
-      if (newRecords.length === 0) {
-        logWithTimestamp("All records already exist, skipping");
-        return 0;
-      }
-      
-      // Prepare ClickHouse insert data
-      const clickhouseData = newRecords.map(record => {
-        const eventTime = new Date(record.createTime).toISOString();
-        return {
-          event_time: eventTime,
-          event_type: "click",
-          event_attributes: JSON.stringify({
-            mongo_id: record._id.toString(),
-            original_type: record.type
-          }),
-          
-          // Link information
-          link_id: record.slugId.toString(),
-          link_slug: "",
-          link_label: record.label || "",
-          link_title: "",
-          link_original_url: record.url || "",
-          link_attributes: "{}",
-          link_created_at: eventTime,
-          link_expires_at: null,
-          link_tags: "[]",
-          
-          // User information (empty as not available in trace)
-          user_id: "",
-          user_name: "",
-          user_email: "",
-          user_attributes: "{}",
-          
-          // Team information (empty as not available in trace)
-          team_id: "",
-          team_name: "",
-          team_attributes: "{}",
-          
-          // Project information (empty as not available in trace)
-          project_id: "",
-          project_name: "",
-          project_attributes: "{}",
-          
-          // QR code information (empty as not available in trace)
-          qr_code_id: "",
-          qr_code_name: "",
-          qr_code_attributes: "{}",
-          
-          // Visitor information
-          visitor_id: record._id.toString(),
-          session_id: `${record._id.toString()}-${record.createTime}`,
-          ip_address: record.ip || "",
-          country: "",
-          city: "",
-          device_type: record.platform || "unknown",
-          browser: record.browser || "",
-          os: record.platformOS || "",
-          user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
-          
-          // Source information
-          referrer: record.url || "",
-          utm_source: "",
-          utm_medium: "",
-          utm_campaign: "",
-          
-          // Interaction information
-          time_spent_sec: 0,
-          is_bounce: true,
-          is_qr_scan: false,
-          conversion_type: "visit",
-          conversion_value: 0
-        };
-      });
-
-      // Generate ClickHouse insert SQL
-      const insertSQL = `
-        INSERT INTO ${clickhouseConfig.clickhouse_database}.events 
-        FORMAT JSONEachRow
-        ${JSON.stringify(clickhouseData)}
-      `;
-
-      try {
-        const response = await fetch(clickhouseConfig.clickhouse_url, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/x-www-form-urlencoded",
-            "Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
-          },
-          body: insertSQL,
-          signal: AbortSignal.timeout(20000)
-        });
-
-        if (!response.ok) {
-          const errorText = await response.text();
-          throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
-        }
-
-        logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
-        return newRecords.length;
-      } catch (err) {
-        logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
-        throw err;
-      }
-    };
-
-    // Check ClickHouse connection before processing
-    const clickhouseConnected = await checkClickHouseConnection();
-    if (!clickhouseConnected && !skip_clickhouse_check) {
-      throw new Error("ClickHouse connection failed, cannot continue sync");
-    }
-
-    // Process records in batches
-    let processedRecords = 0;
-    let totalBatchRecords = 0;
-
-    for (let page = 0; processedRecords < recordsToProcess; page++) {
-      if (checkTimeout()) {
-        logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
-        break;
-      }
-
-      logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
-      
-      const records = await traceCollection.find(
-        query,
-        {
-          allowDiskUse: true,
-          sort: { createTime: 1 },
-          skip: page * batch_size,
-          limit: batch_size
-        }
-      ).toArray();
-
-      if (records.length === 0) {
-        logWithTimestamp("No more records found, sync complete");
-        break;
-      }
-
-      const batchSize = await processRecords(records);
-      processedRecords += records.length;
-      totalBatchRecords += batchSize;
-
-      logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
-    }
-
-    return {
-      success: true,
-      records_processed: processedRecords,
-      records_synced: totalBatchRecords,
-      message: "Data sync completed"
-    };
-  } catch (err) {
-    console.error("Error during sync:", err);
-    return {
-      success: false,
-      error: err instanceof Error ? err.message : String(err),
-      stack: err instanceof Error ? err.stack : undefined
-    };
-  } finally {
-    await client.close();
-    console.log("MongoDB connection closed");
-  }
-} 

windmill/scripts/.gitignore

@@ -0,0 +1,2 @@
+/node_modules
+/package-lock.json

windmill/scripts/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "scripts",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "dependencies": {
+    "date-fns": "^4.1.0",
+    "dotenv": "^16.5.0",
+    "fs-extra": "^11.3.0",
+    "mongodb": "^6.16.0",
+    "node-fetch": "^2.7.0"
+  }
+}

windmill/scripts/sync_mongo_to_clickhouse.js

@@ -0,0 +1,714 @@
+// 从MongoDB的trace表同步数据到ClickHouse的events表
+// 
+// 支持以下同步模式：
+// 1. 增量同步：基于上次同步状态，只同步新数据（默认模式）
+// 2. 自定义时间范围同步：通过指定开始时间和结束时间，同步特定时间范围内的数据
+//    - 可以通过时间戳参数(startTime/endTime)指定范围
+//    - 也可以通过日期字符串参数(startDate/endDate)指定范围，支持ISO格式或yyyy-MM-dd格式
+
+const { MongoClient, ObjectId } = require('mongodb');
+const fs = require('fs');
+const path = require('path');
+const fetch = require('node-fetch');
+
+// 同步状态键名和保存路径
+const SYNC_STATE_FILE = path.join(__dirname, 'mongo_sync_state.json');
+
+// 直接使用配置值
+const mongoConfig = {
+  url: "mongodb://10.0.1.41:27017",
+  db: "main" // 注意：请替换为您的实际数据库名称
+};
+
+const clickhouseConfig = {
+  clickhouse_host: "10.0.1.60",
+  clickhouse_port: "8123",
+  clickhouse_user: "admin",
+  clickhouse_password: "your_secure_password",
+  clickhouse_database: "shorturl_analytics",
+  clickhouse_url: "http://10.0.1.60:8123"
+};
+
+// 封装本地读取变量函数
+async function getVariable(key) {
+  try {
+    if (key === 'f/shorturl_analytics/mongodb') {
+      return mongoConfig;
+    } else if (key === 'f/shorturl_analytics/clickhouse') {
+      return clickhouseConfig;
+    } else if (key === 'f/shorturl_analytics/mongo_sync_state') {
+      if (fs.existsSync(SYNC_STATE_FILE)) {
+        return JSON.parse(fs.readFileSync(SYNC_STATE_FILE, 'utf8'));
+      }
+    }
+    return null;
+  } catch (error) {
+    console.error(`获取变量失败: ${error.message}`);
+    return null;
+  }
+}
+
+// 封装本地保存变量函数
+async function setVariable(key, value) {
+  try {
+    if (key === 'f/shorturl_analytics/mongo_sync_state') {
+      fs.writeFileSync(SYNC_STATE_FILE, JSON.stringify(value, null, 2));
+    }
+  } catch (error) {
+    console.error(`保存变量失败: ${error.message}`);
+    throw error;
+  }
+}
+
+// 日期字符串转时间戳工具函数（接受ISO字符串或yyyy-MM-dd格式）
+function dateToTimestamp(dateStr) {
+  try {
+    // 尝试直接解析完整的ISO日期字符串
+    const date = new Date(dateStr);
+    
+    // 检查是否为有效日期
+    if (isNaN(date.getTime())) {
+      // 尝试解析yyyy-MM-dd格式，默认设置为当天的00:00:00
+      const parts = dateStr.split('-');
+      if (parts.length === 3) {
+        const year = parseInt(parts[0], 10);
+        const month = parseInt(parts[1], 10) - 1; // 月份从0开始
+        const day = parseInt(parts[2], 10);
+        
+        const dateObj = new Date(year, month, day, 0, 0, 0);
+        return dateObj.getTime();
+      }
+      throw new Error(`无法解析日期字符串: ${dateStr}`);
+    }
+    
+    return date.getTime();
+  } catch (err) {
+    throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+// 从URL中提取UTM参数的函数，增强版
+function extractUtmParams(url, debug = false) {
+  const defaultUtmParams = {
+    utm_source: "",
+    utm_medium: "",
+    utm_campaign: "",
+    utm_term: "",
+    utm_content: ""
+  };
+  
+  if (!url) return defaultUtmParams;
+  
+  if (debug) {
+    console.log(`[UTM提取] 原始URL: ${url}`);
+  }
+  
+  // 准备一个解析后的参数对象
+  const params = { ...defaultUtmParams };
+  
+  // 尝试多种方法提取UTM参数
+  
+  // 方法1: 使用URL对象解析
+  try {
+    // 先处理URL，确保是完整的URL格式
+    let normalizedUrl = url;
+    if (!url.match(/^https?:\/\//i)) {
+      normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
+    }
+    
+    const urlObj = new URL(normalizedUrl);
+    
+    // 读取URL参数
+    if (urlObj.searchParams.has('utm_source')) 
+      params.utm_source = urlObj.searchParams.get('utm_source') || "";
+    if (urlObj.searchParams.has('utm_medium')) 
+      params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
+    if (urlObj.searchParams.has('utm_campaign')) 
+      params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
+    if (urlObj.searchParams.has('utm_term')) 
+      params.utm_term = urlObj.searchParams.get('utm_term') || "";
+    if (urlObj.searchParams.has('utm_content')) 
+      params.utm_content = urlObj.searchParams.get('utm_content') || "";
+    
+    if (debug) {
+      console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
+    }
+    
+    // 如果至少找到一个UTM参数，则返回
+    if (params.utm_source || params.utm_medium || params.utm_campaign || 
+        params.utm_term || params.utm_content) {
+      return params;
+    }
+  } catch (err) {
+    if (debug) {
+      console.log(`[UTM提取] URL对象解析失败，尝试正则表达式`);
+    }
+  }
+  
+  // 方法2: 使用正则表达式提取参数
+  // 使用正则表达式（最安全的方法，适用于任何格式）
+  const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
+  if (sourceMatch && sourceMatch[1]) {
+    try {
+      params.utm_source = decodeURIComponent(sourceMatch[1]);
+    } catch (err) {
+      params.utm_source = sourceMatch[1];
+    }
+  }
+  
+  const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
+  if (mediumMatch && mediumMatch[1]) {
+    try {
+      params.utm_medium = decodeURIComponent(mediumMatch[1]);
+    } catch (err) {
+      params.utm_medium = mediumMatch[1];
+    }
+  }
+  
+  const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
+  if (campaignMatch && campaignMatch[1]) {
+    try {
+      params.utm_campaign = decodeURIComponent(campaignMatch[1]);
+    } catch (err) {
+      params.utm_campaign = campaignMatch[1];
+    }
+  }
+  
+  const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
+  if (termMatch && termMatch[1]) {
+    try {
+      params.utm_term = decodeURIComponent(termMatch[1]);
+    } catch (err) {
+      params.utm_term = termMatch[1];
+    }
+  }
+  
+  const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
+  if (contentMatch && contentMatch[1]) {
+    try {
+      params.utm_content = decodeURIComponent(contentMatch[1]);
+    } catch (err) {
+      params.utm_content = contentMatch[1];
+    }
+  }
+  
+  if (debug) {
+    console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
+  }
+  
+  return params;
+}
+
+// 解析命令行参数
+function parseCommandLineArgs() {
+  const args = {};
+  process.argv.slice(2).forEach(arg => {
+    if (arg.startsWith('--')) {
+      const [key, value] = arg.substring(2).split('=');
+      args[key] = value || true;
+    }
+  });
+  return args;
+}
+
+async function main() {
+  const args = parseCommandLineArgs();
+  
+  // 参数设置
+  const batch_size = parseInt(args['batch-size'] || '1000');
+  const max_records = parseInt(args['max-records'] || '9999999');
+  const timeout_minutes = parseInt(args['timeout'] || '60');
+  const skip_clickhouse_check = args['skip-clickhouse-check'] === 'true';
+  const force_insert = args['force-insert'] !== 'false';
+  const database_override = args['database'] || 'shorturl_analytics';
+  const reset_sync_state = args['reset-sync-state'] === 'true';
+  const debug_utm = args['debug-utm'] === 'true';
+  const start_time = args['start-time'] ? parseInt(args['start-time']) : undefined;
+  const end_time = args['end-time'] ? parseInt(args['end-time']) : undefined;
+  const use_custom_time_range = args['use-custom-time-range'] === 'true';
+  const start_date = args['start-date'];
+  const end_date = args['end-date'];
+  
+  const logWithTimestamp = (message) => {
+    const now = new Date();
+    console.log(`[${now.toISOString()}] ${message}`);
+  };
+
+  logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
+  logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
+  
+  let customStartTime = start_time;
+  let customEndTime = end_time;
+  let useCustomTimeRange = use_custom_time_range;
+  
+  // 处理日期字符串参数，转换为时间戳
+  if (start_date) {
+    try {
+      customStartTime = dateToTimestamp(start_date);
+      logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${customStartTime}`);
+      useCustomTimeRange = true;
+    } catch (err) {
+      logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  
+  if (end_date) {
+    try {
+      customEndTime = dateToTimestamp(end_date);
+      // 如果是日期格式，设置为当天结束时间 (23:59:59.999)
+      if (end_date.split('-').length === 3 && end_date.length <= 10) {
+        customEndTime += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
+        logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${customEndTime}`);
+      } else {
+        logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${customEndTime}`);
+      }
+      useCustomTimeRange = true;
+    } catch (err) {
+      logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  
+  if (skip_clickhouse_check) {
+    logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
+  }
+  if (force_insert) {
+    logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
+  }
+  if (reset_sync_state) {
+    logWithTimestamp("⚠️ 警告: 已启用重置同步状态，将从头开始同步数据");
+  }
+  if (debug_utm) {
+    logWithTimestamp("已启用UTM参数调试日志");
+  }
+  if (useCustomTimeRange) {
+    if (customStartTime) {
+      logWithTimestamp(`已启用自定义时间范围：开始时间 ${new Date(customStartTime).toISOString()}`);
+    }
+    if (customEndTime) {
+      logWithTimestamp(`已启用自定义时间范围：结束时间 ${new Date(customEndTime).toISOString()}`);
+    }
+  }
+
+  // 设置超时
+  const startTime = Date.now();
+  const timeoutMs = timeout_minutes * 60 * 1000;
+  
+  // 检查是否超时
+  const checkTimeout = () => {
+    if (Date.now() - startTime > timeoutMs) {
+      console.log(`运行时间超过${timeout_minutes}分钟，暂停执行`);
+      return true;
+    }
+    return false;
+  };
+
+  // 获取上次同步状态
+  let lastSyncState = null;
+  if (!reset_sync_state) {
+    try {
+      const rawSyncState = await getVariable("f/shorturl_analytics/mongo_sync_state");
+      if (rawSyncState) {
+        lastSyncState = rawSyncState;
+      }
+    } catch (error) {
+      logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
+    }
+  }
+
+  if (lastSyncState) {
+    logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
+    if (lastSyncState.last_sync_id) {
+      logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
+    }
+  } else {
+    logWithTimestamp("没有找到上次同步状态，将从头开始同步");
+  }
+
+  // 连接MongoDB
+  const client = new MongoClient(mongoConfig.url);
+  try {
+    await client.connect();
+    console.log("MongoDB连接成功");
+
+    const db = client.db(mongoConfig.db);
+    const traceCollection = db.collection("trace");
+    const shortCollection = db.collection("short");
+    
+    // 构建查询条件
+    const query = {
+      type: 1  // 只同步type为1的记录
+    };
+
+    // 根据时间范围参数构建查询条件
+    if (useCustomTimeRange) {
+      // 使用自定义时间范围
+      const timeQuery = {};
+      
+      if (customStartTime) {
+        timeQuery.$gte = customStartTime;
+        logWithTimestamp(`将只同步createTime >= ${customStartTime} (${new Date(customStartTime).toISOString()}) 的记录`);
+      }
+      
+      if (customEndTime) {
+        timeQuery.$lte = customEndTime;
+        logWithTimestamp(`将只同步createTime <= ${customEndTime} (${new Date(customEndTime).toISOString()}) 的记录`);
+      }
+      
+      // 只有当至少指定了一个时间限制时才添加时间查询条件
+      if (Object.keys(timeQuery).length > 0) {
+        query.createTime = timeQuery;
+      }
+    } 
+    // 如果不使用自定义时间范围，且有上次同步状态，则只获取更新的记录
+    else if (lastSyncState && lastSyncState.last_sync_time) {
+      // 使用上次同步时间作为过滤条件
+      query.createTime = { $gt: lastSyncState.last_sync_time };
+      logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
+    }
+
+    // 计算总记录数
+    const totalRecords = await traceCollection.countDocuments(query);
+    console.log(`找到 ${totalRecords} 条新记录需要同步`);
+
+    // 限制此次处理的记录数量
+    const recordsToProcess = Math.min(totalRecords, max_records);
+    console.log(`本次将处理 ${recordsToProcess} 条记录`);
+
+    if (totalRecords === 0) {
+      console.log("没有新记录需要同步，任务完成");
+      return {
+        success: true,
+        records_synced: 0,
+        message: "没有新记录需要同步"
+      };
+    }
+
+    // 检查ClickHouse连接状态
+    const checkClickHouseConnection = async () => {
+      if (skip_clickhouse_check) {
+        logWithTimestamp("已启用跳过ClickHouse检查，不测试连接");
+        return true;
+      }
+      
+      try {
+        logWithTimestamp("测试ClickHouse连接...");
+        const clickhouseUrl = clickhouseConfig.clickhouse_url;
+        const response = await fetch(clickhouseUrl, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`,
+          },
+          body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
+        });
+        
+        if (response.ok) {
+          logWithTimestamp("ClickHouse连接测试成功");
+          return true;
+        } else {
+          const errorText = await response.text();
+          logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
+          return false;
+        }
+      } catch (err) {
+        logWithTimestamp(`ClickHouse连接测试失败: ${err.message}`);
+        return false;
+      }
+    };
+
+    // 在处理记录前先检查ClickHouse连接
+    const clickhouseConnected = await checkClickHouseConnection();
+    if (!clickhouseConnected && !skip_clickhouse_check) {
+      logWithTimestamp("⚠️ ClickHouse连接测试失败，请启用skip_clickhouse_check=true参数来跳过连接检查");
+      throw new Error("ClickHouse连接失败，无法继续同步");
+    }
+
+    // 处理记录的函数
+    const processRecords = async (records) => {
+      if (records.length === 0) return 0;
+      
+      logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
+      
+      // 强制使用所有记录，不检查重复
+      const newRecords = records;
+      
+      logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
+      
+      // 获取链接信息
+      const slugIds = newRecords.map(record => new ObjectId(record.slugId));
+      logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
+      const shortLinks = await shortCollection.find({
+        _id: { $in: slugIds }
+      }).toArray();
+      
+      // 创建映射用于快速查找
+      const shortLinksMap = new Map(shortLinks.map((link) => [link._id.toString(), link]));
+      logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息，${newRecords.length - shortLinks.length} 条数据将使用占位符`);
+      
+      // 准备ClickHouse插入数据
+      const clickhouseData = newRecords.map(record => {
+        const eventTime = new Date(record.createTime);
+        
+        // 获取对应的短链接信息
+        const shortLink = shortLinksMap.get(record.slugId.toString());
+        
+        // 提取URL中的UTM参数
+        if (debug_utm && record.url) {
+          logWithTimestamp(`======== UTM参数调试 ========`);
+          logWithTimestamp(`记录ID: ${record._id.toString()}`);
+          logWithTimestamp(`原始URL: ${record.url}`);
+        }
+        
+        const utmParams = extractUtmParams(record.url || "", debug_utm);
+        
+        if (debug_utm) {
+          logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
+          logWithTimestamp(`===========================`);
+        }
+        
+        // 保存提取的UTM参数和URL到event_attributes
+        const eventAttributes = {
+          mongo_id: record._id.toString(),
+          url: record.url || "",
+          ...(record.url ? { raw_url: record.url } : {})
+        };
+        
+        // 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
+        return {
+          // UUID将由ClickHouse自动生成 (event_id)
+          event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
+          event_type: record.type === 1 ? "visit" : "custom",
+          event_attributes: JSON.stringify(eventAttributes),
+          link_id: record.slugId.toString(),
+          link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
+          link_label: record.label || "",
+          link_title: shortLink?.title || "unknown_title", // 使用占位符
+          link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
+          link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
+          link_created_at: shortLink?.createTime 
+            ? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
+            : eventTime.toISOString().replace('T', ' ').replace('Z', ''),
+          link_expires_at: shortLink?.expiresAt 
+            ? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '') 
+            : null,
+          link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
+          user_id: shortLink?.user || "unknown_user", // 使用占位符
+          user_name: "unknown_user", // 使用占位符
+          user_email: "",
+          user_attributes: "{}",
+          team_id: shortLink?.teamId || "unknown_team", // 使用占位符
+          team_name: "unknown_team", // 使用占位符
+          team_attributes: "{}",
+          project_id: shortLink?.projectId || "unknown_project", // 使用占位符
+          project_name: "unknown_project", // 使用占位符
+          project_attributes: "{}",
+          qr_code_id: "",
+          qr_code_name: "",
+          qr_code_attributes: "{}",
+          visitor_id: record._id.toString(),
+          session_id: record._id.toString() + "-" + record.createTime,
+          ip_address: record.ip || "0.0.0.0", // 使用占位符
+          country: "", 
+          city: "",
+          device_type: record.platform || "unknown",
+          browser: record.browser || "unknown", // 使用占位符
+          os: record.platformOS || "unknown", // 使用占位符
+          user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
+          referrer: record.url || "",
+          utm_source: utmParams.utm_source || "",
+          utm_medium: utmParams.utm_medium || "",
+          utm_campaign: utmParams.utm_campaign || "",
+          utm_term: utmParams.utm_term || "",
+          utm_content: utmParams.utm_content || "",
+          time_spent_sec: 0,
+          is_bounce: true,
+          is_qr_scan: false,
+          conversion_type: "visit",
+          conversion_value: 0,
+          req_full_path: record.url || ""
+        };
+      });
+
+      // 生成ClickHouse插入SQL
+      const insertSQL = `
+        INSERT INTO ${clickhouseConfig.clickhouse_database}.events 
+        (event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title, 
+         link_original_url, link_attributes, link_created_at, link_expires_at, link_tags, 
+         user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes, 
+         project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes, 
+         visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent, 
+         referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec, 
+         is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
+        VALUES ${clickhouseData.map(record => {
+          // 确保所有字符串值都是字符串类型，并安全处理替换
+          const safeReplace = (val) => {
+            // 确保值是字符串，如果是null或undefined则使用空字符串
+            const str = val === null || val === undefined ? "" : String(val);
+            // 安全替换单引号
+            return str.replace(/'/g, "''");
+          };
+          
+          return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}', 
+            '${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}', 
+            '${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}', 
+            ${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}', 
+            '${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}', 
+            '${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}', 
+            '${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}', 
+            '${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}', 
+            '${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}', 
+            '${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}', 
+            '${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}', 
+            '${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}', 
+            '${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}', 
+            '${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan}, 
+            '${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
+        }).join(", ")}
+      `;
+
+      if (insertSQL.length === 0) {
+        console.log("没有新记录需要插入");
+        return 0;
+      }
+
+      // 发送请求到ClickHouse
+      const clickhouseUrl = clickhouseConfig.clickhouse_url;
+      try {
+        logWithTimestamp("发送插入请求到ClickHouse...");
+        const response = await fetch(clickhouseUrl, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`
+          },
+          body: insertSQL,
+        });
+
+        if (!response.ok) {
+          const errorText = await response.text();
+          throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
+        }
+
+        logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
+        return newRecords.length;
+      } catch (err) {
+        logWithTimestamp(`向ClickHouse插入数据失败: ${err.message}`);
+        throw err;
+      }
+    };
+
+    // 批量处理记录
+    let processedRecords = 0;
+    let totalBatchRecords = 0;
+    let lastSyncTime = 0;
+
+    for (let page = 0; processedRecords < recordsToProcess; page++) {
+      // 检查超时
+      if (checkTimeout()) {
+        logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录，因超时暂停执行`);
+        break;
+      }
+
+      // 每批次都输出进度
+      logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
+      
+      logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
+      
+      // 查询MongoDB数据
+      const records = await traceCollection.find(query)
+        .sort({ createTime: 1 })
+        .skip(page * batch_size)
+        .limit(batch_size)
+        .toArray();
+
+      if (records.length === 0) {
+        logWithTimestamp("没有找到更多数据，同步结束");
+        break;
+      }
+      
+      // 找到数据，开始处理
+      logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
+      // 输出当前批次的部分数据信息
+      if (records.length > 0) {
+        logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
+        if (records.length > 1) {
+          logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
+        }
+        
+        // 如果开启了调试，输出一些URL样本
+        if (debug_utm) {
+          const sampleSize = Math.min(5, records.length);
+          logWithTimestamp(`URL样本 (前${sampleSize}条):`);
+          for (let i = 0; i < sampleSize; i++) {
+            if (records[i].url) {
+              logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
+            }
+          }
+        }
+      }
+
+      const batchSize = await processRecords(records);
+      processedRecords += records.length;
+      totalBatchRecords += batchSize;
+
+      // 更新最后处理的记录时间和ID
+      if (records.length > 0) {
+        const lastRecord = records[records.length - 1];
+        lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
+      }
+
+      logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
+    }
+
+    // 更新同步状态
+    if (processedRecords > 0 && lastSyncTime > 0) {
+      // 只在非自定义时间范围模式下更新同步状态
+      if (!useCustomTimeRange) {
+        // 创建新的同步状态，简化对象结构
+        const newSyncState = {
+          last_sync_time: lastSyncTime,
+          records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords
+        };
+        
+        try {
+          // 保存同步状态
+          await setVariable("f/shorturl_analytics/mongo_sync_state", newSyncState);
+          logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
+        } catch (err) {
+          logWithTimestamp(`更新同步状态失败: ${err.message}，将继续执行`);
+        }
+      } else {
+        logWithTimestamp("使用自定义时间范围模式，不更新全局同步状态");
+      }
+    }
+
+    return {
+      success: true,
+      records_processed: processedRecords,
+      records_synced: totalBatchRecords,
+      last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
+      message: useCustomTimeRange ? "自定义时间范围数据同步完成" : "数据同步完成",
+      custom_time_range_used: useCustomTimeRange
+    };
+  } catch (err) {
+    console.error("同步过程中发生错误:", err);
+    return {
+      success: false,
+      error: err.message,
+      stack: err.stack
+    };
+  } finally {
+    // 关闭MongoDB连接
+    await client.close();
+    console.log("MongoDB连接已关闭");
+  }
+}
+
+// 执行主函数
+main().then(result => {
+  console.log("任务执行结果:", result);
+  process.exit(result.success ? 0 : 1);
+}).catch(err => {
+  console.error("执行出错:", err);
+  process.exit(1);
+});

windmill/sync_mongo_short_to_postgres_short_url_shorturl.ts

@@ -0,0 +1,522 @@
+// 从MongoDB的main.short表同步数据到PostgreSQL的short_url.shorturl表
+import { getVariable, setVariable, getResource } from "npm:windmill-client@1";
+import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
+import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
+
+interface MongoConfig {
+  host: string;
+  port: string;
+  db: string;
+  username: string;
+  password: string;
+}
+
+interface PostgresConfig {
+  host: string;
+  port: number;
+  database: string;
+  user: string;
+  password: string;
+  schema: string;
+}
+
+// 扩展ShortRecord接口以包含更多可能的字段
+interface ShortRecord {
+  _id: ObjectId;
+  origin: string;
+  slug: string;
+  domain: string | null;
+  createTime: number | { $numberLong: string } | string;
+  // 可选字段
+  expiredAt?: number | { $numberLong: string } | string | null;
+  expiredUrl?: string | null;
+  password?: string | null;
+  image?: string | null;
+  title?: string | null;
+  description?: string | null;
+}
+
+interface SyncState {
+  last_sync_time: number;
+  records_synced: number;
+  last_sync_id?: string;
+}
+
+// 同步状态键名
+const SYNC_STATE_KEY = "f/limq/mongo_short_to_postgres_shorturl_shorturl_state";
+
+export async function main(
+  batch_size = 1000,
+  max_records = 9999999,
+  timeout_minutes = 60,
+  skip_duplicate_check = false,
+  force_insert = false,
+  reset_sync_state = false,
+  postgres_schema = "short_url", // 添加schema参数，允许运行时指定
+  postgres_database = "postgres", // 添加数据库名称参数，默认为postgres
+  domain = "upj.to" // 添加domain参数，允许用户指定域名
+) {
+  const logWithTimestamp = (message: string) => {
+    const now = new Date();
+    console.log(`[${now.toISOString()}] ${message}`);
+  };
+
+  logWithTimestamp("开始执行MongoDB到PostgreSQL的同步任务");
+  logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
+  logWithTimestamp(`使用域名: ${domain}`);
+  if (skip_duplicate_check) {
+    logWithTimestamp("⚠️ 警告: 已启用跳过重复检查模式，不会检查记录是否已存在");
+  }
+  if (force_insert) {
+    logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
+  }
+  if (reset_sync_state) {
+    logWithTimestamp("⚠️ 警告: 已启用重置同步状态，将从头开始同步数据");
+  }
+
+  // 设置超时
+  const startTime = Date.now();
+  const timeoutMs = timeout_minutes * 60 * 1000;
+  
+  // 检查是否超时
+  const checkTimeout = () => {
+    if (Date.now() - startTime > timeoutMs) {
+      logWithTimestamp(`运行时间超过${timeout_minutes}分钟，暂停执行`);
+      return true;
+    }
+    return false;
+  };
+
+  // 日期解析函数，处理不同格式的日期
+  const parseDate = (dateValue: any): Date | null => {
+    if (!dateValue) return null;
+
+    // 处理 MongoDB $numberLong 格式
+    if (dateValue.$numberLong) {
+      return new Date(Number(dateValue.$numberLong));
+    }
+
+    // 处理普通时间戳
+    if (typeof dateValue === 'number') {
+      return new Date(dateValue);
+    }
+
+    // 处理 ISO 字符串格式
+    if (typeof dateValue === 'string') {
+      const date = new Date(dateValue);
+      return isNaN(date.getTime()) ? null : date;
+    }
+
+    return null;
+  };
+
+  // 获取MongoDB和PostgreSQL的连接信息
+  let mongoConfig: MongoConfig;
+  let postgresConfig: PostgresConfig;
+  
+  try {
+    const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
+    if (typeof rawMongoConfig === "string") {
+      try {
+        mongoConfig = JSON.parse(rawMongoConfig);
+      } catch (e) {
+        console.error("MongoDB配置解析失败:", e);
+        throw e;
+      }
+    } else {
+      mongoConfig = rawMongoConfig as MongoConfig;
+    }
+    
+    // 使用getResource获取PostgreSQL资源
+    try {
+      logWithTimestamp("正在获取PostgreSQL资源...");
+      const resourceConfig = await getResource("f/limq/production_supabase");
+      
+      // 将resource转换为PostgresConfig
+      postgresConfig = {
+        host: resourceConfig.host || "",
+        port: Number(resourceConfig.port) || 5432,
+        user: resourceConfig.user || "",
+        password: resourceConfig.password || "",
+        database: resourceConfig.database || postgres_database, // 使用提供的数据库名称作为备选
+        schema: resourceConfig.schema || postgres_schema // 使用提供的schema作为备选
+      };
+      
+      // 检查并记录配置信息
+      if (!postgresConfig.database || postgresConfig.database === "undefined") {
+        postgresConfig.database = postgres_database;
+        logWithTimestamp(`数据库名称未指定或为"undefined"，使用提供的值: ${postgresConfig.database}`);
+      }
+      
+      if (!postgresConfig.schema || postgresConfig.schema === "undefined") {
+        postgresConfig.schema = postgres_schema;
+        logWithTimestamp(`Schema未指定或为"undefined"，使用提供的值: ${postgresConfig.schema}`);
+      }
+      
+      logWithTimestamp(`PostgreSQL配置: 数据库=${postgresConfig.database}, Schema=${postgresConfig.schema}`);
+    } catch (e) {
+      console.error("获取PostgreSQL资源失败:", e);
+      throw e;
+    }
+    
+    console.log("MongoDB配置:", JSON.stringify({
+      ...mongoConfig,
+      password: "****" // 隐藏密码
+    }));
+    console.log("PostgreSQL配置:", JSON.stringify({
+      ...postgresConfig,
+      password: "****" // 隐藏密码
+    }));
+  } catch (error) {
+    console.error("获取配置失败:", error);
+    throw error;
+  }
+
+  // 获取上次同步状态
+  let lastSyncState: SyncState | null = null;
+  if (!reset_sync_state) {
+    try {
+      const rawSyncState = await getVariable(SYNC_STATE_KEY);
+      if (rawSyncState) {
+        if (typeof rawSyncState === "string") {
+          try {
+            lastSyncState = JSON.parse(rawSyncState);
+          } catch (e) {
+            logWithTimestamp(`解析上次同步状态失败: ${e}, 将从头开始同步`);
+          }
+        } else {
+          lastSyncState = rawSyncState as SyncState;
+        }
+      }
+    } catch (error) {
+      logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
+    }
+  }
+
+  if (lastSyncState) {
+    logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
+    if (lastSyncState.last_sync_id) {
+      logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
+    }
+  } else {
+    logWithTimestamp("没有找到上次同步状态，将从头开始同步");
+  }
+
+  // 构建MongoDB连接URL
+  let mongoUrl = "mongodb://";
+  if (mongoConfig.username && mongoConfig.password) {
+    mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
+  }
+  mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
+
+  console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
+
+  // 构建PostgreSQL连接URL
+  const pgConnectionString = `postgres://${postgresConfig.user}:${postgresConfig.password}@${postgresConfig.host}:${postgresConfig.port}/${postgresConfig.database}`;
+  console.log(`PostgreSQL连接URL: ${pgConnectionString.replace(/:[^:]*@/, ":****@")}`);
+
+  // 连接MongoDB
+  const mongoClient = new MongoClient();
+  let pgClient: Client | null = null;
+  
+  try {
+    await mongoClient.connect(mongoUrl);
+    logWithTimestamp("MongoDB连接成功");
+
+    // 连接PostgreSQL
+    pgClient = new Client(pgConnectionString);
+    await pgClient.connect();
+    logWithTimestamp("PostgreSQL连接成功");
+
+    // 确认PostgreSQL schema存在
+    try {
+      await pgClient.queryArray(`SELECT 1 FROM information_schema.schemata WHERE schema_name = '${postgresConfig.schema}'`);
+      logWithTimestamp(`PostgreSQL schema '${postgresConfig.schema}' 已确认存在`);
+    } catch (error) {
+      logWithTimestamp(`检查PostgreSQL schema失败: ${error}`);
+      throw new Error(`Schema '${postgresConfig.schema}' 可能不存在`);
+    }
+
+    const db = mongoClient.database(mongoConfig.db);
+    const shortCollection = db.collection<ShortRecord>("short");
+    
+    // 构建查询条件，根据上次同步状态获取新记录
+    const query: Record<string, unknown> = {};
+
+    // 如果有上次同步状态，则只获取更新的记录
+    if (lastSyncState && lastSyncState.last_sync_time) {
+      // 使用上次同步时间作为过滤条件
+      query.createTime = { $gt: lastSyncState.last_sync_time };
+      logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
+    }
+
+    // 计算总记录数
+    const totalRecords = await shortCollection.countDocuments(query);
+    logWithTimestamp(`找到 ${totalRecords} 条新记录需要同步`);
+
+    // 限制此次处理的记录数量
+    const recordsToProcess = Math.min(totalRecords, max_records);
+    logWithTimestamp(`本次将处理 ${recordsToProcess} 条记录`);
+
+    if (totalRecords === 0) {
+      logWithTimestamp("没有新记录需要同步，任务完成");
+      return {
+        success: true,
+        records_synced: 0,
+        message: "没有新记录需要同步"
+      };
+    }
+
+    // 检查记录是否已经存在于PostgreSQL中
+    const checkExistingRecords = async (records: ShortRecord[]): Promise<ShortRecord[]> => {
+      if (records.length === 0) return [];
+      
+      // 如果跳过重复检查或强制插入，则直接返回所有记录
+      if (skip_duplicate_check || force_insert) {
+        logWithTimestamp(`已跳过重复检查，准备处理所有 ${records.length} 条记录`);
+        return records;
+      }
+      
+      logWithTimestamp(`正在检查 ${records.length} 条记录是否已存在于PostgreSQL中...`);
+      
+      try {
+        // 提取所有记录的slugs
+        const slugs = records.map(record => record.slug);
+        
+        // 查询PostgreSQL中是否已存在这些slugs
+        const result = await pgClient!.queryArray(`
+          SELECT slug FROM ${postgresConfig.schema}.shorturl 
+          WHERE slug = ANY($1::text[])
+        `, [slugs]);
+        
+        // 将已存在的slugs加入到集合中
+        const existingSlugs = new Set<string>();
+        for (const row of result.rows) {
+          existingSlugs.add(row[0] as string);
+        }
+        
+        logWithTimestamp(`检测到 ${existingSlugs.size} 条记录已存在于PostgreSQL中`);
+        
+        // 过滤出不存在的记录
+        const newRecords = records.filter(record => !existingSlugs.has(record.slug));
+        logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
+        
+        return newRecords;
+      } catch (err) {
+        const error = err as Error;
+        logWithTimestamp(`PostgreSQL查询出错: ${error.message}`);
+        if (skip_duplicate_check) {
+          logWithTimestamp("已启用跳过重复检查，将继续处理所有记录");
+          return records;
+        } else {
+          throw error;
+        }
+      }
+    };
+
+    // 处理记录的函数
+    const processRecords = async (records: ShortRecord[]) => {
+      if (records.length === 0) return 0;
+      
+      logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
+      
+      // 检查记录是否已存在
+      let newRecords;
+      try {
+        newRecords = await checkExistingRecords(records);
+      } catch (err) {
+        const error = err as Error;
+        logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
+        if (!skip_duplicate_check && !force_insert) {
+          throw error;
+        }
+        // 如果跳过检查或强制插入，则使用所有记录
+        logWithTimestamp("将使用所有记录进行处理");
+        newRecords = records;
+      }
+      
+      if (newRecords.length === 0) {
+        logWithTimestamp("所有记录都已存在，跳过处理");
+        return 0;
+      }
+      
+      logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
+      
+      // 批量插入PostgreSQL
+      try {
+        // 开始事务
+        await pgClient!.queryArray('BEGIN');
+        
+        let insertedCount = 0;
+        
+        // 由于参数可能很多，按小批次处理
+        const smallBatchSize = 100;
+        for (let i = 0; i < newRecords.length; i += smallBatchSize) {
+          const batchRecords = newRecords.slice(i, i + smallBatchSize);
+          
+          // 构造批量插入语句
+          const placeholders = [];
+          const values = [];
+          let valueIndex = 1;
+          
+          for (const record of batchRecords) {
+            // 参考提供的字段处理方式处理数据
+            const createdAt = parseDate(record.createTime);
+            const updatedAt = createdAt; // 设置更新时间等于创建时间
+            const fullShortUrl = `${domain}/${record.slug}`;
+            
+            placeholders.push(`($${valueIndex}, $${valueIndex+1}, $${valueIndex+2}, $${valueIndex+3}, $${valueIndex+4}, $${valueIndex+5}, $${valueIndex+6}, $${valueIndex+7}, $${valueIndex+8}, $${valueIndex+9}, $${valueIndex+10}, $${valueIndex+11}, $${valueIndex+12})`);
+            
+            values.push(
+              record._id.toString(), // id
+              record.slug, // slug
+              domain, // domain (使用提供的域名)
+              record.slug, // name (使用slug作为name)
+              record.slug, // title (使用slug作为title)
+              record.origin || '', // origin
+              createdAt, // created_at
+              updatedAt, // updated_at
+              fullShortUrl, // full_short_url
+              record.image || null, // image
+              record.description || null, // description
+              record.expiredUrl || null, // expired_url
+              parseDate(record.expiredAt) // expired_at
+            );
+            
+            valueIndex += 13;
+          }
+          
+          const query = `
+            INSERT INTO ${postgresConfig.schema}.shorturl 
+            (id, slug, domain, name, title, origin, created_at, updated_at, full_short_url, image, description, expired_url, expired_at)
+            VALUES ${placeholders.join(', ')}
+          `;
+          
+          await pgClient!.queryArray(query, values);
+          insertedCount += batchRecords.length;
+          logWithTimestamp(`已插入 ${insertedCount}/${newRecords.length} 条记录`);
+        }
+        
+        // 提交事务
+        await pgClient!.queryArray('COMMIT');
+        
+        logWithTimestamp(`成功插入 ${insertedCount} 条记录到PostgreSQL`);
+        return insertedCount;
+      } catch (err) {
+        const error = err as Error;
+        // 发生错误，回滚事务
+        await pgClient!.queryArray('ROLLBACK');
+        logWithTimestamp(`向PostgreSQL插入数据失败: ${error.message}`);
+        throw error;
+      }
+    };
+
+    // 批量处理记录
+    let processedRecords = 0;
+    let totalBatchRecords = 0;
+    let lastSyncTime = 0;
+    let lastSyncId = "";
+
+    for (let page = 0; processedRecords < recordsToProcess; page++) {
+      // 检查超时
+      if (checkTimeout()) {
+        logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录，因超时暂停执行`);
+        break;
+      }
+
+      // 每批次都输出进度
+      logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
+      
+      logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
+      const records = await shortCollection.find(
+        query,
+        {
+          sort: { createTime: 1 },
+          skip: page * batch_size,
+          limit: batch_size
+        }
+      ).toArray();
+
+      if (records.length === 0) {
+        logWithTimestamp("没有找到更多数据，同步结束");
+        break;
+      }
+      
+      // 找到数据，开始处理
+      logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
+      
+      // 输出当前批次的部分数据信息
+      if (records.length > 0) {
+        logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, slug=${records[0].slug}, 时间=${new Date(typeof records[0].createTime === 'number' ? records[0].createTime : 0).toISOString()}`);
+        if (records.length > 1) {
+          const lastRec = records[records.length-1];
+          logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${lastRec._id}, slug=${lastRec.slug}, 时间=${new Date(typeof lastRec.createTime === 'number' ? lastRec.createTime : 0).toISOString()}`);
+        }
+      }
+
+      const batchSize = await processRecords(records);
+      processedRecords += records.length;
+      totalBatchRecords += batchSize;
+
+      // 更新最后处理的记录时间和ID
+      if (records.length > 0) {
+        const lastRecord = records[records.length - 1];
+        // 提取数字时间戳
+        let lastCreateTime = 0;
+        if (typeof lastRecord.createTime === 'number') {
+          lastCreateTime = lastRecord.createTime;
+        } else if (lastRecord.createTime && lastRecord.createTime.$numberLong) {
+          lastCreateTime = Number(lastRecord.createTime.$numberLong);
+        }
+        
+        lastSyncTime = Math.max(lastSyncTime, lastCreateTime);
+        lastSyncId = lastRecord._id.toString();
+      }
+
+      logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
+    }
+
+    // 更新同步状态
+    if (processedRecords > 0 && lastSyncTime > 0) {
+      // 创建新的同步状态
+      const newSyncState: SyncState = {
+        last_sync_time: lastSyncTime,
+        records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + totalBatchRecords,
+        last_sync_id: lastSyncId
+      };
+      
+      try {
+        // 保存同步状态
+        await setVariable(SYNC_STATE_KEY, newSyncState);
+        logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
+      } catch (err) {
+        const error = err as Error;
+        logWithTimestamp(`更新同步状态失败: ${error.message}`);
+      }
+    }
+
+    return {
+      success: true,
+      records_processed: processedRecords,
+      records_synced: totalBatchRecords,
+      last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
+      message: "数据同步完成"
+    };
+  } catch (err) {
+    const error = err as Error;
+    console.error("同步过程中发生错误:", error);
+    return {
+      success: false,
+      error: error.message,
+      stack: error.stack
+    };
+  } finally {
+    // 关闭连接
+    if (pgClient) {
+      await pgClient.end();
+      logWithTimestamp("PostgreSQL连接已关闭");
+    }
+    await mongoClient.close();
+    logWithTimestamp("MongoDB连接已关闭");
+  }
+} 

windmill/sync_mongo_to_events.ts

@@ -1,5 +1,13 @@
-// Sync data from MongoDB trace table to ClickHouse events table
-import { getVariable } from "npm:windmill-client@1";
+// 从MongoDB的trace表同步数据到ClickHouse的events表
+// 
+// 支持以下同步模式：
+// 1. 增量同步：基于上次同步状态，只同步新数据（默认模式）
+// 2. 自定义时间范围同步：通过指定开始时间和结束时间，同步特定时间范围内的数据
+//    - 可以通过时间戳参数(start_time/end_time)指定范围
+//    - 也可以通过日期字符串参数(start_date/end_date)指定范围，支持ISO格式或yyyy-MM-dd格式
+//
+// 使用自定义时间范围时，将不会更新同步状态，避免干扰增量同步进度
+import { getVariable, setVariable } from "npm:windmill-client@1";
 import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
 
 interface MongoConfig {
@@ -15,6 +23,7 @@ interface ClickHouseConfig {
   clickhouse_port: number;
   clickhouse_user: string;
   clickhouse_password: string;
+  clickhouse_database: string;
   clickhouse_url: string;
 }
 
@@ -32,6 +41,7 @@ interface TraceRecord {
   createTime: number;
 }
 
+// 添加 ShortRecord 接口定义
 interface ShortRecord {
   _id: ObjectId;
   slug: string;          // 短链接的slug部分
@@ -48,9 +58,161 @@ interface ShortRecord {
   projectId?: string;    // 项目ID
 }
 
-interface ClickHouseRow {
-  event_id: string;
-  event_attributes: string;
+interface SyncState {
+  last_sync_time: number;
+  records_synced: number;
+  last_sync_id?: string;
+}
+
+// 定义UTM参数接口
+interface UtmParams {
+  utm_source: string;
+  utm_medium: string;
+  utm_campaign: string;
+  utm_term: string;
+  utm_content: string;
+}
+
+// 同步状态键名
+const SYNC_STATE_KEY = "f/shorturl_analytics/mongo_sync_state";
+
+// 日期字符串转时间戳工具函数（接受ISO字符串或yyyy-MM-dd格式）
+function dateToTimestamp(dateStr: string): number {
+  try {
+    // 尝试直接解析完整的ISO日期字符串
+    const date = new Date(dateStr);
+    
+    // 检查是否为有效日期
+    if (isNaN(date.getTime())) {
+      // 尝试解析yyyy-MM-dd格式，默认设置为当天的00:00:00
+      const parts = dateStr.split('-');
+      if (parts.length === 3) {
+        const year = parseInt(parts[0], 10);
+        const month = parseInt(parts[1], 10) - 1; // 月份从0开始
+        const day = parseInt(parts[2], 10);
+        
+        const dateObj = new Date(year, month, day, 0, 0, 0);
+        return dateObj.getTime();
+      }
+      throw new Error(`无法解析日期字符串: ${dateStr}`);
+    }
+    
+    return date.getTime();
+  } catch (err) {
+    throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+// 从URL中提取UTM参数的函数，增强版
+function extractUtmParams(url: string, debug = false): UtmParams {
+  const defaultUtmParams: UtmParams = {
+    utm_source: "",
+    utm_medium: "",
+    utm_campaign: "",
+    utm_term: "",
+    utm_content: ""
+  };
+  
+  if (!url) return defaultUtmParams;
+  
+  if (debug) {
+    console.log(`[UTM提取] 原始URL: ${url}`);
+  }
+  
+  // 准备一个解析后的参数对象
+  const params: UtmParams = { ...defaultUtmParams };
+  
+  // 尝试多种方法提取UTM参数
+  
+  // 方法1: 使用URL对象解析
+  try {
+    // 先处理URL，确保是完整的URL格式
+    let normalizedUrl = url;
+    if (!url.match(/^https?:\/\//i)) {
+      normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
+    }
+    
+    const urlObj = new URL(normalizedUrl);
+    
+    // 读取URL参数
+    if (urlObj.searchParams.has('utm_source')) 
+      params.utm_source = urlObj.searchParams.get('utm_source') || "";
+    if (urlObj.searchParams.has('utm_medium')) 
+      params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
+    if (urlObj.searchParams.has('utm_campaign')) 
+      params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
+    if (urlObj.searchParams.has('utm_term')) 
+      params.utm_term = urlObj.searchParams.get('utm_term') || "";
+    if (urlObj.searchParams.has('utm_content')) 
+      params.utm_content = urlObj.searchParams.get('utm_content') || "";
+    
+    if (debug) {
+      console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
+    }
+    
+    // 如果至少找到一个UTM参数，则返回
+    if (params.utm_source || params.utm_medium || params.utm_campaign || 
+        params.utm_term || params.utm_content) {
+      return params;
+    }
+  } catch (_err) {
+    if (debug) {
+      console.log(`[UTM提取] URL对象解析失败，尝试正则表达式`);
+    }
+  }
+  
+  // 方法2: 使用正则表达式提取参数
+  // 使用正则表达式（最安全的方法，适用于任何格式）
+  const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
+  if (sourceMatch && sourceMatch[1]) {
+    try {
+      params.utm_source = decodeURIComponent(sourceMatch[1]);
+    } catch (_) {
+      params.utm_source = sourceMatch[1];
+    }
+  }
+  
+  const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
+  if (mediumMatch && mediumMatch[1]) {
+    try {
+      params.utm_medium = decodeURIComponent(mediumMatch[1]);
+    } catch (_) {
+      params.utm_medium = mediumMatch[1];
+    }
+  }
+  
+  const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
+  if (campaignMatch && campaignMatch[1]) {
+    try {
+      params.utm_campaign = decodeURIComponent(campaignMatch[1]);
+    } catch (_) {
+      params.utm_campaign = campaignMatch[1];
+    }
+  }
+  
+  const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
+  if (termMatch && termMatch[1]) {
+    try {
+      params.utm_term = decodeURIComponent(termMatch[1]);
+    } catch (_) {
+      params.utm_term = termMatch[1];
+    }
+  }
+  
+  const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
+  if (contentMatch && contentMatch[1]) {
+    try {
+      params.utm_content = decodeURIComponent(contentMatch[1]);
+    } catch (_) {
+      params.utm_content = contentMatch[1];
+    }
+  }
+  
+  if (debug) {
+    console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
+  }
+  
+  return params;
 }
 
 export async function main(
@@ -58,90 +220,246 @@ export async function main(
   max_records = 9999999,
   timeout_minutes = 60,
   skip_clickhouse_check = false,
-  force_insert = false
+  force_insert = true,
+  database_override = "shorturl_analytics", // 添加数据库名称参数，默认为shorturl_analytics
+  reset_sync_state = false, // 添加参数用于重置同步状态
+  debug_utm = false, // 添加参数控制UTM调试日志输出
+  start_time?: number, // 添加参数指定同步的开始时间戳，可选
+  end_time?: number, // 添加参数指定同步的结束时间戳，可选
+  use_custom_time_range = false, // 添加参数控制是否使用自定义时间范围
+  start_date?: string, // 添加开始日期字符串参数（ISO格式或yyyy-MM-dd格式）
+  end_date?: string // 添加结束日期字符串参数（ISO格式或yyyy-MM-dd格式）
 ) {
   const logWithTimestamp = (message: string) => {
     const now = new Date();
     console.log(`[${now.toISOString()}] ${message}`);
   };
 
-  logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
-  logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
+  logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
+  logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
+  
+  // 处理日期字符串参数，转换为时间戳
+  if (start_date) {
+    try {
+      start_time = dateToTimestamp(start_date);
+      logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${start_time}`);
+      use_custom_time_range = true;
+    } catch (err) {
+      logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  
+  if (end_date) {
+    try {
+      end_time = dateToTimestamp(end_date);
+      // 如果是日期格式，设置为当天结束时间 (23:59:59.999)
+      if (end_date.split('-').length === 3 && end_date.length <= 10) {
+        end_time += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
+        logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${end_time}`);
+      } else {
+        logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${end_time}`);
+      }
+      use_custom_time_range = true;
+    } catch (err) {
+      logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  
+  if (skip_clickhouse_check) {
+    logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
+  }
+  if (force_insert) {
+    logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
+  }
+  if (reset_sync_state) {
+    logWithTimestamp("⚠️ 警告: 已启用重置同步状态，将从头开始同步数据");
+  }
+  if (debug_utm) {
+    logWithTimestamp("已启用UTM参数调试日志");
+  }
+  if (use_custom_time_range) {
+    if (start_time) {
+      logWithTimestamp(`已启用自定义时间范围：开始时间 ${new Date(start_time).toISOString()}`);
+    }
+    if (end_time) {
+      logWithTimestamp(`已启用自定义时间范围：结束时间 ${new Date(end_time).toISOString()}`);
+    }
+  }
 
-  // Set timeout
+  // 设置超时
   const startTime = Date.now();
   const timeoutMs = timeout_minutes * 60 * 1000;
   
+  // 检查是否超时
   const checkTimeout = () => {
     if (Date.now() - startTime > timeoutMs) {
-      console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
+      console.log(`运行时间超过${timeout_minutes}分钟，暂停执行`);
       return true;
     }
     return false;
   };
 
-  // Get MongoDB and ClickHouse connection info
+  // 获取MongoDB和ClickHouse的连接信息
   let mongoConfig: MongoConfig;
   let clickhouseConfig: ClickHouseConfig;
   
   try {
     const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
-    mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
+    console.log("原始MongoDB配置:", JSON.stringify(rawMongoConfig));
+    
+    // 尝试解析配置，如果是字符串形式
+    if (typeof rawMongoConfig === "string") {
+      try {
+        mongoConfig = JSON.parse(rawMongoConfig);
+      } catch (e) {
+        console.error("MongoDB配置解析失败:", e);
+        throw e;
+      }
+    } else {
+      mongoConfig = rawMongoConfig as MongoConfig;
+    }
     
     const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
-    clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
+    console.log("原始ClickHouse配置:", JSON.stringify(rawClickhouseConfig));
+    
+    // 尝试解析配置，如果是字符串形式
+    if (typeof rawClickhouseConfig === "string") {
+      try {
+        clickhouseConfig = JSON.parse(rawClickhouseConfig);
+      } catch (e) {
+        console.error("ClickHouse配置解析失败:", e);
+        throw e;
+      }
+    } else {
+      clickhouseConfig = rawClickhouseConfig as ClickHouseConfig;
+    }
+    
+    // 检查并修复数据库配置
+    if (!clickhouseConfig.clickhouse_database || clickhouseConfig.clickhouse_database === "undefined") {
+      logWithTimestamp(`⚠️ 警告: 数据库名称未定义或为'undefined'，使用提供的默认值: ${database_override}`);
+      clickhouseConfig.clickhouse_database = database_override;
+    }
+    
+    console.log("MongoDB配置解析为:", JSON.stringify(mongoConfig));
+    console.log("ClickHouse配置解析为:", JSON.stringify({
+      ...clickhouseConfig,
+      clickhouse_password: "****" // 隐藏密码
+    }));
+    
+    logWithTimestamp(`将使用ClickHouse数据库: ${clickhouseConfig.clickhouse_database}`);
   } catch (error) {
-    console.error("Failed to get config:", error);
+    console.error("获取配置失败:", error);
     throw error;
   }
 
-  // Build MongoDB connection URL
+  // 获取上次同步状态
+  let lastSyncState: SyncState | null = null;
+  if (!reset_sync_state) {
+    try {
+      const rawSyncState = await getVariable(SYNC_STATE_KEY);
+      if (rawSyncState) {
+        if (typeof rawSyncState === "string") {
+          try {
+            lastSyncState = JSON.parse(rawSyncState);
+          } catch (e) {
+            logWithTimestamp(`解析上次同步状态失败: ${e}, 将从头开始同步`);
+          }
+        } else {
+          lastSyncState = rawSyncState as SyncState;
+        }
+      }
+    } catch (error) {
+      logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
+    }
+  }
+
+  if (lastSyncState) {
+    logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
+    if (lastSyncState.last_sync_id) {
+      logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
+    }
+  } else {
+    logWithTimestamp("没有找到上次同步状态，将从头开始同步");
+  }
+
+  // 构建MongoDB连接URL
   let mongoUrl = "mongodb://";
   if (mongoConfig.username && mongoConfig.password) {
     mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
   }
   mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
 
-  // Connect to MongoDB
+  console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
+
+  // 连接MongoDB
   const client = new MongoClient();
   try {
     await client.connect(mongoUrl);
-    console.log("MongoDB connected successfully");
+    console.log("MongoDB连接成功");
 
     const db = client.database(mongoConfig.db);
     const traceCollection = db.collection<TraceRecord>("trace");
+    // 添加对short集合的引用
     const shortCollection = db.collection<ShortRecord>("short");
     
-    // Build query conditions
+    // 构建查询条件，根据上次同步状态获取新记录
     const query: Record<string, unknown> = {
-      type: 1  // Only sync records with type 1
+      // 删除了 type: 1 的条件，将同步所有数据
     };
 
-    // Count total records
-    const totalRecords = await traceCollection.countDocuments(query);
-    console.log(`Found ${totalRecords} records to sync`);
+    // 根据时间范围参数构建查询条件
+    if (use_custom_time_range) {
+      // 使用自定义时间范围
+      const timeQuery: Record<string, number> = {};
+      
+      if (start_time) {
+        timeQuery.$gte = start_time;
+        logWithTimestamp(`将只同步createTime >= ${start_time} (${new Date(start_time).toISOString()}) 的记录`);
+      }
+      
+      if (end_time) {
+        timeQuery.$lte = end_time;
+        logWithTimestamp(`将只同步createTime <= ${end_time} (${new Date(end_time).toISOString()}) 的记录`);
+      }
+      
+      // 只有当至少指定了一个时间限制时才添加时间查询条件
+      if (Object.keys(timeQuery).length > 0) {
+        query.createTime = timeQuery;
+      }
+    } 
+    // 如果不使用自定义时间范围，且有上次同步状态，则只获取更新的记录
+    else if (lastSyncState && lastSyncState.last_sync_time) {
+      // 使用上次同步时间作为过滤条件
+      query.createTime = { $gt: lastSyncState.last_sync_time };
+      logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
+    }
 
+    // 计算总记录数
+    const totalRecords = await traceCollection.countDocuments(query);
+    console.log(`找到 ${totalRecords} 条新记录需要同步`);
+
+    // 限制此次处理的记录数量
     const recordsToProcess = Math.min(totalRecords, max_records);
-    console.log(`Will process ${recordsToProcess} records`);
+    console.log(`本次将处理 ${recordsToProcess} 条记录`);
 
     if (totalRecords === 0) {
-      console.log("No records to sync, task completed");
+      console.log("没有新记录需要同步，任务完成");
       return {
         success: true,
         records_synced: 0,
-        message: "No records to sync"
+        message: "没有新记录需要同步"
       };
     }
 
-    // Check ClickHouse connection
+    // 检查ClickHouse连接状态
     const checkClickHouseConnection = async (): Promise<boolean> => {
       if (skip_clickhouse_check) {
-        logWithTimestamp("Skipping ClickHouse connection check");
+        logWithTimestamp("已启用跳过ClickHouse检查，不测试连接");
         return true;
       }
       
       try {
-        logWithTimestamp("Testing ClickHouse connection...");
+        logWithTimestamp("测试ClickHouse连接...");
         const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
         const response = await fetch(clickhouseUrl, {
           method: "POST",
@@ -149,184 +467,197 @@ export async function main(
             "Content-Type": "application/x-www-form-urlencoded",
             "Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
           },
-          body: "SELECT 1",
+          body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
+          // 设置5秒超时
           signal: AbortSignal.timeout(5000)
         });
         
         if (response.ok) {
-          logWithTimestamp("ClickHouse connection test successful");
+          logWithTimestamp("ClickHouse连接测试成功");
           return true;
         } else {
           const errorText = await response.text();
-          logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
+          logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
           return false;
         }
       } catch (err) {
-        logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
+        const error = err as Error;
+        logWithTimestamp(`ClickHouse连接测试失败: ${error.message}`);
         return false;
       }
     };
 
-    // Check if records exist in ClickHouse
-    const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
-      if (records.length === 0) return [];
-      
-      if (skip_clickhouse_check || force_insert) {
-        logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
-        return records;
-      }
-      
-      try {
-        const recordIds = records.map(record => record._id.toString());
-        
-        const query = `
-          SELECT event_id 
-          FROM shorturl_analytics.events 
-          WHERE event_attributes LIKE '%"mongo_id":"%' 
-          AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
-          FORMAT JSON
-        `;
-        
-        const response = await fetch(clickhouseConfig.clickhouse_url, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/x-www-form-urlencoded",
-            "Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
-          },
-          body: query,
-          signal: AbortSignal.timeout(10000)
-        });
-        
-        if (!response.ok) {
-          const errorText = await response.text();
-          throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
-        }
-        
-        const result = await response.json();
-        const existingIds = new Set(result.data.map((row: ClickHouseRow) => {
-          const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
-          return matches ? matches[1] : null;
-        }).filter(Boolean));
-        
-        return records.filter(record => !existingIds.has(record._id.toString()));
-      } catch (err) {
-        logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
-        return skip_clickhouse_check ? records : [];
-      }
-    };
+    // 在处理记录前先检查ClickHouse连接
+    const clickhouseConnected = await checkClickHouseConnection();
+    if (!clickhouseConnected && !skip_clickhouse_check) {
+      logWithTimestamp("⚠️ ClickHouse连接测试失败，请启用skip_clickhouse_check=true参数来跳过连接检查");
+      throw new Error("ClickHouse连接失败，无法继续同步");
+    }
 
-    // Process records function
+    // 处理记录的函数
     const processRecords = async (records: TraceRecord[]) => {
       if (records.length === 0) return 0;
       
-      const newRecords = await checkExistingRecords(records);
-      if (newRecords.length === 0) {
-        logWithTimestamp("All records already exist, skipping");
-        return 0;
-      }
+      logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
       
-      // Get link information for all records
+      // 强制使用所有记录，不检查重复
+      const newRecords = records;
+      
+      logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
+      
+      // 获取链接信息 - 新增代码
       const slugIds = newRecords.map(record => record.slugId);
+      logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
       const shortLinks = await shortCollection.find({
         _id: { $in: slugIds }
       }).toArray();
-
-      // Create a map for quick lookup
-      const shortLinksMap = new Map(shortLinks.map(link => [link._id.toString(), link]));
       
-      // Prepare ClickHouse insert data
+      // 创建映射用于快速查找 - 新增代码
+      const shortLinksMap = new Map(shortLinks.map((link: ShortRecord) => [link._id.toString(), link]));
+      logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息，${newRecords.length - shortLinks.length} 条数据将使用占位符`);
+      
+      // 准备ClickHouse插入数据
       const clickhouseData = newRecords.map(record => {
-        const shortLink = shortLinksMap.get(record.slugId.toString());
+        const eventTime = new Date(record.createTime);
         
-        // 将毫秒时间戳转换为 DateTime64(3) 格式
-        const formatDateTime = (timestamp: number) => {
-          return new Date(timestamp).toISOString().replace('T', ' ').replace('Z', '');
+        // 获取对应的短链接信息 - 新增代码
+        const shortLink = shortLinksMap.get(record.slugId.toString()) as ShortRecord | undefined;
+        
+        // 提取URL中的UTM参数 - 增加调试日志
+        if (debug_utm && record.url) {
+          logWithTimestamp(`======== UTM参数调试 ========`);
+          logWithTimestamp(`记录ID: ${record._id.toString()}`);
+          logWithTimestamp(`原始URL: ${record.url}`);
+        }
+        
+        const utmParams = extractUtmParams(record.url || "", debug_utm);
+        
+        if (debug_utm) {
+          logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
+          logWithTimestamp(`===========================`);
+        }
+        
+        // 保存提取的UTM参数和URL到event_attributes
+        const eventAttributes = {
+          mongo_id: record._id.toString(),
+          url: record.url || "",
+          ...(record.url ? { raw_url: record.url } : {})
         };
         
+        // 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
         return {
-          // Event base information
-          event_id: record._id.toString(),
-          event_time: formatDateTime(record.createTime),
-          event_type: "click",
-          event_attributes: JSON.stringify({
-            original_type: record.type
-          }),
-          
-          // Link information from short collection
+          // UUID将由ClickHouse自动生成 (event_id)
+          event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
+          event_type: "click", // 将所有event_type都设置为click
+          event_attributes: JSON.stringify(eventAttributes),
           link_id: record.slugId.toString(),
-          link_slug: shortLink?.slug || "",
+          link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
           link_label: record.label || "",
-          link_title: "",
-          link_original_url: shortLink?.origin || "",
-          link_attributes: JSON.stringify({
-            domain: shortLink?.domain || null
-          }),
-          link_created_at: shortLink?.createTime ? formatDateTime(shortLink.createTime) : formatDateTime(record.createTime),
-          link_expires_at: shortLink?.expiresAt ? formatDateTime(shortLink.expiresAt) : null,
-          link_tags: "[]",  // Empty array as default
-          
-          // User information
-          user_id: shortLink?.user || "",
-          user_name: "",
+          link_title: shortLink?.title || "unknown_title", // 使用占位符
+          link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
+          link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
+          link_created_at: shortLink?.createTime 
+            ? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
+            : eventTime.toISOString().replace('T', ' ').replace('Z', ''),
+          link_expires_at: shortLink?.expiresAt 
+            ? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '') 
+            : null,
+          link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
+          user_id: "3680f452-e404-4339-a3d2-2a8e1ff92102", // 使用占位符
+          user_name: "unknown_user", // 使用占位符
           user_email: "",
           user_attributes: "{}",
-          
-          // Team information
-          team_id: shortLink?.teamId || "",
-          team_name: "",
+          team_id: "e02251eb-eb98-47c8-b5dd-4f6e4fdb1f49", // 使用占位符
+          team_name: "", // 使用占位符
           team_attributes: "{}",
-          
-          // Project information
-          project_id: shortLink?.projectId || "",
-          project_name: "",
+          project_id: "34cdb8b9-8b8e-4033-876a-0632002ef1f9", // 使用占位符
+          project_name: "", // 使用占位符
           project_attributes: "{}",
-          
-          // QR code information
           qr_code_id: "",
           qr_code_name: "",
           qr_code_attributes: "{}",
-          
-          // Visitor information
-          visitor_id: "",  // Empty string as default
-          session_id: `${record.slugId.toString()}-${record.createTime}`,
-          ip_address: record.ip || "",
-          country: "",
+          visitor_id: record._id.toString(),
+          session_id: record._id.toString() + "-" + record.createTime,
+          ip_address: record.ip || "0.0.0.0", // 使用占位符
+          country: "", 
           city: "",
-          device_type: record.platform || "",
-          browser: record.browser || "",
-          os: record.platformOS || "",
-          user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
-          
-          // Source information
+          device_type: record.platform || "unknown",
+          browser: record.browser || "unknown", // 使用占位符
+          os: record.platformOS || "unknown", // 使用占位符
+          user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
           referrer: record.url || "",
-          utm_source: "",
-          utm_medium: "",
-          utm_campaign: "",
-          
-          // Interaction information
+          utm_source: utmParams.utm_source || "",
+          utm_medium: utmParams.utm_medium || "",
+          utm_campaign: utmParams.utm_campaign || "",
+          utm_term: utmParams.utm_term || "",
+          utm_content: utmParams.utm_content || "",
           time_spent_sec: 0,
           is_bounce: true,
           is_qr_scan: false,
           conversion_type: "visit",
-          conversion_value: 0
+          conversion_value: 0,
+          req_full_path: record.url || ""
         };
       });
 
-      // Generate ClickHouse insert SQL
-      const rows = clickhouseData.map(row => {
-        // 只需要处理JSON字符串的转义
-        const formattedRow = {
-          ...row,
-          event_attributes: row.event_attributes.replace(/\\/g, '\\\\'),
-          link_attributes: row.link_attributes.replace(/\\/g, '\\\\')
-        };
-        return JSON.stringify(formattedRow);
-      }).join('\n');
+      // 生成ClickHouse插入SQL
+      const insertSQL = `
+        INSERT INTO ${clickhouseConfig.clickhouse_database}.events 
+        (event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title, 
+         link_original_url, link_attributes, link_created_at, link_expires_at, link_tags, 
+         user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes, 
+         project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes, 
+         visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent, 
+         referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec, 
+         is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
+        VALUES ${clickhouseData.map(record => {
+          // 增强版安全替换函数，处理所有特殊字符
+          const safeReplace = (val: unknown): string => {
+            // 确保值是字符串，如果是null或undefined则使用空字符串
+            const str = val === null || val === undefined ? "" : String(val);
+            
+            // 转义所有可能导致SQL注入或格式错误的字符
+            // 1. 先替换所有反斜杠
+            // 2. 再替换单引号
+            // 3. 替换所有控制字符和特殊字符
+            return str
+              .replace(/\\/g, "\\\\")  // 转义反斜杠
+              .replace(/'/g, "\\'")    // 转义单引号
+              .replace(/\r/g, "\\r")   // 转义回车
+              .replace(/\n/g, "\\n")   // 转义换行
+              .replace(/\t/g, "\\t")   // 转义制表符
+              .replace(/\0/g, "")      // 移除空字符
+              .replace(/[\x00-\x1F\x7F-\x9F]/g, ""); // 移除所有控制字符
+          };
+          
+          return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}', 
+            '${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}', 
+            '${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}', 
+            ${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}', 
+            '${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}', 
+            '${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}', 
+            '${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}', 
+            '${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}', 
+            '${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}', 
+            '${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}', 
+            '${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}', 
+            '${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}', 
+            '${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}', 
+            '${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan}, 
+            '${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
+        }).join(", ")}
+      `;
 
-      const insertSQL = `INSERT INTO shorturl_analytics.events FORMAT JSONEachRow\n${rows}`;
+      if (insertSQL.length === 0) {
+        console.log("没有新记录需要插入");
+        return 0;
+      }
 
+      // 发送请求到ClickHouse，添加20秒超时
+      const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
       try {
-        const response = await fetch(clickhouseConfig.clickhouse_url, {
+        logWithTimestamp("发送插入请求到ClickHouse...");
+        const response = await fetch(clickhouseUrl, {
           method: "POST",
           headers: {
             "Content-Type": "application/x-www-form-urlencoded",
@@ -338,35 +669,34 @@ export async function main(
 
         if (!response.ok) {
           const errorText = await response.text();
-          throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
+          throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
         }
 
-        logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
+        logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
         return newRecords.length;
       } catch (err) {
-        logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
-        throw err;
+        const error = err as Error;
+        logWithTimestamp(`向ClickHouse插入数据失败: ${error.message}`);
+        throw error;
       }
     };
 
-    // Check ClickHouse connection before processing
-    const clickhouseConnected = await checkClickHouseConnection();
-    if (!clickhouseConnected && !skip_clickhouse_check) {
-      throw new Error("ClickHouse connection failed, cannot continue sync");
-    }
-
-    // Process records in batches
+    // 批量处理记录
     let processedRecords = 0;
     let totalBatchRecords = 0;
+    let lastSyncTime = 0;
 
     for (let page = 0; processedRecords < recordsToProcess; page++) {
+      // 检查超时
       if (checkTimeout()) {
-        logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
+        logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录，因超时暂停执行`);
         break;
       }
 
-      logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
+      // 每批次都输出进度
+      logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
       
+      logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
       const records = await traceCollection.find(
         query,
         {
@@ -378,32 +708,86 @@ export async function main(
       ).toArray();
 
       if (records.length === 0) {
-        logWithTimestamp("No more records found, sync complete");
+        logWithTimestamp("没有找到更多数据，同步结束");
         break;
       }
+      
+      // 找到数据，开始处理
+      logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
+      // 输出当前批次的部分数据信息
+      if (records.length > 0) {
+        logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
+        if (records.length > 1) {
+          logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
+        }
+        
+        // 如果开启了调试，输出一些URL样本
+        if (debug_utm) {
+          const sampleSize = Math.min(5, records.length);
+          logWithTimestamp(`URL样本 (前${sampleSize}条):`);
+          for (let i = 0; i < sampleSize; i++) {
+            if (records[i].url) {
+              logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
+            }
+          }
+        }
+      }
 
       const batchSize = await processRecords(records);
       processedRecords += records.length;
       totalBatchRecords += batchSize;
 
-      logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
+      // 更新最后处理的记录时间和ID
+      if (records.length > 0) {
+        const lastRecord = records[records.length - 1];
+        lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
+      }
+
+      logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
+    }
+
+    // 更新同步状态
+    if (processedRecords > 0 && lastSyncTime > 0) {
+      // 只在非自定义时间范围模式下更新同步状态
+      if (!use_custom_time_range) {
+        // 创建新的同步状态，简化对象结构
+        const newSyncState: SyncState = {
+          last_sync_time: lastSyncTime,
+          records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords, // 使用处理的总记录数，而不是实际插入数
+        };
+        
+        try {
+          // 保存同步状态
+          await setVariable(SYNC_STATE_KEY, newSyncState);
+          logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
+        } catch (err) {
+          const error = err as Error;
+          logWithTimestamp(`更新同步状态失败: ${error.message}，将继续执行`);
+          // 不抛出错误，继续执行
+        }
+      } else {
+        logWithTimestamp("使用自定义时间范围模式，不更新全局同步状态");
+      }
     }
 
     return {
       success: true,
       records_processed: processedRecords,
       records_synced: totalBatchRecords,
-      message: "Data sync completed"
+      last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
+      message: use_custom_time_range ? "自定义时间范围数据同步完成" : "数据同步完成",
+      custom_time_range_used: use_custom_time_range
     };
   } catch (err) {
-    console.error("Error during sync:", err);
+    console.error("同步过程中发生错误:", err);
     return {
       success: false,
       error: err instanceof Error ? err.message : String(err),
       stack: err instanceof Error ? err.stack : undefined
     };
   } finally {
+    // 关闭MongoDB连接
     await client.close();
-    console.log("MongoDB connection closed");
+    console.log("MongoDB连接已关闭");
   }
-} 
+}

windmill/sync_shorturl_schema_to_clickhouse.ts

@@ -0,0 +1,527 @@
+// 文件名: sync_shorturl_schema_to_clickhouse.ts
+// 描述: 此脚本用于同步PostgreSQL中的short_url.shorturl表数据到ClickHouse
+// 创建日期: 2023-11-21
+
+import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
+import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
+
+// 同步状态接口
+interface SyncState {
+  last_sync_time: string;   // 上次同步的结束时间
+  records_synced: number;   // 累计同步的记录数
+  last_run: string;         // 上次运行的时间
+}
+
+// 同步状态键名
+const SYNC_STATE_KEY = "f/shorturl_analytics/shorturl_to_clickhouse_state";
+
+// PostgreSQL配置接口
+interface PgConfig {
+  host: string;
+  port: number;
+  user: string;
+  password: string;
+  dbname?: string;
+  [key: string]: unknown;
+}
+
+// ClickHouse配置接口
+interface ChConfig {
+  clickhouse_host: string;
+  clickhouse_port: number;
+  clickhouse_user: string;
+  clickhouse_password: string;
+  clickhouse_url?: string;
+}
+
+// Shorturl数据接口
+interface ShortUrlData {
+  id: string;
+  slug: string;
+  origin: string; // 对应ClickHouse中的original_url
+  title?: string;
+  description?: string;
+  created_at?: string;
+  updated_at?: string;
+  deleted_at?: string;
+  expires_at?: string; // 注意这里已更正为expires_at
+  domain?: string; // 添加domain字段
+}
+
+/**
+ * 同步PostgreSQL short_url.shorturl表数据到ClickHouse
+ */
+export async function main(
+  /** 是否为测试模式（不执行实际更新） */
+  dry_run = false,
+  /** 是否显示详细日志 */
+  verbose = false,
+  /** 是否重置同步状态（从头开始同步） */
+  reset_sync_state = false,
+  /** 如果没有同步状态，往前查询多少小时的数据（默认1小时） */
+  default_hours_back = 1
+) {
+  // 初始化日志函数
+  const log = (message: string, isVerbose = false) => {
+    if (!isVerbose || verbose) {
+      console.log(message);
+    }
+  };
+
+  // 获取同步状态
+  let syncState: SyncState | null = null;
+  if (!reset_sync_state) {
+    try {
+      log("获取同步状态...", true);
+      const rawState = await getVariable(SYNC_STATE_KEY);
+      if (rawState) {
+        if (typeof rawState === "string") {
+          syncState = JSON.parse(rawState);
+        } else {
+          syncState = rawState as SyncState;
+        }
+        log(`找到上次同步状态: 最后同步时间 ${syncState.last_sync_time}, 已同步记录数 ${syncState.records_synced}`, true);
+      }
+    } catch (error) {
+      log(`获取同步状态失败: ${error}, 将使用默认设置`, true);
+    }
+  } else {
+    log("重置同步状态，从头开始同步", true);
+  }
+
+  // 设置时间范围
+  const oneHourAgo = new Date(Date.now() - default_hours_back * 60 * 60 * 1000).toISOString();
+  // 如果有同步状态，使用上次同步时间作为开始时间；否则使用默认时间
+  const start_time = syncState ? syncState.last_sync_time : oneHourAgo;
+  const end_time = new Date().toISOString();
+
+  log(`开始同步shorturl表数据: ${start_time} 至 ${end_time}`);
+
+  let pgPool: Pool | null = null;
+
+  try {
+    // 1. 获取数据库配置
+    log("获取PostgreSQL数据库配置...", true);
+    const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
+    
+    // 2. 创建PostgreSQL连接池
+    pgPool = new Pool({
+      hostname: pgConfig.host,
+      port: pgConfig.port,
+      user: pgConfig.user,
+      password: pgConfig.password,
+      database: pgConfig.dbname || 'postgres'
+    }, 3);
+    
+    // 3. 获取需要更新的数据
+    const shorturlData = await getShortUrlData(pgPool, start_time, end_time, log);
+    log(`成功获取 ${shorturlData.length} 条shorturl数据`);
+    
+    if (shorturlData.length === 0) {
+      // 更新同步状态，即使没有新数据
+      if (!dry_run) {
+        await updateSyncState(end_time, syncState ? syncState.records_synced : 0, log);
+      }
+      return { success: true, message: "没有找到需要更新的数据", updated: 0 };
+    }
+    
+    // 4. 获取ClickHouse配置
+    const chConfig = await getClickHouseConfig();
+    
+    // 5. 执行更新
+    if (!dry_run) {
+      const shorturlUpdated = await updateClickHouseShortUrl(shorturlData, chConfig, log);
+      
+      // 更新同步状态
+      const totalSynced = (syncState ? syncState.records_synced : 0) + shorturlUpdated;
+      await updateSyncState(end_time, totalSynced, log);
+      
+      return { 
+        success: true, 
+        message: "shorturl表数据同步完成", 
+        shorturl_updated: shorturlUpdated,
+        total_synced: totalSynced,
+        sync_state: {
+          last_sync_time: end_time,
+          records_synced: totalSynced
+        }
+      };
+    } else {
+      log("测试模式: 不执行实际更新");
+      return { 
+        success: true, 
+        dry_run: true, 
+        shorturl_count: shorturlData.length,
+        shorturl_sample: shorturlData.slice(0, 1)
+      };
+    }
+  } catch (error) {
+    const errorMessage = `同步过程中发生错误: ${(error as Error).message}`;
+    log(errorMessage);
+    if ((error as Error).stack) {
+      log(`错误堆栈: ${(error as Error).stack}`, true);
+    }
+    return { success: false, message: errorMessage };
+  } finally {
+    if (pgPool) {
+      await pgPool.end();
+      log("PostgreSQL连接池已关闭", true);
+    }
+  }
+}
+
+/**
+ * 更新同步状态
+ */
+async function updateSyncState(lastSyncTime: string, recordsSynced: number, log: (message: string, isVerbose?: boolean) => void): Promise<void> {
+  try {
+    const newState: SyncState = {
+      last_sync_time: lastSyncTime,
+      records_synced: recordsSynced,
+      last_run: new Date().toISOString()
+    };
+    
+    await setVariable(SYNC_STATE_KEY, newState);
+    log(`同步状态已更新: 最后同步时间 ${lastSyncTime}, 累计同步记录数 ${recordsSynced}`, true);
+  } catch (error) {
+    log(`更新同步状态失败: ${error}`, true);
+    // 继续执行，不中断同步过程
+  }
+}
+
+/**
+ * 从PostgreSQL获取shorturl数据
+ */
+async function getShortUrlData(
+  pgPool: Pool,
+  startTime: string,
+  endTime: string,
+  log: (message: string, isVerbose?: boolean) => void
+): Promise<ShortUrlData[]> {
+  const client = await pgPool.connect();
+  
+  try {
+    log(`获取shorturl数据 (${startTime} 至 ${endTime})`, true);
+    
+    const query = `
+      SELECT 
+        id, 
+        slug, 
+        origin,
+        title,
+        description,
+        domain,
+        created_at,
+        updated_at,
+        deleted_at,
+        expired_at as expires_at
+      FROM 
+        short_url.shorturl
+      WHERE 
+        (created_at >= $1 AND created_at <= $2)
+        OR (updated_at >= $1 AND updated_at <= $2)
+    `;
+    
+    const result = await client.queryObject(query, [startTime, endTime]);
+    return result.rows as ShortUrlData[];
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * 格式化日期时间为ClickHouse可接受的格式
+ */
+function formatDateTime(dateStr: string | null | undefined): string {
+  if (!dateStr) return 'NULL';
+  
+  try {
+    // 将日期字符串转换为ISO格式
+    const date = new Date(dateStr);
+    if (isNaN(date.getTime())) {
+      return 'NULL';
+    }
+    
+    // 返回ISO格式的日期字符串，ClickHouse可以解析
+    return `parseDateTimeBestEffort('${date.toISOString()}')`;
+  } catch (error) {
+    console.error(`日期格式化错误: ${error}`);
+    return 'NULL';
+  }
+}
+
+/**
+ * 格式化进度显示
+ */
+function formatProgress(current: number, total: number): string {
+  const percent = Math.round((current / total) * 100);
+  const progressBar = '[' + '='.repeat(Math.floor(percent / 5)) + ' '.repeat(20 - Math.floor(percent / 5)) + ']';
+  return `${progressBar} ${percent}% (${current}/${total})`;
+}
+
+/**
+ * 更新ClickHouse中的shorturl表数据
+ */
+async function updateClickHouseShortUrl(
+  shorturls: ShortUrlData[],
+  chConfig: ChConfig,
+  log: (message: string, isVerbose?: boolean) => void
+): Promise<number> {
+  if (shorturls.length === 0) {
+    log('没有找到shorturl数据，跳过shorturl表更新');
+    return 0;
+  }
+  
+  log(`准备更新 ${shorturls.length} 条shorturl数据`);
+  
+  // 检查ClickHouse中是否存在shorturl表
+  const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.shorturl');
+  
+  if (!tableExists) {
+    log('ClickHouse中未找到shorturl表，请先创建表');
+    return 0;
+  }
+  
+  let updatedCount = 0;
+  const startTime = Date.now();
+  
+  // 使用批量插入更高效
+  const batchSize = 50; // 降低批次大小，使查询更稳定
+  for (let i = 0; i < shorturls.length; i += batchSize) {
+    const batch = shorturls.slice(i, i + batchSize);
+    let successCount = 0;
+    
+    // 显示批处理进度信息
+    const batchNumber = Math.floor(i / batchSize) + 1;
+    const totalBatches = Math.ceil(shorturls.length / batchSize);
+    log(`处理批次 ${batchNumber}/${totalBatches}: ${formatProgress(i, shorturls.length)}`);
+    
+    // 对每条记录使用单独的INSERT ... SELECT ... WHERE NOT EXISTS语句
+    for (let j = 0; j < batch.length; j++) {
+      const shorturl = batch[j];
+      // 显示记录处理细节进度
+      const overallProgress = i + j + 1;
+      if (overallProgress % 10 === 0 || overallProgress === shorturls.length) {
+        // 每10条记录或最后一条记录显示一次进度
+        const elapsedSeconds = (Date.now() - startTime) / 1000;
+        const recordsPerSecond = overallProgress / elapsedSeconds;
+        const remainingRecords = shorturls.length - overallProgress;
+        const estimatedSecondsRemaining = remainingRecords / recordsPerSecond;
+        
+        log(`总进度: ${formatProgress(overallProgress, shorturls.length)} - 速率: ${recordsPerSecond.toFixed(1)}条/秒 - 预计剩余时间: ${formatTime(estimatedSecondsRemaining)}`);
+      }
+      
+      try {
+        const insertQuery = `
+          INSERT INTO shorturl_analytics.shorturl
+          SELECT
+            '${escapeString(shorturl.id)}' AS id,
+            '${escapeString(shorturl.id)}' AS external_id,
+            'shorturl' AS type,
+            '${escapeString(shorturl.slug)}' AS slug,
+            '${escapeString(shorturl.origin)}' AS original_url,
+            ${shorturl.title ? `'${escapeString(shorturl.title)}'` : 'NULL'} AS title,
+            ${shorturl.description ? `'${escapeString(shorturl.description)}'` : 'NULL'} AS description,
+            '{}' AS attributes,
+            1 AS schema_version,
+            '' AS creator_id,
+            '' AS creator_email,
+            '' AS creator_name,
+            ${formatDateTime(shorturl.created_at)} AS created_at,
+            ${formatDateTime(shorturl.updated_at)} AS updated_at,
+            ${formatDateTime(shorturl.deleted_at)} AS deleted_at,
+            '[]' AS projects,
+            '[]' AS teams,
+            '[]' AS tags,
+            '[]' AS qr_codes,
+            '[]' AS channels,
+            '[]' AS favorites,
+            ${formatDateTime(shorturl.expires_at)} AS expires_at,
+            0 AS click_count,
+            0 AS unique_visitors,
+            ${shorturl.domain ? `'${escapeString(shorturl.domain)}'` : 'NULL'} AS domain
+          WHERE NOT EXISTS (
+            SELECT 1 FROM shorturl_analytics.shorturl WHERE id = '${escapeString(shorturl.id)}'
+          )
+        `;
+        
+        await executeClickHouseQuery(chConfig, insertQuery);
+        successCount++;
+        log(`成功处理shorturl: ${shorturl.id}`, true);
+      } catch (error) {
+        log(`处理shorturl ${shorturl.id} 失败: ${(error as Error).message}`);
+        
+        // 尝试使用简单插入作为备选方案
+        try {
+          log(`尝试替代方法更新: ${shorturl.id}`, true);
+          
+          // 先检查记录是否存在
+          const checkQuery = `SELECT count() FROM shorturl_analytics.shorturl WHERE id = '${escapeString(shorturl.id)}'`;
+          const existsResult = await executeClickHouseQuery(chConfig, checkQuery);
+          const exists = parseInt(existsResult.trim()) > 0;
+          
+          if (!exists) {
+            const fallbackQuery = `
+              INSERT INTO shorturl_analytics.shorturl (
+                id, external_id, type, slug, original_url, 
+                title, description, attributes, schema_version, 
+                creator_id, creator_email, creator_name, 
+                created_at, updated_at, deleted_at,
+                projects, teams, tags, qr_codes, channels, favorites,
+                expires_at, click_count, unique_visitors, domain
+              ) VALUES (
+                '${escapeString(shorturl.id)}',
+                '${escapeString(shorturl.id)}',
+                'shorturl',
+                '${escapeString(shorturl.slug)}',
+                '${escapeString(shorturl.origin)}',
+                ${shorturl.title ? `'${escapeString(shorturl.title)}'` : 'NULL'},
+                ${shorturl.description ? `'${escapeString(shorturl.description)}'` : 'NULL'},
+                '{}',
+                1,
+                '',
+                '',
+                '',
+                ${formatDateTime(shorturl.created_at)},
+                ${formatDateTime(shorturl.updated_at)},
+                ${formatDateTime(shorturl.deleted_at)},
+                '[]',
+                '[]',
+                '[]',
+                '[]',
+                '[]',
+                '[]',
+                ${formatDateTime(shorturl.expires_at)},
+                0,
+                0,
+                ${shorturl.domain ? `'${escapeString(shorturl.domain)}'` : 'NULL'}
+              )
+            `;
+            
+            await executeClickHouseQuery(chConfig, fallbackQuery);
+            successCount++;
+            log(`备选方式插入成功: ${shorturl.id}`, true);
+          } else {
+            log(`记录已存在，跳过: ${shorturl.id}`, true);
+          }
+        } catch (fallbackError) {
+          log(`备选方式失败 ${shorturl.id}: ${(fallbackError as Error).message}`);
+        }
+      }
+    }
+    
+    updatedCount += successCount;
+    log(`批次 ${batchNumber}/${totalBatches} 完成: ${successCount}/${batch.length} 条成功 (总计: ${updatedCount}/${shorturls.length})`);
+  }
+  
+  const totalTime = (Date.now() - startTime) / 1000;
+  log(`同步完成! 总计处理: ${updatedCount}/${shorturls.length} 条记录, 耗时: ${formatTime(totalTime)}, 平均速率: ${(updatedCount / totalTime).toFixed(1)}条/秒`);
+  
+  return updatedCount;
+}
+
+/**
+ * 获取ClickHouse配置
+ */
+async function getClickHouseConfig(): Promise<ChConfig> {
+  try {
+    const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
+    
+    // 确保配置不为空
+    if (!chConfigJson) {
+      throw new Error("未找到ClickHouse配置");
+    }
+    
+    // 解析JSON字符串为对象
+    let chConfig: ChConfig;
+    if (typeof chConfigJson === 'string') {
+      try {
+        chConfig = JSON.parse(chConfigJson);
+      } catch {
+        throw new Error("ClickHouse配置不是有效的JSON");
+      }
+    } else {
+      chConfig = chConfigJson as ChConfig;
+    }
+    
+    // 验证并构建URL
+    if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
+      chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
+    }
+    
+    if (!chConfig.clickhouse_url) {
+      throw new Error("ClickHouse配置缺少URL");
+    }
+    
+    return chConfig;
+  } catch (error) {
+    throw new Error(`获取ClickHouse配置失败: ${(error as Error).message}`);
+  }
+}
+
+/**
+ * 检查ClickHouse中是否存在指定表
+ */
+async function checkClickHouseTable(chConfig: ChConfig, tableName: string): Promise<boolean> {
+  try {
+    const query = `EXISTS TABLE ${tableName}`;
+    const result = await executeClickHouseQuery(chConfig, query);
+    return result.trim() === '1';
+  } catch (error) {
+    console.error(`检查表 ${tableName} 失败:`, error);
+    return false;
+  }
+}
+
+/**
+ * 执行ClickHouse查询
+ */
+async function executeClickHouseQuery(chConfig: ChConfig, query: string): Promise<string> {
+  // 确保URL有效
+  if (!chConfig.clickhouse_url) {
+    throw new Error("无效的ClickHouse URL: 未定义");
+  }
+  
+  // 执行HTTP请求
+  try {
+    const response = await fetch(chConfig.clickhouse_url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
+      },
+      body: query,
+    });
+    
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`ClickHouse查询失败 (${response.status}): ${errorText}`);
+    }
+    
+    return await response.text();
+  } catch (error) {
+    throw new Error(`执行ClickHouse查询失败: ${(error as Error).message}`);
+  }
+}
+
+/**
+ * 转义字符串，避免SQL注入
+ */
+function escapeString(str: string): string {
+  if (!str) return '';
+  return str.replace(/'/g, "''");
+}
+
+/**
+ * 格式化时间（秒）为可读格式
+ */
+function formatTime(seconds: number): string {
+  const mins = Math.floor(seconds / 60);
+  const secs = Math.floor(seconds % 60);
+  
+  if (mins === 0) {
+    return `${secs}秒`;
+  } else {
+    return `${mins}分${secs}秒`;
+  }
+} 

windmill/sync_shorturl_to_clickhouse.ts

@@ -4,7 +4,7 @@
 // 描述: 此脚本从PostgreSQL数据库获取所有shorturl类型的资源及其关联数据，并同步到ClickHouse
 
 import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
-import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
+import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
 
 // 资源属性接口
 interface ResourceAttributes {
@@ -37,6 +37,15 @@ interface PgConfig {
   [key: string]: unknown;
 }
 
+// 上次同步状态接口
+interface SyncState {
+  lastSyncTime: string;
+  lastRunTime: string;
+}
+
+// 状态变量名称
+const STATE_VARIABLE_PATH = "f/shorturl_analytics/shorturl_sync_state";
+
 // Windmill函数定义
 export async function main(
   /** PostgreSQL和ClickHouse同步脚本 */
@@ -47,9 +56,11 @@ export async function main(
     includeDeleted?: boolean;
     /** 是否执行实际写入操作 */
     dryRun?: boolean;
-    /** 开始时间（ISO格式）*/
+    /** 是否强制全量同步 */
+    forceFullSync?: boolean;
+    /** 手动指定开始时间（ISO格式）- 会覆盖自动增量设置 */
     startTime?: string;
-    /** 结束时间（ISO格式）*/
+    /** 手动指定结束时间（ISO格式）*/
     endTime?: string;
   }
 ) {
@@ -57,8 +68,41 @@ export async function main(
   const limit = params.limit || 500;
   const includeDeleted = params.includeDeleted || false;
   const dryRun = params.dryRun || false;
-  const startTime = params.startTime ? new Date(params.startTime) : undefined;
-  const endTime = params.endTime ? new Date(params.endTime) : undefined;
+  const forceFullSync = params.forceFullSync || false;
+  
+  // 获取当前时间作为本次运行时间
+  const currentRunTime = new Date().toISOString();
+  
+  // 初始化同步状态
+  let syncState: SyncState;
+  let startTime: Date | undefined;
+  const endTime: Date | undefined = params.endTime ? new Date(params.endTime) : new Date();
+  
+  // 如果强制全量同步或手动指定了开始时间，则使用指定的开始时间
+  if (forceFullSync || params.startTime) {
+    startTime = params.startTime ? new Date(params.startTime) : undefined;
+    console.log(`使用${params.startTime ? '手动指定' : '全量同步'} - 开始时间: ${startTime ? startTime.toISOString() : '无限制'}`);
+  } 
+  // 否则尝试获取上次同步时间作为增量同步的开始时间点
+  else {
+    try {
+      // 获取上次同步状态
+      const stateStr = await getVariable(STATE_VARIABLE_PATH);
+      if (stateStr) {
+        syncState = JSON.parse(stateStr);
+        console.log(`获取到上次同步状态: 同步时间=${syncState.lastSyncTime}, 运行时间=${syncState.lastRunTime}`);
+        
+        // 使用上次运行时间作为本次的开始时间 (减去1分钟防止边界问题)
+        const lastRunTime = new Date(syncState.lastRunTime);
+        lastRunTime.setMinutes(lastRunTime.getMinutes() - 1);
+        startTime = lastRunTime;
+      } else {
+        console.log("未找到上次同步状态，将执行全量同步");
+      }
+    } catch (error: unknown) {
+      console.log(`获取同步状态出错: ${error instanceof Error ? error.message : String(error)}，将执行全量同步`);
+    }
+  }
 
   console.log(`开始同步PostgreSQL shorturl数据到ClickHouse`);
   console.log(`参数: limit=${limit}, includeDeleted=${includeDeleted}, dryRun=${dryRun}`);
@@ -67,7 +111,7 @@ export async function main(
 
   // 获取数据库配置
   console.log("获取PostgreSQL数据库配置...");
-  const pgConfig = await getResource('f/limq/postgresql') as PgConfig;
+  const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
   console.log(`数据库连接配置: host=${pgConfig.host}, port=${pgConfig.port}, database=${pgConfig.dbname || 'postgres'}, user=${pgConfig.user}`);
   
   let pgPool: Pool | null = null;
@@ -106,6 +150,8 @@ export async function main(
     console.log(`获取到 ${shorturls.length} 个shorturl资源`);
     
     if (shorturls.length === 0) {
+      // 即使没有数据也更新状态
+      await updateSyncState(currentRunTime);
       return { synced: 0, message: "没有找到需要同步的shorturl资源" };
     }
 
@@ -120,7 +166,11 @@ export async function main(
       // 写入ClickHouse
       const inserted = await insertToClickhouse(clickhouseData);
       console.log(`成功写入 ${inserted} 条记录到ClickHouse`);
-      return { synced: inserted, message: "同步完成" };
+      
+      // 更新同步状态
+      await updateSyncState(currentRunTime);
+      
+      return { synced: inserted, message: "同步完成", lastSyncTime: currentRunTime };
     } else {
       console.log("Dry run模式 - 不执行实际写入");
       console.log(`将写入 ${clickhouseData.length} 条记录到ClickHouse`);
@@ -146,6 +196,22 @@ export async function main(
   }
 }
 
+// 更新同步状态
+async function updateSyncState(currentRunTime: string): Promise<void> {
+  try {
+    const syncState: SyncState = {
+      lastSyncTime: new Date().toISOString(), // 记录数据同步完成的时间
+      lastRunTime: currentRunTime              // 记录本次运行的时间点
+    };
+    
+    console.log(`更新同步状态: ${JSON.stringify(syncState)}`);
+    await setVariable(STATE_VARIABLE_PATH, JSON.stringify(syncState));
+  } catch (error: unknown) {
+    console.error(`更新同步状态失败: ${error instanceof Error ? error.message : String(error)}`);
+    // 不中断主流程，即使状态更新失败
+  }
+}
+
 // 从PostgreSQL获取所有shorturl资源
 async function fetchShorturlResources(
   pgPool: Pool,
@@ -185,8 +251,9 @@ async function fetchShorturlResources(
     query += ` AND r.deleted_at IS NULL`;
   }
 
+  // 修改为同时考虑created_at和updated_at，确保捕获自上次同步以来创建或更新的记录
   if (options.startTime) {
-    query += ` AND r.created_at >= $${paramCount}`;
+    query += ` AND (r.created_at >= $${paramCount} OR r.updated_at >= $${paramCount})`;
     params.push(options.startTime);
     paramCount++;
   }
@@ -197,7 +264,8 @@ async function fetchShorturlResources(
     paramCount++;
   }
 
-  query += ` ORDER BY r.created_at DESC LIMIT $${paramCount}`;
+  // 优先按更新时间排序，确保最近更新的记录先处理
+  query += ` ORDER BY r.updated_at DESC, r.created_at DESC LIMIT $${paramCount}`;
   params.push(options.limit);
 
   const client = await pgPool.connect();

windmill/sync_shorturl_to_clickhouse_intime.ts

@@ -0,0 +1,660 @@
+// 文件名: sync_resource_relations.ts
+// 描述: 此脚本用于同步PostgreSQL中资源关联数据到ClickHouse
+// 作者: AI Assistant
+// 创建日期: 2023-10-31
+
+import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
+import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
+
+// PostgreSQL配置接口
+interface PgConfig {
+  host: string;
+  port: number;
+  user: string;
+  password: string;
+  dbname?: string;
+  [key: string]: unknown;
+}
+
+// ClickHouse配置接口
+interface ChConfig {
+  clickhouse_host: string;
+  clickhouse_port: number;
+  clickhouse_user: string;
+  clickhouse_password: string;
+  clickhouse_url?: string;
+}
+
+// 资源相关接口定义
+interface TeamData {
+  team_id: string;
+  team_name: string;
+  team_description?: string;
+  project_id?: string;
+}
+
+interface ProjectData {
+  project_id: string;
+  project_name: string;
+  project_description?: string;
+  assigned_at?: string;
+  resource_id?: string;
+}
+
+interface TagData {
+  tag_id: string;
+  tag_name: string;
+  tag_type?: string;
+  created_at?: string;
+  resource_id?: string;
+}
+
+interface FavoriteData {
+  favorite_id: string;
+  user_id: string;
+  first_name?: string;
+  last_name?: string;
+  email?: string;
+  created_at?: string;
+}
+
+// 资源关联数据接口
+interface ResourceRelations {
+  resource_id: string;
+  teams?: TeamData[];
+  projects?: ProjectData[];
+  tags?: TagData[];
+  favorites?: FavoriteData[];
+  external_id?: string;
+  type?: string;
+  attributes?: Record<string, unknown>;
+}
+
+/**
+ * 同步PostgreSQL资源关联数据到ClickHouse
+ */
+export async function main(
+  params: {
+    /** 要同步的资源ID列表 */
+    resource_ids: string[];
+    /** 是否同步teams数据 */
+    sync_teams?: boolean;
+    /** 是否同步projects数据 */
+    sync_projects?: boolean;
+    /** 是否同步tags数据 */
+    sync_tags?: boolean;
+    /** 是否同步favorites数据 */
+    sync_favorites?: boolean;
+    /** 是否为测试模式（不执行实际更新） */
+    dry_run?: boolean;
+    /** 是否显示详细日志 */
+    verbose?: boolean;
+  }
+) {
+  // 设置默认参数
+  const resource_ids = params.resource_ids || [];
+  const sync_teams = params.sync_teams !== false;
+  const sync_projects = params.sync_projects !== false;
+  const sync_tags = params.sync_tags !== false;
+  const sync_favorites = params.sync_favorites !== false;
+  const dry_run = params.dry_run || false;
+  const verbose = params.verbose || false;
+
+  if (resource_ids.length === 0) {
+    return { success: false, message: "至少需要提供一个资源ID" };
+  }
+
+  // 初始化日志函数
+  const log = (message: string, isVerbose = false) => {
+    if (!isVerbose || verbose) {
+      console.log(message);
+    }
+  };
+
+  log(`开始同步资源关联数据: ${resource_ids.join(", ")}`);
+  log(`同步选项: teams=${sync_teams}, projects=${sync_projects}, tags=${sync_tags}, favorites=${sync_favorites}`, true);
+
+  let pgPool: Pool | null = null;
+
+  try {
+    // 1. 获取数据库配置
+    log("获取PostgreSQL数据库配置...", true);
+    const pgConfig = await getResource('f/limq/postgresql') as PgConfig;
+    
+    // 2. 创建PostgreSQL连接池
+    pgPool = new Pool({
+      hostname: pgConfig.host,
+      port: pgConfig.port,
+      user: pgConfig.user,
+      password: pgConfig.password,
+      database: pgConfig.dbname || 'postgres'
+    }, 3);
+    
+    // 3. 获取需要更新的资源完整数据
+    const resourcesData = await getResourcesWithRelations(pgPool, resource_ids, {
+      sync_teams,
+      sync_projects,
+      sync_tags,
+      sync_favorites
+    }, log);
+    
+    log(`成功获取 ${resourcesData.length} 个资源的关联数据`);
+    
+    if (resourcesData.length === 0) {
+      return { success: true, message: "没有找到需要更新的资源数据", updated: 0 };
+    }
+    
+    // 4. 获取ClickHouse配置
+    const chConfig = await getClickHouseConfig();
+    
+    // 5. 对每个资源执行更新
+    if (!dry_run) {
+      // 5a. 更新shorturl表数据
+      const shorturlUpdated = await updateClickHouseShorturl(resourcesData, chConfig, log);
+      
+      // 5b. 更新events表数据
+      const eventsUpdated = await updateClickHouseEvents(resourcesData, chConfig, log);
+      
+      return { 
+        success: true, 
+        message: "资源关联数据同步完成", 
+        shorturl_updated: shorturlUpdated,
+        events_updated: eventsUpdated,
+        total_updated: shorturlUpdated + eventsUpdated 
+      };
+    } else {
+      log("测试模式: 不执行实际更新");
+      if (resourcesData.length > 0) {
+        log("示例数据:");
+        log(JSON.stringify(resourcesData[0], null, 2));
+      }
+      return { success: true, dry_run: true, resources: resourcesData };
+    }
+  } catch (error) {
+    const errorMessage = `同步过程中发生错误: ${(error as Error).message}`;
+    log(errorMessage);
+    if ((error as Error).stack) {
+      log(`错误堆栈: ${(error as Error).stack}`, true);
+    }
+    return { success: false, message: errorMessage };
+  } finally {
+    if (pgPool) {
+      await pgPool.end();
+      log("PostgreSQL连接池已关闭", true);
+    }
+  }
+}
+
+/**
+ * 从PostgreSQL获取资源及其关联数据
+ */
+async function getResourcesWithRelations(
+  pgPool: Pool,
+  resourceIds: string[],
+  options: {
+    sync_teams: boolean;
+    sync_projects: boolean;
+    sync_tags: boolean;
+    sync_favorites: boolean;
+  },
+  log: (message: string, isVerbose?: boolean) => void
+): Promise<ResourceRelations[]> {
+  const client = await pgPool.connect();
+  
+  try {
+    // 准备资源IDs参数
+    const resourceIdsParam = resourceIds.map(id => `'${id}'`).join(',');
+    
+    // 1. 获取基本资源信息
+    log(`获取资源基本信息: ${resourceIdsParam}`, true);
+    const resourcesQuery = `
+      SELECT 
+        r.id, 
+        r.external_id, 
+        r.type, 
+        r.attributes, 
+        r.schema_version,
+        r.created_at,
+        r.updated_at
+      FROM 
+        limq.resources r
+      WHERE 
+        r.id IN (${resourceIdsParam})
+        AND r.deleted_at IS NULL
+    `;
+    
+    const resourcesResult = await client.queryObject(resourcesQuery);
+    
+    if (resourcesResult.rows.length === 0) {
+      log(`未找到有效的资源数据`, true);
+      return [];
+    }
+    
+    // 处理每个资源
+    const enrichedResources: ResourceRelations[] = [];
+    
+    for (const resource of resourcesResult.rows) {
+      const resourceId = resource.id as string;
+      log(`处理资源ID: ${resourceId}`, true);
+      
+      // 初始化关联数据对象
+      const relationData: ResourceRelations = {
+        resource_id: resourceId,
+        external_id: resource.external_id as string,
+        type: resource.type as string,
+        attributes: parseJsonField(resource.attributes)
+      };
+      
+      // 2. 获取项目关联
+      if (options.sync_projects) {
+        const projectsQuery = `
+          SELECT 
+            pr.resource_id, pr.project_id, 
+            p.name as project_name, p.description as project_description,
+            pr.assigned_at
+          FROM 
+            limq.project_resources pr
+          JOIN 
+            limq.projects p ON pr.project_id = p.id
+          WHERE 
+            pr.resource_id = $1
+            AND p.deleted_at IS NULL
+        `;
+        
+        const projectsResult = await client.queryObject(projectsQuery, [resourceId]);
+        relationData.projects = projectsResult.rows as ProjectData[];
+        log(`找到 ${projectsResult.rows.length} 个关联项目`, true);
+      }
+      
+      // 3. 获取标签关联
+      if (options.sync_tags) {
+        const tagsQuery = `
+          SELECT 
+            rt.resource_id, rt.tag_id, rt.created_at,
+            t.name as tag_name, t.type as tag_type
+          FROM 
+            limq.resource_tags rt
+          JOIN 
+            limq.tags t ON rt.tag_id = t.id
+          WHERE 
+            rt.resource_id = $1
+            AND t.deleted_at IS NULL
+        `;
+        
+        const tagsResult = await client.queryObject(tagsQuery, [resourceId]);
+        relationData.tags = tagsResult.rows as TagData[];
+        log(`找到 ${tagsResult.rows.length} 个关联标签`, true);
+      }
+      
+      // 4. 获取团队关联（通过项目）
+      if (options.sync_teams && relationData.projects && relationData.projects.length > 0) {
+        const projectIds = relationData.projects.map((p: ProjectData) => p.project_id);
+        
+        if (projectIds.length > 0) {
+          const teamsQuery = `
+            SELECT 
+              tp.team_id, tp.project_id,
+              t.name as team_name, t.description as team_description
+            FROM 
+              limq.team_projects tp
+            JOIN 
+              limq.teams t ON tp.team_id = t.id
+            WHERE 
+              tp.project_id = ANY($1::uuid[])
+              AND t.deleted_at IS NULL
+          `;
+          
+          const teamsResult = await client.queryObject(teamsQuery, [projectIds]);
+          relationData.teams = teamsResult.rows as TeamData[];
+          log(`找到 ${teamsResult.rows.length} 个关联团队`, true);
+        }
+      }
+      
+      // 5. 获取收藏关联
+      if (options.sync_favorites) {
+        const favoritesQuery = `
+          SELECT 
+            f.id as favorite_id, f.user_id, f.created_at,
+            u.first_name, u.last_name, u.email
+          FROM 
+            limq.favorite f
+          JOIN 
+            limq.users u ON f.user_id = u.id
+          WHERE 
+            f.favoritable_id = $1 
+            AND f.favoritable_type = 'resource'
+            AND f.deleted_at IS NULL
+        `;
+        
+        const favoritesResult = await client.queryObject(favoritesQuery, [resourceId]);
+        relationData.favorites = favoritesResult.rows as FavoriteData[];
+        log(`找到 ${favoritesResult.rows.length} 个收藏记录`, true);
+      }
+      
+      // 添加到结果集
+      enrichedResources.push(relationData);
+    }
+    
+    return enrichedResources;
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * 更新ClickHouse中的shorturl表数据
+ */
+async function updateClickHouseShorturl(
+  resources: ResourceRelations[],
+  chConfig: ChConfig,
+  log: (message: string, isVerbose?: boolean) => void
+): Promise<number> {
+  // 只处理类型为shorturl的资源
+  const shorturls = resources.filter(r => r.type === 'shorturl');
+  
+  if (shorturls.length === 0) {
+    log('没有找到shorturl类型的资源，跳过shorturl表更新');
+    return 0;
+  }
+  
+  log(`准备更新 ${shorturls.length} 个shorturl资源`);
+  
+  let updatedCount = 0;
+  
+  // 检查ClickHouse中是否存在shorturl表
+  const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.shorturl');
+  
+  if (!tableExists) {
+    log('ClickHouse中未找到shorturl表，请先创建表');
+    return 0;
+  }
+  
+  // 对每个资源执行更新
+  for (const shorturl of shorturls) {
+    try {
+      // 格式化团队数据
+      const teams = JSON.stringify(shorturl.teams || []);
+      
+      // 格式化项目数据
+      const projects = JSON.stringify(shorturl.projects || []);
+      
+      // 格式化标签数据
+      const tags = JSON.stringify((shorturl.tags || []).map((t: TagData) => ({
+        tag_id: t.tag_id,
+        tag_name: t.tag_name,
+        tag_type: t.tag_type,
+        created_at: t.created_at
+      })));
+      
+      // 格式化收藏数据
+      const favorites = JSON.stringify((shorturl.favorites || []).map((f: FavoriteData) => ({
+        favorite_id: f.favorite_id,
+        user_id: f.user_id,
+        user_name: `${f.first_name || ""} ${f.last_name || ""}`.trim(),
+        created_at: f.created_at
+      })));
+      
+      // 尝试更新ClickHouse数据
+      const updateQuery = `
+        ALTER TABLE shorturl_analytics.shorturl
+        UPDATE 
+          teams = '${escapeString(teams)}',
+          projects = '${escapeString(projects)}',
+          tags = '${escapeString(tags)}',
+          favorites = '${escapeString(favorites)}'
+        WHERE id = '${shorturl.resource_id}'
+      `;
+      
+      await executeClickHouseQuery(chConfig, updateQuery);
+      log(`更新shorturl完成: ${shorturl.resource_id}`, true);
+      updatedCount++;
+      
+    } catch (error) {
+      log(`更新shorturl ${shorturl.resource_id} 失败: ${(error as Error).message}`);
+    }
+  }
+  
+  return updatedCount;
+}
+
+/**
+ * 更新ClickHouse中的events表数据
+ */
+async function updateClickHouseEvents(
+  resources: ResourceRelations[],
+  chConfig: ChConfig,
+  log: (message: string, isVerbose?: boolean) => void
+): Promise<number> {
+  // 过滤出有external_id的资源
+  const resourcesWithExternalId = resources.filter(r => r.external_id && r.external_id.trim() !== '');
+  
+  if (resourcesWithExternalId.length === 0) {
+    log('没有找到具有external_id的资源，跳过events表更新');
+    return 0;
+  }
+  
+  log(`准备更新events表中与 ${resourcesWithExternalId.length} 个外部ID相关的记录`);
+  
+  // 检查ClickHouse中是否存在events表
+  const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.events');
+  
+  if (!tableExists) {
+    log('ClickHouse中未找到events表，请先创建表');
+    return 0;
+  }
+  
+  // 提取所有的external_id
+  const externalIds = resourcesWithExternalId.map(r => r.external_id).filter(Boolean) as string[];
+  
+  // 构建资源数据映射（使用external_id作为键）
+  const resourceMapByExternalId = resourcesWithExternalId.reduce((map, resource) => {
+    if (resource.external_id) {
+      map[resource.external_id] = resource;
+    }
+    return map;
+  }, {} as Record<string, ResourceRelations>);
+  
+  // 获取ClickHouse中相关资源的事件记录数量
+  let updatedCount = 0;
+  
+  try {
+    // 格式化外部ID列表
+    const formattedExternalIds = externalIds.map(id => `'${id}'`).join(', ');
+    
+    // 先查询是否有相关事件
+    const countQuery = `
+      SELECT COUNT(*) as count
+      FROM shorturl_analytics.events 
+      WHERE event_id IN (${formattedExternalIds})
+    `;
+    
+    const countResult = await executeClickHouseQuery(chConfig, countQuery);
+    const eventCount = parseInt(countResult.trim(), 10);
+    
+    if (eventCount === 0) {
+      // 尝试另一种查询方式
+      const alternateCountQuery = `
+        SELECT COUNT(*) as count
+        FROM shorturl_analytics.events 
+        WHERE link_id IN (${formattedExternalIds})
+      `;
+      
+      const alternateCountResult = await executeClickHouseQuery(chConfig, alternateCountQuery);
+      const alternateEventCount = parseInt(alternateCountResult.trim(), 10);
+      
+      if (alternateEventCount === 0) {
+        log('没有找到相关事件记录，跳过events表更新');
+        log(`已尝试的匹配字段: event_id，link_id`, true);
+        return 0;
+      } else {
+        log(`找到 ${alternateEventCount} 条以link_id匹配的事件记录需要更新`);
+      }
+    } else {
+      log(`找到 ${eventCount} 条以event_id匹配的事件记录需要更新`);
+    }
+    
+    // 批量更新每个资源相关的事件记录
+    for (const externalId of externalIds) {
+      const resource = resourceMapByExternalId[externalId];
+      
+      if (!resource) continue;
+      
+      // 获取关联数据
+      const tags = resource.tags ? JSON.stringify(resource.tags) : null;
+      
+      if (tags) {
+        // 尝试通过event_id更新事件标签
+        const updateTagsQueryByEventId = `
+          ALTER TABLE shorturl_analytics.events
+          UPDATE link_tags = '${escapeString(tags)}'
+          WHERE event_id = '${externalId}'
+        `;
+        
+        await executeClickHouseQuery(chConfig, updateTagsQueryByEventId);
+        log(`尝试通过event_id更新事件标签: ${externalId}`, true);
+        
+        // 尝试通过link_id更新事件标签
+        const updateTagsQueryByLinkId = `
+          ALTER TABLE shorturl_analytics.events
+          UPDATE link_tags = '${escapeString(tags)}'
+          WHERE link_id = '${externalId}'
+        `;
+        
+        await executeClickHouseQuery(chConfig, updateTagsQueryByLinkId);
+        log(`尝试通过link_id更新事件标签: ${externalId}`, true);
+      }
+      
+      // 如果资源有resource_id，也尝试使用它来更新
+      if (resource.resource_id) {
+        const updateByResourceId = `
+          ALTER TABLE shorturl_analytics.events
+          UPDATE link_tags = '${escapeString(tags || '[]')}'
+          WHERE link_id = '${resource.resource_id}'
+        `;
+        
+        await executeClickHouseQuery(chConfig, updateByResourceId);
+        log(`尝试通过resource_id更新事件标签: ${resource.resource_id}`, true);
+      }
+      
+      updatedCount++;
+    }
+    
+    log(`已尝试更新 ${updatedCount} 个资源的事件记录`);
+    
+  } catch (error) {
+    log(`更新events表失败: ${(error as Error).message}`);
+  }
+  
+  return updatedCount;
+}
+
+/**
+ * 获取ClickHouse配置
+ */
+async function getClickHouseConfig(): Promise<ChConfig> {
+  try {
+    const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
+    
+    // 确保配置不为空
+    if (!chConfigJson) {
+      throw new Error("未找到ClickHouse配置");
+    }
+    
+    // 解析JSON字符串为对象
+    let chConfig: ChConfig;
+    if (typeof chConfigJson === 'string') {
+      try {
+        chConfig = JSON.parse(chConfigJson);
+      } catch (_) {
+        throw new Error("ClickHouse配置不是有效的JSON");
+      }
+    } else {
+      chConfig = chConfigJson as ChConfig;
+    }
+    
+    // 验证并构建URL
+    if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
+      chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
+    }
+    
+    if (!chConfig.clickhouse_url) {
+      throw new Error("ClickHouse配置缺少URL");
+    }
+    
+    return chConfig;
+  } catch (error) {
+    throw new Error(`获取ClickHouse配置失败: ${(error as Error).message}`);
+  }
+}
+
+/**
+ * 检查ClickHouse中是否存在指定表
+ */
+async function checkClickHouseTable(chConfig: ChConfig, tableName: string): Promise<boolean> {
+  try {
+    const query = `EXISTS TABLE ${tableName}`;
+    const result = await executeClickHouseQuery(chConfig, query);
+    return result.trim() === '1';
+  } catch (error) {
+    console.error(`检查表 ${tableName} 失败:`, error);
+    return false;
+  }
+}
+
+/**
+ * 执行ClickHouse查询
+ */
+async function executeClickHouseQuery(chConfig: ChConfig, query: string): Promise<string> {
+  // 确保URL有效
+  if (!chConfig.clickhouse_url) {
+    throw new Error("无效的ClickHouse URL: 未定义");
+  }
+  
+  // 执行HTTP请求
+  try {
+    const response = await fetch(chConfig.clickhouse_url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
+      },
+      body: query,
+    });
+    
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`ClickHouse查询失败 (${response.status}): ${errorText}`);
+    }
+    
+    return await response.text();
+  } catch (error) {
+    throw new Error(`执行ClickHouse查询失败: ${(error as Error).message}`);
+  }
+}
+
+/**
+ * 解析JSON字段
+ */
+function parseJsonField(field: unknown): Record<string, unknown> {
+  if (!field) return {};
+  
+  try {
+    if (typeof field === 'string') {
+      return JSON.parse(field);
+    } else if (typeof field === 'object') {
+      return field as Record<string, unknown>;
+    }
+  } catch (error) {
+    console.warn(`无法解析JSON字段:`, error);
+  }
+  
+  return {};
+}
+
+/**
+ * 转义字符串，避免SQL注入
+ */
+function escapeString(str: string): string {
+  if (!str) return '';
+  return str.replace(/'/g, "''");
+}