98 Commits

Author SHA1 Message Date
Liam
51e168ee3b eventid 2025-04-28 20:18:46 +08:00
Liam
cf0f35e274 activies api 2025-04-28 19:48:18 +08:00
3162836e91 Refactor route protection by replacing ProtectedRoute with ClientRouteGuard in analytics, create-shorturl, and links pages to standardize authentication handling across the application. 2025-04-24 01:41:44 +08:00
d80d5e976b Update authentication redirect to use environment variable for site URL, ensuring proper redirection based on configuration. This change enhances flexibility for different deployment environments. 2025-04-24 00:34:47 +08:00
5d5b501a66 Remove development and production environment configuration files (.env.development and .env.production) to streamline project setup and enhance security by eliminating sensitive information from version control. 2025-04-24 00:21:13 +08:00
fe40aad835 Enhance MongoDB to ClickHouse synchronization script by adding support for custom time range synchronization, allowing users to specify start and end dates. Update .env file to include MongoDB connection URL and add .gitignore for script dependencies. 2025-04-24 00:09:24 +08:00
92db5ad783 Update authentication redirects to use environment variable for site URL, enhancing flexibility for different environments. Add NEXT_PUBLIC_SITE_URL to .env for production URL configuration. 2025-04-23 22:38:56 +08:00
b94a91914a Enhance GET request validation in activities route to ensure both slug and domain are provided together, or alternatively, a date range. Update error messages for clarity. 2025-04-23 22:27:49 +08:00
8551f5c445 csv text 2025-04-23 21:16:32 +08:00
dafa7f53ac create en 2025-04-23 21:03:16 +08:00
0203cb4041 Add Google sign-in functionality to Login and Register pages, including error handling and UI updates for better user experience. 2025-04-23 20:52:04 +08:00
ced29201da Refactor limqRequest function to remove default localhost URL and add CORS mode for API requests. 2025-04-23 09:16:29 +08:00
a8c94c9621 Update .env file to change NEXT_PUBLIC_LIMQ_API URL from localhost to production endpoint. 2025-04-22 21:15:44 +08:00
4736ebe060 Add .env file for environment configuration, including settings for ClickHouse, Redis, Supabase, and public Next.js variables. Update .gitignore to exclude .env file from version control. 2025-04-22 16:36:05 +08:00
6858f2fda5 Add team and project selection to Create Short URL form, including validation for required fields and domain input. 2025-04-22 16:34:18 +08:00
42f5be4dcb Add "Create Short URL" link to Header, remove Navbar component, and implement Create Short URL page with form handling and validation. 2025-04-22 13:07:20 +08:00
05af4aae70 Add limqRequest function for API interactions and implement default team creation in TeamSelector. Remove test user auto-registration from AuthProvider. 2025-04-22 13:07:08 +08:00
ed1d2e59f6 Update sync_mongo_to_events.ts to force insert records and simplify ClickHouse checks. Removed existing record validation logic and added placeholders for missing data attributes. 2025-04-21 23:54:15 +08:00
3cbb76db36 Refactor login page to isolate message handling into a separate component and utilize Suspense for loading. This improves code organization and maintains functionality for displaying messages from URL parameters. 2025-04-18 22:04:29 +08:00
ecef81b0ee Remove .env.local file and simplify login page by removing GitHub and Google sign-in options. 2025-04-18 21:58:32 +08:00
9cb85a2910 Enhance login page to display messages from URL parameters and update email redirect options for sign-up confirmation. 2025-04-18 21:50:06 +08:00
3af015ca44 env 2025-04-18 21:29:34 +08:00
f6f24d3450 Update development server port in package.json to 3007 2025-04-18 21:13:37 +08:00
4262f789da utm sync 2025-04-18 08:33:39 +08:00
2e34cd5b4b activity csv data 2025-04-17 23:48:46 +08:00
2cb45781c7 Add req_full_path to Event interface, implement activities API for event retrieval, and enhance sync script with short link details 2025-04-17 22:23:38 +08:00
53e1611670 auto refresh 2025-04-17 18:28:08 +08:00
6025641ab1 sync events intime 2025-04-17 14:22:50 +08:00
b9c2828e54 Add domain field to shortlink API responses and sync script 2025-04-16 21:32:49 +08:00
b1753449f5 sync url 2025-04-16 20:55:48 +08:00
85f29d8b49 click subpath match 2025-04-10 18:31:24 +08:00
b8cd3716c4 click subpath 2025-04-10 18:07:10 +08:00
48d5bdafa4 click subpath 2025-04-10 17:19:40 +08:00
ace231b93f rm parameters 2025-04-10 15:10:26 +08:00
e101d19e00 add path cont 2025-04-10 12:14:54 +08:00
a8576121e9 add path cont 2025-04-10 11:44:03 +08:00
8b407975e5 sync & read me 2025-04-09 19:20:40 +08:00
ede83068af fix modal 2025-04-08 12:41:41 +08:00
d21026eafd fix build 2025-04-08 12:00:58 +08:00
6940d60510 time chart int 2025-04-08 07:46:20 +08:00
4e7266240d persisten storge 2025-04-08 07:04:02 +08:00
db70602e9f hide filter 2025-04-08 00:03:13 +08:00
d0e83f697b take shorturl data 2025-04-07 23:20:48 +08:00
ed327ad3f0 change route 2025-04-07 22:27:02 +08:00
f782dba0c9 links info 2025-04-07 22:17:53 +08:00
0c4a67e769 links search 2025-04-07 22:08:12 +08:00
694e005101 links 2025-04-07 21:58:28 +08:00
523e99a001 links 2025-04-07 21:54:05 +08:00
33dbf62665 links 2025-04-07 21:48:24 +08:00
1a9e28bd7e show label 2025-04-03 21:57:55 +08:00
d1d21948b6 tag fix 2025-04-03 17:56:16 +08:00
f32a45d24a utm 2025-04-03 17:50:45 +08:00
d61b8a62ff utm 2025-04-03 16:27:04 +08:00
0b41f3ea42 geo main 2025-04-02 22:23:49 +08:00
63f434fd93 geo tab 2025-04-02 21:36:13 +08:00
95f230b996 geo 2025-04-02 20:50:10 +08:00
0f8419778c device ana 2025-04-02 20:18:08 +08:00
a6f7172ec4 fix filter 2025-04-02 20:05:33 +08:00
8054b0235d events trend filters 2025-04-02 18:05:45 +08:00
b0dbd088e7 rm no used page 2025-04-02 17:56:15 +08:00
bf7c62fdc9 events pages 2025-04-02 10:04:36 +08:00
9cb9f62686 events filter 2025-04-02 08:55:46 +08:00
4b7fb7a887 rearrange pages 2025-04-01 23:44:01 +08:00
bdae5c164c move events pos 2025-04-01 23:04:15 +08:00
9fa61ccf8d summary filter 2025-04-01 23:00:35 +08:00
b187bdefdf sidebar collapse 2025-04-01 22:51:11 +08:00
87c3803236 add filter 2025-04-01 22:40:33 +08:00
75adb36111 rm evets id filter 2025-04-01 22:35:10 +08:00
a4ef2c3147 events team 2025-04-01 22:34:07 +08:00
57e16144a9 team filter 2025-04-01 22:26:46 +08:00
1be6a6dbf0 move page 2025-04-01 22:18:25 +08:00
36f22059e9 move device 2025-04-01 21:22:23 +08:00
a8d364be1f tags selector 2025-04-01 20:09:49 +08:00
326a6c6d63 project selector 2025-04-01 20:03:15 +08:00
0a881fd180 team selector 2025-04-01 19:51:30 +08:00
1b901bda90 rm dack 2025-04-01 19:43:30 +08:00
53822f1087 team selector 2025-04-01 19:04:13 +08:00
1978e0224e supabase client tool 2025-04-01 17:45:12 +08:00
c0649ce10f component supabase 2025-04-01 17:36:28 +08:00
696a434b95 login page 2025-04-01 14:57:22 +08:00
b8e6180212 dashboard data 2025-04-01 12:40:57 +08:00
6beb6c3666 rm swagger 2025-04-01 12:00:13 +08:00
17b588e249 use next dev 2025-04-01 11:51:27 +08:00
26db8fe76d percent 2025-03-28 15:15:24 +08:00
4ad505cda1 fix build 2025-03-26 21:50:04 +08:00
7a03396cdd add event 2025-03-26 20:19:37 +08:00
e9b9950ed3 add event example desc 2025-03-26 19:21:57 +08:00
f5b14bf936 event track api 2025-03-26 18:19:37 +08:00
ca8a7d56f1 swagerr doc 2025-03-26 17:39:58 +08:00
913c9cd289 swagger configure 2025-03-26 17:17:47 +08:00
e916eab92c mv folder 2025-03-26 16:39:04 +08:00
63a578ef38 event api 2025-03-26 12:04:53 +08:00
b4aa765c17 event api 2025-03-26 11:26:53 +08:00
c0e5a9ccb2 add pie chart 2025-03-26 11:18:36 +08:00
1755b44a39 style 2025-03-25 22:24:52 +08:00
e0ac87fb25 events 2025-03-25 21:12:03 +08:00
ecf21a812f dashboard page good 2025-03-25 21:02:17 +08:00
efdfe8bf8e front 2025-03-25 20:54:02 +08:00
90 changed files with 16179 additions and 3452 deletions

31
.env Normal file
View File

@@ -0,0 +1,31 @@
PORT=3007
MONGO_URL="mongodb://10.0.1.41:27017"
# ClickHouse Configuration
CLICKHOUSE_HOST=10.0.1.60
CLICKHOUSE_PORT=8123
CLICKHOUSE_USER=admin
CLICKHOUSE_PASSWORD=your_secure_password
CLICKHOUSE_DATABASE=shorturl_analytics
CLICKHOUSE_URL=http://10.0.1.60:8123
REDIS_HOST="localhost"
REDIS_PORT="6379"
REDIS_PASSWORD=""
# Supabase Configuration
SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
SUPABASE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc0NDM0NTY0MywiZXhwIjoyMDU5OTIxNjQzfQ.ZenTsEAdGiDu1DCCOT7G8xxvgFXKLl4qhHB-AhSVf6w"
SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
# Next.js Public Environment Variables (accessible in browser)
NEXT_PUBLIC_SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
NEXT_PUBLIC_LIMQ_API="https://app.upj.to"
# Application URL for redirects (replace with your production URL)
NEXT_PUBLIC_SITE_URL="https://main.upj.to"

2
.gitignore vendored
View File

@@ -31,7 +31,7 @@ yarn-error.log*
.pnpm-debug.log*
# env files (can opt-in for committing if needed)
.env*
# .env
# vercel
.vercel

View File

@@ -0,0 +1,47 @@
# Date Format Handling for ClickHouse Events API
## Problem Description
The event tracking API was experiencing issues with date format compatibility when inserting records into the ClickHouse database. ClickHouse has specific requirements for datetime formats, particularly for its `DateTime64` type fields, which weren't being properly addressed in the original implementation.
## Root Cause
- JavaScript's default date serialization (`toISOString()`) produces formats like `2023-08-24T12:34:56.789Z`, which include `T` as a separator and `Z` as the UTC timezone indicator
- ClickHouse prefers datetime values in the format `YYYY-MM-DD HH:MM:SS.SSS` for seamless parsing
- The mismatch between these formats was causing insertion errors in the database
## Solution Implemented
We created a `formatDateTime` utility function that properly formats JavaScript Date objects for ClickHouse compatibility:
```typescript
const formatDateTime = (date: Date) => {
return date.toISOString().replace('T', ' ').replace('Z', '');
};
```
This function:
1. Takes a JavaScript Date object as input
2. Converts it to ISO format string
3. Replaces the 'T' separator with a space
4. Removes the trailing 'Z' UTC indicator
The solution was applied to all date fields in the event payload:
- `event_time`
- `link_created_at`
- `link_expires_at`
## Additional Improvements
- We standardized date handling by using a consistent `currentTime` variable
- Added type checking for JSON fields to ensure proper serialization
- Improved error handling for date parsing failures
## Best Practices for ClickHouse Date Handling
1. Always format dates as `YYYY-MM-DD HH:MM:SS.SSS` when inserting into ClickHouse
2. Use consistent date handling utilities across your application
3. Consider timezone handling explicitly when needed
4. For query parameters, use ClickHouse's `parseDateTimeBestEffort` function when possible
5. Test with various date formats and edge cases to ensure robustness

42
README-auth-setup.md Normal file
View File

@@ -0,0 +1,42 @@
# 身份验证重定向 URL 配置指南
## 问题:注册后确认邮件链接指向 localhost
如果您在生产环境中使用此应用,并且发现用户注册后收到的确认邮件中的链接指向 `localhost` 而非您的实际网站域名,请按照以下步骤解决:
## 解决方案
### 1. 设置环境变量
在项目根目录的 `.env.production` 文件中,确保 `NEXT_PUBLIC_SITE_URL` 变量设置为您的实际生产域名:
```
NEXT_PUBLIC_SITE_URL="https://您的真实域名.com"
```
### 2. 在 Supabase 控制台中配置
登录 [Supabase 控制台](https://app.supabase.com/),然后:
1. 选择您的项目
2. 导航到 **Authentication** > **URL Configuration**
3.**Site URL** 字段中输入您的实际网站 URL
4.**Redirect URLs** 部分添加:
- `https://您的真实域名.com/auth/callback`
### 3. 本地开发与生产环境
- **开发环境**:使用 `.env.development` 文件中的设置,通常为 `http://localhost:3007`
- **生产环境**:使用 `.env.production` 文件中的设置,应为您的实际域名
### 4. 部署后验证
项目重新部署后:
1. 尝试注册一个新账户
2. 检查收到的确认邮件,确认链接现在指向您的实际域名而非 localhost
## 技术说明
身份验证流程中,应用使用环境变量 `NEXT_PUBLIC_SITE_URL` 构建重定向 URL。如果未设置此变量它会回退到使用 `window.location.origin`,这在本地开发时会是 `localhost`
通过正确设置此变量,您可以确保无论在何处运行应用,邮件中的链接都能正确指向应用的实际位置。

50
README-google-auth.md Normal file
View File

@@ -0,0 +1,50 @@
# 配置 Google 登录功能
为了启用 Google 登录功能,您需要在 Supabase 和 Google Cloud Platform 进行配置。
## 步骤 1: 创建 Google OAuth 客户端
1. 访问 [Google Cloud Console](https://console.cloud.google.com/)
2. 创建一个新项目或选择现有项目
3. 在左侧菜单中导航到 "API 和服务" > "OAuth 同意屏幕"
4. 选择用户类型(外部或内部),然后点击"创建"
5. 填写必要的信息(应用名称、用户支持电子邮件等)并保存
6. 导航到 "API 和服务" > "凭据"
7. 点击"创建凭据" > "OAuth 客户端 ID"
8. 应用类型选择 "Web 应用"
9. 名称中输入您的应用名称
10. 添加以下已获授权的重定向 URI:
- `https://mwwvqwevplndzvmqmrxa.supabase.co/auth/v1/callback`
11. 点击"创建"
12. 复制生成的 "客户端 ID" 和 "客户端密钥"
## 步骤 2: 在 Supabase 中配置 Google 提供商
1. 登录 [Supabase 仪表板](https://app.supabase.com)
2. 选择您的项目
3. 导航到 "身份验证" > "提供商"
4. 找到 Google 提供商并启用它
5. 粘贴您刚才获取的 "客户端 ID" 和 "客户端密钥"
6. 保存配置
## 步骤 3: 更新重定向 URL如有需要
如果您的应用需要在登录后重定向到特定页面,请确保在 Google Cloud Console 和 Supabase 中配置了正确的重定向 URL。
在 Supabase 中:
1. 导航到 "身份验证" > "URL 配置"
2. 添加您的前端 URL 到站点 URL 字段中
3. 设置重定向 URL通常是您的前端 URL
## 测试
1. 在您的应用中,尝试使用 Google 登录
2. 验证认证流程,确保可以成功登录并重定向到应用
3. 检查 Supabase 中的用户数据,确认新用户已创建
## 故障排除
- 确保重定向 URI 完全匹配
- 确保 OAuth 同意屏幕已正确配置
- 查看 Supabase 和应用程序中的日志以获取详细的错误信息
- 如果遇到 CORS 错误,检查您的站点 URL 配置

View File

@@ -1,152 +0,0 @@
import { Router } from 'express';
import type { Request, Response } from 'express';
import type { ApiResponse, EventsQueryParams } from '../lib/types';
import {
getEvents,
getEventsSummary,
getTimeSeriesData,
getGeoAnalytics,
getDeviceAnalytics
} from '../lib/analytics';
const router = Router();
// 获取事件列表
router.get('/', async (req: Request, res: Response) => {
try {
const params: EventsQueryParams = {
startTime: req.query.startTime as string,
endTime: req.query.endTime as string,
eventType: req.query.eventType as string,
linkId: req.query.linkId as string,
linkSlug: req.query.linkSlug as string,
userId: req.query.userId as string,
teamId: req.query.teamId as string,
projectId: req.query.projectId as string,
page: req.query.page ? parseInt(req.query.page as string, 10) : 1,
pageSize: req.query.pageSize ? parseInt(req.query.pageSize as string, 10) : 20,
sortBy: req.query.sortBy as string,
sortOrder: req.query.sortOrder as 'asc' | 'desc'
};
const { events, total } = await getEvents(params);
const response: ApiResponse<typeof events> = {
success: true,
data: events,
meta: {
total,
page: params.page,
pageSize: params.pageSize
}
};
res.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
res.status(500).json(response);
}
});
// 获取事件概览
router.get('/summary', async (req: Request, res: Response) => {
try {
const summary = await getEventsSummary({
startTime: req.query.startTime as string,
endTime: req.query.endTime as string,
linkId: req.query.linkId as string
});
const response: ApiResponse<typeof summary> = {
success: true,
data: summary
};
res.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
res.status(500).json(response);
}
});
// 获取时间序列数据
router.get('/time-series', async (req: Request, res: Response) => {
try {
const data = await getTimeSeriesData({
startTime: req.query.startTime as string,
endTime: req.query.endTime as string,
linkId: req.query.linkId as string,
granularity: (req.query.granularity || 'day') as 'hour' | 'day' | 'week' | 'month'
});
const response: ApiResponse<typeof data> = {
success: true,
data
};
res.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
res.status(500).json(response);
}
});
// 获取地理位置分析
router.get('/geo', async (req: Request, res: Response) => {
try {
const data = await getGeoAnalytics({
startTime: req.query.startTime as string,
endTime: req.query.endTime as string,
linkId: req.query.linkId as string,
groupBy: (req.query.groupBy || 'country') as 'country' | 'city'
});
const response: ApiResponse<typeof data> = {
success: true,
data
};
res.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
res.status(500).json(response);
}
});
// 获取设备分析
router.get('/devices', async (req: Request, res: Response) => {
try {
const data = await getDeviceAnalytics({
startTime: req.query.startTime as string,
endTime: req.query.endTime as string,
linkId: req.query.linkId as string
});
const response: ApiResponse<typeof data> = {
success: true,
data
};
res.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
res.status(500).json(response);
}
});
export default router;

28
app/(app)/layout.tsx Normal file
View File

@@ -0,0 +1,28 @@
import '../globals.css';
import type { Metadata } from 'next';
import { Sidebar } from '@/app/components/Sidebar';
export const metadata: Metadata = {
title: 'ShortURL Analytics',
description: 'Analytics for your shortened URLs',
};
export default function AppLayout({
children,
}: {
children: React.ReactNode;
}) {
return (
<div className="flex h-screen bg-gray-50">
{/* 侧边栏 */}
<Sidebar />
{/* 主内容区域 */}
<div className="flex-1 flex flex-col overflow-auto">
<main className="flex-1 overflow-y-auto">
{children}
</main>
</div>
</div>
);
}

65
app/(app)/page.tsx Normal file
View File

@@ -0,0 +1,65 @@
export default function HomePage() {
return (
<div className="container mx-auto px-4 py-8">
<div className="text-center">
<h1 className="text-4xl font-bold text-gray-900 mb-8">
Welcome to ShortURL Analytics
</h1>
</div>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-8">
<a
href="/dashboard"
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 mb-2">
Dashboard
</h2>
<p className="text-gray-600">
Get an overview of all your short URL analytics data.
</p>
</a>
<a
href="/events"
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 mb-2">
Event Tracking
</h2>
<p className="text-gray-600">
View detailed events for all your short URLs.
</p>
</a>
<a
href="/analytics"
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 mb-2">
URL Analysis
</h2>
<p className="text-gray-600">
Analyze performance of specific short URLs.
</p>
</a>
<a
href="/account"
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 mb-2">
Account Settings
</h2>
<p className="text-gray-600">
Manage your account and team settings.
</p>
</a>
</div>
</div>
);
}

1123
app/analytics/page.tsx Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,126 @@
# Activities API Documentation
## Overview
The Activities API provides event tracking data for short URLs. It allows retrieving visitor activity information based on various filters such as URL slug, domain, and date ranges.
## Endpoint
```
GET /api/activities
```
## Request Parameters
| Parameter | Type | Required | Description |
|------------|---------|----------|-------------|
| `slug` | string | No* | The short URL slug to filter events by |
| `domain` | string | No* | The domain to filter events by |
| `startTime`| string | No* | Start time for date range filter (ISO format) |
| `endTime` | string | No* | End time for date range filter (ISO format) |
| `page` | integer | No | Page number for pagination (default: 1) |
| `pageSize` | integer | No | Number of records per page (default: 50) |
| `format` | string | No | Response format, set to 'csv' for CSV output (default: JSON) |
\* Either `slug`+`domain` combination OR at least one of `startTime`/`endTime` must be provided.
## Response Formats
### JSON Format (Default)
JSON responses include the following structure:
```json
{
"success": true,
"data": [
{
"id": "event-id",
"type": "event-type",
"time": "timestamp",
"visitor": {
"id": "visitor-id",
"ipAddress": "ip-address",
"userAgent": "user-agent-string",
"referrer": "referrer-url"
},
"device": {
"type": "device-type",
"browser": "browser-name",
"os": "operating-system"
},
"location": {
"country": "country-code",
"city": "city-name"
},
"link": {
"id": "link-id",
"slug": "link-slug",
"originalUrl": "original-url",
"label": "link-label",
"tags": ["tag1", "tag2"]
},
"utm": {
"source": "utm-source",
"medium": "utm-medium",
"campaign": "utm-campaign",
"term": "utm-term",
"content": "utm-content"
}
}
],
"meta": {
"total": 100,
"page": 1,
"pageSize": 50
}
}
```
In case of an error:
```json
{
"success": false,
"data": null,
"error": "Error message description"
}
```
### CSV Format
When `format=csv` is specified, the response is returned as plain text in CSV format with the following columns:
- `time`: Timestamp of the event
- `activity`: Type of activity/event
- `campaign`: UTM campaign value (defaults to "demo" if not found)
- `clientId`: Visitor ID
- `originPath`: Original request path or referrer URL
## Examples
### Get activities for a specific short URL
```
GET /api/activities?slug=promo123&domain=googleads.link
```
### Get activities within a date range
```
GET /api/activities?startTime=2023-06-01T00:00:00Z&endTime=2023-06-30T23:59:59Z
```
### Get events as CSV
```
GET /api/activities?slug=promo123&domain=googleads.link&format=csv
```
### Pagination example
```
GET /api/activities?slug=promo123&domain=googleads.link&page=2&pageSize=20
```
## Error Codes
| Status Code | Description |
|-------------|-------------|
| 400 | Missing required parameters |
| 500 | Server error while processing the request |
## Notes
- For privacy and security reasons, some fields may be omitted or anonymized based on user settings.
- The CSV format is optimized for easy import into spreadsheet applications.
- When using the CSV format, the response is returned as plain text rather than a downloadable file.

250
app/api/activities/route.ts Normal file
View File

@@ -0,0 +1,250 @@
import { NextRequest, NextResponse } from 'next/server';
import { getEvents } from '@/lib/analytics';
import { ApiResponse } from '@/lib/types';
// Extended Event type with required fields
interface EventWithFullPath {
event_id?: string;
event_time?: string;
event_type?: string;
visitor_id?: string;
ip_address?: string;
req_full_path?: string;
referrer?: string;
event_attributes?: string | Record<string, unknown>;
link_tags?: string | string[];
link_id?: string;
link_slug?: string;
link_original_url?: string;
link_label?: string;
device_type?: string;
browser?: string;
os?: string;
country?: string;
city?: string;
[key: string]: unknown;
}
export async function GET(request: NextRequest) {
try {
const { searchParams } = new URL(request.url);
// Get parameters
const slug = searchParams.get('slug');
const domain = searchParams.get('domain');
const format = searchParams.get('format');
// Optional date range parameters
const startTime = searchParams.get('startTime') || undefined;
const endTime = searchParams.get('endTime') || undefined;
// Check if either slug or domain is provided without the other
if ((slug && !domain) || (!slug && domain)) {
return NextResponse.json({
success: false,
error: 'Both slug and domain parameters must be provided together'
}, { status: 400 });
}
// Ensure either slug+domain or date range is provided
if ((!slug && !domain) && (!startTime && !endTime)) {
return NextResponse.json({
success: false,
error: 'Missing filter parameters: provide either slug+domain or date range'
}, { status: 400 });
}
// Construct the shortUrl from domain and slug if both are provided
let shortUrl = undefined;
if (slug && domain) {
shortUrl = `https://${domain}/${slug}`;
// Log the request for debugging
console.log('Activities API received parameters:', {
slug,
domain,
shortUrl,
startTime,
endTime
});
} else {
console.log('Activities API using time range filter:', {
startTime,
endTime
});
}
// Set default page size and page
const page = parseInt(searchParams.get('page') || '1');
const pageSize = parseInt(searchParams.get('pageSize') || '50');
// Get events for the specified filters
const { events, total } = await getEvents({
linkSlug: slug || undefined,
page,
pageSize,
startTime,
endTime,
sortBy: 'event_time',
sortOrder: 'desc'
});
// If format=csv, return CSV format data
if (format === 'csv') {
// CSV header line
let csvContent = 'time,activity,campaign,clientId,originPath\n';
// Helper function to extract utm_campaign from URL
const extractUtmCampaign = (url: string | null | undefined): string => {
if (!url) return 'demo';
try {
// Try to parse URL and extract utm_campaign parameter
const urlObj = new URL(url.startsWith('http') ? url : `https://example.com${url}`);
const campaign = urlObj.searchParams.get('utm_campaign');
if (campaign) return campaign;
// If utm_campaign is not found or URL parsing fails, use regex as fallback
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
} catch {
// If URL parsing fails, try regex directly
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
}
return 'demo'; // Default value
};
// Process each event record
events.forEach(event => {
// 使用类型断言处理扩展字段
const eventWithFullPath = event as unknown as EventWithFullPath;
// Get the full URL from appropriate field
// Try different possible fields that might contain the URL
const fullUrl = eventWithFullPath.req_full_path || eventWithFullPath.referrer || '';
// Extract campaign from URL
const campaign = extractUtmCampaign(fullUrl);
// Format time
const time = eventWithFullPath.event_time ?
new Date(eventWithFullPath.event_time).toISOString().replace('T', ' ').slice(0, 19) :
'';
// Determine activity (event_type)
const activity = eventWithFullPath.event_type || '';
// 修改使用link_label替代visitor_id作为clientId
const clientId = eventWithFullPath.link_label || 'undefined';
// Original path - 修正使用link_original_url作为原始URL来源
const originPath = eventWithFullPath.link_original_url || 'undefined';
// Add to CSV content
csvContent += `${time},${activity},${campaign},${clientId},${originPath}\n`;
});
// No need to generate filename since we're not using Content-Disposition header
// Return CSV response without forcing download
return new NextResponse(csvContent, {
headers: {
'Content-Type': 'text/plain'
}
});
}
// Process the events to extract useful information
const processedEvents = events.map(event => {
// Parse JSON strings to objects safely
let eventAttributes: Record<string, unknown> = {};
try {
if (typeof event.event_attributes === 'string') {
eventAttributes = JSON.parse(event.event_attributes);
} else if (typeof event.event_attributes === 'object') {
eventAttributes = event.event_attributes;
}
} catch {
// Keep default empty object if parsing fails
}
// Extract tags
let tags: string[] = [];
try {
if (typeof event.link_tags === 'string') {
const parsedTags = JSON.parse(event.link_tags);
if (Array.isArray(parsedTags)) {
tags = parsedTags;
}
} else if (Array.isArray(event.link_tags)) {
tags = event.link_tags;
}
} catch {
// If parsing fails, keep tags as empty array
}
// Return a simplified event object
return {
id: event.event_id,
type: event.event_type,
time: event.event_time,
visitor: {
id: event.visitor_id,
ipAddress: event.ip_address,
userAgent: eventAttributes.user_agent as string || null,
referrer: eventAttributes.referrer as string || null
},
device: {
type: event.device_type,
browser: event.browser,
os: event.os
},
location: {
country: event.country,
city: event.city
},
link: {
id: event.link_id,
slug: event.link_slug,
originalUrl: event.link_original_url,
label: event.link_label,
tags
},
utm: {
source: eventAttributes.utm_source as string || null,
medium: eventAttributes.utm_medium as string || null,
campaign: eventAttributes.utm_campaign as string || null,
term: eventAttributes.utm_term as string || null,
content: eventAttributes.utm_content as string || null
}
};
});
// Return processed events
const response: ApiResponse<typeof processedEvents> = {
success: true,
data: processedEvents,
meta: {
total,
page,
pageSize
}
};
return NextResponse.json(response);
} catch (error) {
console.error('Error retrieving activities:', error);
const response: ApiResponse<null> = {
success: false,
data: null,
error: error instanceof Error ? error.message : 'An error occurred while retrieving activities'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -6,10 +6,21 @@ export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
// 获取团队、项目和标签筛选参数
const teamIds = searchParams.getAll('teamId');
const projectIds = searchParams.getAll('projectId');
const tagIds = searchParams.getAll('tagId');
const data = await getDeviceAnalytics({
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
linkId: searchParams.get('linkId') || undefined
linkId: searchParams.get('linkId') || undefined,
// 添加团队、项目和标签筛选
teamIds: teamIds.length > 0 ? teamIds : undefined,
projectIds: projectIds.length > 0 ? projectIds : undefined,
tagIds: tagIds.length > 0 ? tagIds : undefined,
// 添加子路径筛选
subpath: searchParams.get('subpath') || undefined
});
const response: ApiResponse<typeof data> = {

View File

@@ -6,11 +6,25 @@ export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
// 获取团队、项目和标签筛选参数
const teamIds = searchParams.getAll('teamId');
const projectIds = searchParams.getAll('projectId');
const tagIds = searchParams.getAll('tagId');
// Get the groupBy parameter
const groupBy = searchParams.get('groupBy') as 'country' | 'city' | 'region' | 'continent' | null;
const data = await getGeoAnalytics({
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
linkId: searchParams.get('linkId') || undefined,
groupBy: (searchParams.get('groupBy') || 'country') as 'country' | 'city'
groupBy: groupBy || undefined,
// 添加团队、项目和标签筛选
teamIds: teamIds.length > 0 ? teamIds : undefined,
projectIds: projectIds.length > 0 ? projectIds : undefined,
tagIds: tagIds.length > 0 ? tagIds : undefined,
// 添加子路径筛选
subpath: searchParams.get('subpath') || undefined
});
const response: ApiResponse<typeof data> = {

View File

@@ -0,0 +1,80 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse } from '@/lib/types';
import { executeQuery } from '@/lib/clickhouse';
export async function GET(request: NextRequest) {
try {
// 获取查询参数
const searchParams = request.nextUrl.searchParams;
const startTime = searchParams.get('startTime');
const endTime = searchParams.get('endTime');
const linkId = searchParams.get('linkId');
if (!startTime || !endTime || !linkId) {
return NextResponse.json({
success: false,
error: 'Missing required parameters'
}, { status: 400 });
}
// 查询链接的点击事件
const query = `
SELECT event_attributes
FROM events
WHERE link_id = '${linkId}'
AND event_time >= parseDateTimeBestEffort('${startTime}')
AND event_time <= parseDateTimeBestEffort('${endTime}')
AND event_type = 'click'
`;
const events = await executeQuery(query);
// 处理事件数据,按路径分组
const pathMap = new Map<string, number>();
let totalClicks = 0;
events.forEach((event: any) => {
try {
if (event.event_attributes) {
const attrs = JSON.parse(event.event_attributes);
if (attrs.full_url) {
// 提取URL的路径和参数部分
const url = new URL(attrs.full_url);
const pathWithParams = url.pathname + (url.search || '');
// 更新路径计数
const currentCount = pathMap.get(pathWithParams) || 0;
pathMap.set(pathWithParams, currentCount + 1);
totalClicks++;
}
}
} catch (error) {
// 忽略解析错误
}
});
// 转换为数组并按点击数排序
const pathData = Array.from(pathMap.entries())
.map(([path, count]) => ({
path,
count,
percentage: totalClicks > 0 ? count / totalClicks : 0,
}))
.sort((a, b) => b.count - a.count);
const response: ApiResponse<typeof pathData> = {
success: true,
data: pathData,
meta: { total: totalClicks }
};
return NextResponse.json(response);
} catch (error) {
console.error('Error fetching path analytics data:', error);
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Internal server error'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -1,50 +1,74 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse, EventsQueryParams, EventType } from '@/lib/types';
import {
getEvents,
getEventsSummary,
getTimeSeriesData,
getGeoAnalytics,
getDeviceAnalytics
} from '@/lib/analytics';
import { getEvents, EventsQueryParams } from '@/lib/analytics';
import { ApiResponse } from '@/lib/types';
// 获取事件列表
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
const { searchParams } = new URL(request.url);
// 获取查询参数
const page = parseInt(searchParams.get('page') || '1');
const pageSize = parseInt(searchParams.get('pageSize') || '20');
const eventType = searchParams.get('eventType') || undefined;
const linkId = searchParams.get('linkId') || undefined;
const linkSlug = searchParams.get('linkSlug') || undefined;
const userId = searchParams.get('userId') || undefined;
const subpath = searchParams.get('subpath') || undefined;
// 获取可能存在的多个团队、项目和标签ID
const teamIds = searchParams.getAll('teamId');
const projectIds = searchParams.getAll('projectId');
const tagIds = searchParams.getAll('tagId');
const startTime = searchParams.get('startTime') || undefined;
const endTime = searchParams.get('endTime') || undefined;
const sortBy = searchParams.get('sortBy') || undefined;
const sortOrder = (searchParams.get('sortOrder') as 'asc' | 'desc') || undefined;
console.log("API接收到的tagIds:", tagIds); // 添加日志便于调试
console.log("API接收到的subpath:", subpath); // 添加日志便于调试
// 获取事件列表
const params: EventsQueryParams = {
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
eventType: searchParams.get('eventType') as EventType || undefined,
linkId: searchParams.get('linkId') || undefined,
linkSlug: searchParams.get('linkSlug') || undefined,
userId: searchParams.get('userId') || undefined,
teamId: searchParams.get('teamId') || undefined,
projectId: searchParams.get('projectId') || undefined,
page: searchParams.has('page') ? parseInt(searchParams.get('page')!, 10) : 1,
pageSize: searchParams.has('pageSize') ? parseInt(searchParams.get('pageSize')!, 10) : 20,
sortBy: searchParams.get('sortBy') || undefined,
sortOrder: (searchParams.get('sortOrder') as 'asc' | 'desc') || undefined
page,
pageSize,
eventType,
linkId,
linkSlug,
userId,
subpath,
teamIds: teamIds.length > 0 ? teamIds : undefined,
projectIds: projectIds.length > 0 ? projectIds : undefined,
tagIds: tagIds.length > 0 ? tagIds : undefined,
startTime,
endTime,
sortBy,
sortOrder
};
const { events, total } = await getEvents(params);
const response: ApiResponse<typeof events> = {
// 记录完整的参数用于调试
console.log("完整请求参数:", JSON.stringify(params));
const result = await getEvents(params);
const response: ApiResponse<typeof result.events> = {
success: true,
data: events,
data: result.events,
meta: {
total,
page: params.page,
pageSize: params.pageSize
total: result.total,
page,
pageSize
}
};
return NextResponse.json(response);
} catch (error) {
console.error('获取事件列表失败:', error);
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
data: null,
error: error instanceof Error ? error.message : '获取事件列表失败'
};
return NextResponse.json(response, { status: 500 });
}

View File

@@ -6,10 +6,27 @@ export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
// 获取可能存在的多个团队、项目和标签ID
const teamIds = searchParams.getAll('teamId');
const projectIds = searchParams.getAll('projectId');
const tagIds = searchParams.getAll('tagId');
// Add debug log to check if linkId is being received
const linkId = searchParams.get('linkId');
const subpath = searchParams.get('subpath');
console.log('Summary API received linkId:', linkId);
console.log('Summary API received subpath:', subpath);
console.log('Summary API full parameters:', Object.fromEntries(searchParams.entries()));
console.log('Summary API URL:', request.url);
const summary = await getEventsSummary({
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
linkId: searchParams.get('linkId') || undefined
linkId: searchParams.get('linkId') || undefined,
teamIds: teamIds.length > 0 ? teamIds : undefined,
projectIds: projectIds.length > 0 ? projectIds : undefined,
tagIds: tagIds.length > 0 ? tagIds : undefined,
subpath: searchParams.get('subpath') || undefined
});
const response: ApiResponse<typeof summary> = {

View File

@@ -15,11 +15,22 @@ export async function GET(request: NextRequest) {
}, { status: 400 });
}
// 获取团队、项目和标签筛选参数
const teamIds = searchParams.getAll('teamId');
const projectIds = searchParams.getAll('projectId');
const tagIds = searchParams.getAll('tagId');
const data = await getTimeSeriesData({
startTime,
endTime,
linkId: searchParams.get('linkId') || undefined,
granularity: (searchParams.get('granularity') || 'day') as 'hour' | 'day' | 'week' | 'month'
granularity: (searchParams.get('granularity') || 'day') as 'hour' | 'day' | 'week' | 'month',
// 添加团队、项目和标签筛选
teamIds: teamIds.length > 0 ? teamIds : undefined,
projectIds: projectIds.length > 0 ? projectIds : undefined,
tagIds: tagIds.length > 0 ? tagIds : undefined,
// 添加子路径筛选
subpath: searchParams.get('subpath') || undefined
});
const response: ApiResponse<typeof data> = {

View File

@@ -0,0 +1,208 @@
# 事件跟踪接口说明
## 概述
该接口用于跟踪用户交互事件并将数据存储到 ClickHouse 数据库中。支持记录各种类型的事件,并可包含与链接、用户、团队、项目等相关的详细信息。
## 接口信息
- **URL**: `/api/events/track`
- **方法**: `POST`
- **Content-Type**: `application/json`
## 请求参数
### 必填字段
| 参数 | 类型 | 描述 |
|------|------|------|
| `event_type` | string | 事件类型,如 'click', 'view', 'conversion' |
### 核心事件字段
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `event_id` | string | 否 | 事件唯一标识符不提供时自动生成UUID |
| `event_time` | string/Date | 否 | 事件发生时间格式为ISO日期字符串默认为当前时间 |
| `event_attributes` | object/string | 否 | 事件相关的其他属性可以是JSON对象或JSON字符串 |
### 链接信息
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `link_id` | string | 否 | 短链接的唯一ID |
| `link_slug` | string | 否 | 短链接的slug部分 |
| `link_label` | string | 否 | 短链接的显示名称 |
| `link_title` | string | 否 | 短链接的标题 |
| `link_original_url` | string | 否 | 原始目标URL |
| `link_attributes` | object/string | 否 | 链接相关的额外属性 |
| `link_created_at` | string/Date | 否 | 链接创建时间 |
| `link_expires_at` | string/Date | 否 | 链接过期时间 |
| `link_tags` | array/string | 否 | 链接标签可以是数组或JSON字符串 |
### 用户信息
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `user_id` | string | 否 | 用户ID |
| `user_name` | string | 否 | 用户名称 |
| `user_email` | string | 否 | 用户邮箱 |
| `user_attributes` | object/string | 否 | 用户相关的其他属性 |
### 团队和项目信息
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `team_id` | string | 否 | 团队ID |
| `team_name` | string | 否 | 团队名称 |
| `team_attributes` | object/string | 否 | 团队相关的其他属性 |
| `project_id` | string | 否 | 项目ID |
| `project_name` | string | 否 | 项目名称 |
| `project_attributes` | object/string | 否 | 项目相关的其他属性 |
### 二维码信息
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `qr_code_id` | string | 否 | 二维码ID |
| `qr_code_name` | string | 否 | 二维码名称 |
| `qr_code_attributes` | object/string | 否 | 二维码相关的其他属性 |
### 访问者信息
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `visitor_id` | string | 否 | 访问者唯一标识符,不提供时自动生成 |
| `session_id` | string | 否 | 会话ID不提供时自动生成 |
| `ip_address` | string | 否 | 访问者IP地址默认从请求头获取 |
| `country` | string | 否 | 访问者所在国家 |
| `city` | string | 否 | 访问者所在城市 |
| `device_type` | string | 否 | 设备类型 (如 desktop, mobile, tablet) |
| `browser` | string | 否 | 浏览器名称 |
| `os` | string | 否 | 操作系统 |
| `user_agent` | string | 否 | 用户代理字符串,默认从请求头获取 |
### 引荐来源信息
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `referrer` | string | 否 | 引荐URL默认从请求头获取 |
| `utm_source` | string | 否 | UTM来源参数 |
| `utm_medium` | string | 否 | UTM媒介参数 |
| `utm_campaign` | string | 否 | UTM活动参数 |
| `utm_term` | string | 否 | UTM术语参数 |
| `utm_content` | string | 否 | UTM内容参数 |
### 交互信息
| 参数 | 类型 | 必填 | 描述 |
|------|------|------|------|
| `time_spent_sec` | number | 否 | 用户在页面上停留的时间默认0 |
| `is_bounce` | boolean | 否 | 是否是跳出只访问一个页面默认true |
| `is_qr_scan` | boolean | 否 | 是否来自二维码扫描默认false |
| `conversion_type` | string | 否 | 转化类型 |
| `conversion_value` | number | 否 | 转化价值默认0 |
## 响应格式
### 成功响应 (201 Created)
```json
{
"success": true,
"message": "Event tracked successfully",
"event_id": "uuid-of-tracked-event"
}
```
### 错误响应
#### 缺少必填字段 (400 Bad Request)
```json
{
"error": "Missing required field: event_type"
}
```
#### 服务器错误 (500 Internal Server Error)
```json
{
"error": "Failed to track event",
"details": "具体错误信息"
}
```
## 使用示例
### 基本事件跟踪请求
```javascript
fetch('/api/events/track', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
event_type: 'click',
link_id: 'abc123',
link_slug: 'promo-summer',
link_original_url: 'https://example.com/summer-promotion'
})
})
```
### 详细事件跟踪请求
```javascript
fetch('/api/events/track', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
event_type: 'conversion',
link_id: 'abc123',
link_slug: 'promo-summer',
link_original_url: 'https://example.com/summer-promotion',
event_attributes: {
page: '/checkout',
product_id: 'xyz789'
},
user_id: 'user123',
team_id: 'team456',
project_id: 'proj789',
visitor_id: 'vis987',
is_bounce: false,
time_spent_sec: 120,
conversion_type: 'purchase',
conversion_value: 99.99,
utm_source: 'email',
utm_campaign: 'summer_sale'
})
})
```
## 注意事项
- 所有对象类型的字段(如 `event_attributes`可以作为对象或预先格式化的JSON字符串传递
- 如果不提供 `event_id``visitor_id``session_id`,系统将自动生成
- 时间戳字段接受ISO格式的日期字符串并会被转换为ClickHouse兼容的格式
UTM 测试示例。1. 电子邮件营销链接
https://short.domain.com/summer?utm_source=newsletter&utm_medium=email&utm_campaign=summer_promo&utm_term=discount&utm_content=header
说明: 用于电子邮件营销活动,跟踪用户从邮件头部横幅点击的流量。
2. 社交媒体广告链接
https://short.domain.com/product?utm_source=instagram&utm_medium=social&utm_campaign=fall_collection&utm_content=story
说明: 用于 Instagram Story 广告,跟踪用户从社交媒体故事广告点击的情况。
3. 搜索引擎广告链接
https://short.domain.com/service?utm_source=google&utm_medium=cpc&utm_campaign=brand_terms&utm_term=service+name
说明: 用于 Google Ads 广告,跟踪用户从搜索引擎付费广告点击的流量,特别是针对特定搜索词。
4. QR 码链接
https://short.domain.com/event?utm_source=flyer&utm_medium=print&utm_campaign=local_event&utm_content=qr_code&source=qr
说明: 用于打印材料上的 QR 码,跟踪用户扫描实体宣传资料的情况。
5. 合作伙伴引荐链接
https://short.domain.com/partner?utm_source=affiliate&utm_medium=referral&utm_campaign=partner_program&utm_content=banner
说明: 用于合作伙伴网站上的推广横幅,跟踪来自联盟营销的转化率。
https://upj.to/5seaii?utm_source=newsletter&utm_medium=email&utm_campaign=summer_promo&utm_term=discount&utm_content=header
https://upj.to/5seaii?utm_source=instagram&utm_medium=social&utm_campaign=fall_collection&utm_content=story
https://upj.to/5seaii?utm_source=google&utm_medium=cpc&utm_campaign=brand_terms&utm_term=service+name
https://upj.to/5seaii?utm_source=flyer&utm_medium=print&utm_campaign=local_event&utm_content=qr_code&source=qr
https://upj.to/5seaii?utm_source=affiliate&utm_medium=referral&utm_campaign=partner_program&utm_content=banner

View File

@@ -0,0 +1,139 @@
import { NextRequest, NextResponse } from 'next/server';
import { Event } from '../../types';
import { v4 as uuid } from 'uuid';
import clickhouse from '@/lib/clickhouse';
// 将时间格式化为ClickHouse兼容的格式YYYY-MM-DD HH:MM:SS.SSS
const formatDateTime = (date: Date) => {
return date.toISOString().replace('T', ' ').replace('Z', '');
};
// Handler for POST request to track events
export async function POST(req: NextRequest) {
try {
// Parse request body
const eventData = await req.json();
// Validate required fields
if (!eventData.event_type) {
return NextResponse.json(
{ error: 'Missing required field: event_type' },
{ status: 400 }
);
}
// 获取当前时间并格式化
const currentTime = formatDateTime(new Date());
// Set default values for required fields if missing
const event: Event = {
// Core event fields
event_id: eventData.event_id || uuid(),
event_time: eventData.event_time ? formatDateTime(new Date(eventData.event_time)) : currentTime,
event_type: eventData.event_type,
event_attributes: eventData.event_attributes || '{}',
// Link information
link_id: eventData.link_id || '',
link_slug: eventData.link_slug || '',
link_label: eventData.link_label || '',
link_title: eventData.link_title || '',
link_original_url: eventData.link_original_url || '',
link_attributes: eventData.link_attributes || '{}',
link_created_at: eventData.link_created_at ? formatDateTime(new Date(eventData.link_created_at)) : currentTime,
link_expires_at: eventData.link_expires_at ? formatDateTime(new Date(eventData.link_expires_at)) : null,
link_tags: eventData.link_tags || '[]',
// User information
user_id: eventData.user_id || '',
user_name: eventData.user_name || '',
user_email: eventData.user_email || '',
user_attributes: eventData.user_attributes || '{}',
// Team information
team_id: eventData.team_id || '',
team_name: eventData.team_name || '',
team_attributes: eventData.team_attributes || '{}',
// Project information
project_id: eventData.project_id || '',
project_name: eventData.project_name || '',
project_attributes: eventData.project_attributes || '{}',
// QR code information
qr_code_id: eventData.qr_code_id || '',
qr_code_name: eventData.qr_code_name || '',
qr_code_attributes: eventData.qr_code_attributes || '{}',
// Visitor information
visitor_id: eventData.visitor_id || uuid(),
session_id: eventData.session_id || uuid(),
ip_address: eventData.ip_address || req.headers.get('x-forwarded-for')?.toString() || '',
country: eventData.country || '',
city: eventData.city || '',
device_type: eventData.device_type || '',
browser: eventData.browser || '',
os: eventData.os || '',
user_agent: eventData.user_agent || req.headers.get('user-agent')?.toString() || '',
// Referrer information
referrer: eventData.referrer || req.headers.get('referer')?.toString() || '',
utm_source: eventData.utm_source || '',
utm_medium: eventData.utm_medium || '',
utm_campaign: eventData.utm_campaign || '',
utm_term: eventData.utm_term || '',
utm_content: eventData.utm_content || '',
// Interaction information
time_spent_sec: eventData.time_spent_sec || 0,
is_bounce: eventData.is_bounce !== undefined ? eventData.is_bounce : true,
is_qr_scan: eventData.is_qr_scan !== undefined ? eventData.is_qr_scan : false,
conversion_type: eventData.conversion_type || '',
conversion_value: eventData.conversion_value || 0,
};
// 确保JSON字符串字段的正确处理
if (typeof event.event_attributes === 'object') {
event.event_attributes = JSON.stringify(event.event_attributes);
}
if (typeof event.link_attributes === 'object') {
event.link_attributes = JSON.stringify(event.link_attributes);
}
if (typeof event.user_attributes === 'object') {
event.user_attributes = JSON.stringify(event.user_attributes);
}
if (typeof event.team_attributes === 'object') {
event.team_attributes = JSON.stringify(event.team_attributes);
}
if (typeof event.project_attributes === 'object') {
event.project_attributes = JSON.stringify(event.project_attributes);
}
if (typeof event.qr_code_attributes === 'object') {
event.qr_code_attributes = JSON.stringify(event.qr_code_attributes);
}
if (typeof event.link_tags === 'object') {
event.link_tags = JSON.stringify(event.link_tags);
}
// Insert event into ClickHouse
await clickhouse.insert({
table: 'events',
values: [event],
format: 'JSONEachRow',
});
// Return success response
return NextResponse.json({
success: true,
message: 'Event tracked successfully',
event_id: event.event_id
}, { status: 201 });
} catch (error) {
console.error('Error tracking event:', error);
return NextResponse.json(
{ error: 'Failed to track event', details: (error as Error).message },
{ status: 500 }
);
}
}

201
app/api/events/utm/route.ts Normal file
View File

@@ -0,0 +1,201 @@
import { NextRequest, NextResponse } from 'next/server';
import clickhouse from '@/lib/clickhouse';
import type { ApiResponse } from '@/lib/types';
interface UtmData {
utm_value: string;
clicks: number;
visitors: number;
avg_time_spent: number;
bounces: number;
conversions: number;
}
// 辅助函数,将日期格式化为标准格式
function formatDateTime(dateString: string): string {
const date = new Date(dateString);
return date.toISOString().split('.')[0];
}
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
// 获取过滤参数
const startTime = searchParams.get('startTime');
const endTime = searchParams.get('endTime');
const linkId = searchParams.get('linkId');
const subpath = searchParams.get('subpath');
// 获取团队、项目和标签筛选参数
const teamIds = searchParams.getAll('teamId');
const projectIds = searchParams.getAll('projectId');
const tagIds = searchParams.getAll('tagId');
const tagNames = searchParams.getAll('tagName');
// 获取UTM类型参数
const utmType = searchParams.get('utmType') || 'source';
// 添加调试日志
console.log('UTM API received parameters:', {
startTime,
endTime,
linkId,
subpath,
teamIds,
projectIds,
tagIds,
tagNames,
utmType,
url: request.url
});
// 构建WHERE子句
let whereClause = '';
const conditions = [];
if (startTime) {
conditions.push(`event_time >= toDateTime('${formatDateTime(startTime)}')`);
}
if (endTime) {
conditions.push(`event_time <= toDateTime('${formatDateTime(endTime)}')`);
}
if (linkId) {
conditions.push(`link_id = '${linkId}'`);
}
// 添加子路径筛选 - 使用更精确的匹配方式
if (subpath && subpath.trim() !== '') {
console.log('====== UTM API SUBPATH DEBUG ======');
console.log('Raw subpath param:', subpath);
// 清理并准备subpath值
let cleanSubpath = subpath.trim();
// 移除开头的斜杠以便匹配
if (cleanSubpath.startsWith('/')) {
cleanSubpath = cleanSubpath.substring(1);
}
// 移除结尾的斜杠以便匹配
if (cleanSubpath.endsWith('/')) {
cleanSubpath = cleanSubpath.substring(0, cleanSubpath.length - 1);
}
console.log('Cleaned subpath:', cleanSubpath);
// 使用正则表达式匹配URL中的第二个路径部分
// 示例: 在 "https://abc.com/slug/subpath/" 中匹配 "subpath"
const condition = `match(JSONExtractString(event_attributes, 'full_url'), '/[^/]+/${cleanSubpath}(/|\\\\?|$)')`;
console.log('Final SQL condition:', condition);
console.log('==================================');
conditions.push(condition);
}
// 添加团队筛选
if (teamIds && teamIds.length > 0) {
// 如果只有一个团队ID
if (teamIds.length === 1) {
conditions.push(`team_id = '${teamIds[0]}'`);
} else {
// 多个团队ID
conditions.push(`team_id IN ('${teamIds.join("','")}')`);
}
}
// 添加项目筛选
if (projectIds && projectIds.length > 0) {
// 如果只有一个项目ID
if (projectIds.length === 1) {
conditions.push(`project_id = '${projectIds[0]}'`);
} else {
// 多个项目ID
conditions.push(`project_id IN ('${projectIds.join("','")}')`);
}
}
// 添加标签筛选
if ((tagIds && tagIds.length > 0) || (tagNames && tagNames.length > 0)) {
// 优先使用tagNames如果有的话
const tagsToUse = tagNames.length > 0 ? tagNames : tagIds;
// 使用与buildFilter函数相同的处理方式
const tagConditions = tagsToUse.map(tag =>
`link_tags LIKE '%${tag}%'`
);
conditions.push(`(${tagConditions.join(' OR ')})`);
}
if (conditions.length > 0) {
whereClause = `WHERE ${conditions.join(' AND ')}`;
}
// 确定要分组的UTM字段
let utmField;
switch (utmType) {
case 'source':
utmField = 'utm_source';
break;
case 'medium':
utmField = 'utm_medium';
break;
case 'campaign':
utmField = 'utm_campaign';
break;
case 'term':
utmField = 'utm_term';
break;
case 'content':
utmField = 'utm_content';
break;
default:
utmField = 'utm_source';
}
// 构建SQL查询
const query = `
SELECT
${utmField} AS utm_value,
COUNT(*) AS clicks,
uniqExact(visitor_id) AS visitors,
round(AVG(time_spent_sec), 2) AS avg_time_spent,
countIf(is_bounce = 1) AS bounces,
countIf(conversion_type IN ('visit', 'stay', 'interact', 'signup', 'subscription', 'purchase')) AS conversions
FROM shorturl_analytics.events
${whereClause}
${whereClause ? 'AND' : 'WHERE'} ${utmField} != ''
GROUP BY utm_value
ORDER BY clicks DESC
LIMIT 100
`;
// 执行查询
const result = await clickhouse.query({
query,
format: 'JSONEachRow',
});
// 获取查询结果
const rows = await result.json();
const data = rows as UtmData[];
// 返回数据
const response: ApiResponse<UtmData[]> = {
success: true,
data
};
return NextResponse.json(response);
} catch (error) {
console.error('Error fetching UTM data:', error);
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

248
app/api/geo/batch/route.ts Normal file
View File

@@ -0,0 +1,248 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse } from '@/lib/types';
interface IpLocationData {
ip: string;
country_name: string;
country_code: string;
city: string;
region: string;
continent_code: string;
continent_name: string;
latitude: number;
longitude: number;
timestamp: number;
}
// Simple in-memory cache on the server side to reduce API calls
const serverCache: Record<string, IpLocationData> = {};
// Cache for IPs that have repeatedly failed to resolve
const failedIPsCache: Record<string, { attempts: number, lastAttempt: number }> = {};
// Cache expiration time (30 days in milliseconds)
const CACHE_EXPIRATION = 30 * 24 * 60 * 60 * 1000; // 30 days
// Max attempts to fetch an IP before considering it permanently failed
const MAX_RETRY_ATTEMPTS = 3;
// Retry timeout - how long to wait before trying a failed IP again (24 hours)
const RETRY_TIMEOUT = 24 * 60 * 60 * 1000;
/**
* Check if an IP has failed too many times and should be skipped
*/
function shouldSkipIP(ip: string): boolean {
if (!failedIPsCache[ip]) return false;
const now = Date.now();
// Skip if max attempts reached
if (failedIPsCache[ip].attempts >= MAX_RETRY_ATTEMPTS) {
return true;
}
// Skip if last attempt was recent
if (now - failedIPsCache[ip].lastAttempt < RETRY_TIMEOUT) {
return true;
}
return false;
}
/**
* Mark an IP as failed
*/
function markIPAsFailed(ip: string): void {
const now = Date.now();
if (failedIPsCache[ip]) {
failedIPsCache[ip] = {
attempts: failedIPsCache[ip].attempts + 1,
lastAttempt: now
};
} else {
failedIPsCache[ip] = {
attempts: 1,
lastAttempt: now
};
}
}
/**
* Get location data for a single IP using ipapi.co
*/
async function fetchIpLocation(ip: string): Promise<IpLocationData | null> {
try {
// Skip this IP if it has failed too many times
if (shouldSkipIP(ip)) {
console.log(`[Server] Skipping blacklisted IP: ${ip}`);
return null;
}
// Check server cache first
const now = Date.now();
if (serverCache[ip] && (now - serverCache[ip].timestamp) < CACHE_EXPIRATION) {
return serverCache[ip];
}
// Add delay to avoid rate limiting (100 requests per minute max)
await new Promise(resolve => setTimeout(resolve, 600)); // ~100 req/min = 1 req per 600ms
const response = await fetch(`https://ipapi.co/${ip}/json/`);
if (!response.ok) {
console.error(`Error fetching location for IP ${ip}: ${response.statusText}`);
markIPAsFailed(ip);
return null;
}
const data = await response.json();
if (data.error) {
console.error(`Error fetching location for IP ${ip}: ${data.reason}`);
markIPAsFailed(ip);
return null;
}
// Reset failed status if successful
if (failedIPsCache[ip]) {
delete failedIPsCache[ip];
}
const locationData: IpLocationData = {
ip: data.ip,
country_name: data.country_name || 'Unknown',
country_code: data.country_code || 'UN',
city: data.city || 'Unknown',
region: data.region || 'Unknown',
continent_code: data.continent_code || 'UN',
continent_name: getContinentName(data.continent_code) || 'Unknown',
latitude: data.latitude || 0,
longitude: data.longitude || 0,
timestamp: Date.now()
};
// Cache the result
serverCache[ip] = locationData;
return locationData;
} catch (error) {
console.error(`Error fetching location for IP ${ip}:`, error);
markIPAsFailed(ip);
return null;
}
}
/**
* Get continent name from continent code
*/
function getContinentName(code?: string): string {
if (!code) return 'Unknown';
const continents: Record<string, string> = {
'AF': 'Africa',
'AN': 'Antarctica',
'AS': 'Asia',
'EU': 'Europe',
'NA': 'North America',
'OC': 'Oceania',
'SA': 'South America'
};
return continents[code] || 'Unknown';
}
/**
* API route handler for batch IP location lookups
*/
export async function POST(request: NextRequest) {
try {
const { ips } = await request.json();
if (!ips || !Array.isArray(ips) || ips.length === 0) {
return NextResponse.json({
success: false,
error: 'Invalid or empty IP list'
}, { status: 400 });
}
// Limit batch size to 50 IPs to prevent abuse
const ipList = ips.slice(0, 50);
const results: Record<string, IpLocationData | null> = {};
// Filter out IPs that should be skipped
const validIPs = ipList.filter(ip => {
if (typeof ip !== 'string' || !ip.trim()) return false;
if (isPrivateIP(ip)) {
results[ip] = getPrivateIPData(ip);
return false;
}
if (shouldSkipIP(ip)) {
console.log(`[Server] Skipping blacklisted IP: ${ip}`);
results[ip] = null;
return false;
}
return true;
});
// Process remaining IPs sequentially to respect rate limits
for (const ip of validIPs) {
results[ip] = await fetchIpLocation(ip);
}
const response: ApiResponse<Record<string, IpLocationData | null>> = {
success: true,
data: results
};
return NextResponse.json(response);
} catch (error) {
console.error('Batch IP lookup error:', error);
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}
/**
* Check if IP is a private/local address
*/
function isPrivateIP(ip: string): boolean {
return (
ip.startsWith('10.') ||
ip.startsWith('192.168.') ||
ip.startsWith('172.16.') ||
ip.startsWith('172.17.') ||
ip.startsWith('172.18.') ||
ip.startsWith('172.19.') ||
ip.startsWith('172.20.') ||
ip.startsWith('172.21.') ||
ip.startsWith('172.22.') ||
ip.startsWith('127.') ||
ip === 'localhost' ||
ip === '::1'
);
}
/**
* Generate location data for private IP addresses
*/
function getPrivateIPData(ip: string): IpLocationData {
return {
ip,
country_name: 'Local Network',
country_code: 'LO',
city: 'Local',
region: 'Local',
continent_code: 'LO',
continent_name: 'Local',
latitude: 0,
longitude: 0,
timestamp: Date.now()
};
}

View File

@@ -0,0 +1,141 @@
import { NextRequest, NextResponse } from 'next/server';
import { executeQuery } from '@/lib/clickhouse';
import type { ApiResponse } from '@/lib/types';
export async function GET(
request: NextRequest,
{ params }: { params: { id: string } }
) {
try {
// Get the id from the URL parameters
const { id } = params;
if (!id) {
return NextResponse.json({
success: false,
error: 'ID parameter is required'
}, { status: 400 });
}
console.log('Fetching shortlink by ID:', id);
// Query to fetch a single shortlink by id
const query = `
SELECT
id,
external_id,
type,
slug,
original_url,
title,
description,
attributes,
schema_version,
creator_id,
creator_email,
creator_name,
created_at,
updated_at,
deleted_at,
projects,
teams,
tags,
qr_codes AS qr_codes,
channels,
favorites,
expires_at,
click_count,
unique_visitors,
domain
FROM shorturl_analytics.shorturl
WHERE id = '${id}' AND deleted_at IS NULL
LIMIT 1
`;
console.log('Executing query:', query);
// Execute the query
const result = await executeQuery(query);
// If no shortlink found with the specified ID
if (!Array.isArray(result) || result.length === 0) {
return NextResponse.json({
success: false,
error: 'Shortlink not found'
}, { status: 404 });
}
// Process the shortlink data
const shortlink = result[0] as any;
// Extract shortUrl from attributes
let shortUrl = '';
try {
if (shortlink.attributes && typeof shortlink.attributes === 'string') {
const attributes = JSON.parse(shortlink.attributes) as { shortUrl?: string };
shortUrl = attributes.shortUrl || '';
}
} catch (e) {
console.error('Error parsing shortlink attributes:', e);
}
// Process teams
let teams: any[] = [];
try {
if (shortlink.teams && typeof shortlink.teams === 'string') {
teams = JSON.parse(shortlink.teams);
}
} catch (e) {
console.error('Error parsing teams:', e);
}
// Process tags
let tags: any[] = [];
try {
if (shortlink.tags && typeof shortlink.tags === 'string') {
tags = JSON.parse(shortlink.tags);
}
} catch (e) {
console.error('Error parsing tags:', e);
}
// Process projects
let projects: any[] = [];
try {
if (shortlink.projects && typeof shortlink.projects === 'string') {
projects = JSON.parse(shortlink.projects);
}
} catch (e) {
console.error('Error parsing projects:', e);
}
// Format the data to match what our store expects
const formattedShortlink = {
id: shortlink.id || '',
externalId: shortlink.external_id || '',
slug: shortlink.slug || '',
originalUrl: shortlink.original_url || '',
title: shortlink.title || '',
shortUrl: shortUrl,
teams: teams,
projects: projects,
tags: tags.map((tag: any) => tag.tag_name || ''),
createdAt: shortlink.created_at,
domain: shortlink.domain || (shortUrl ? new URL(shortUrl).hostname : '')
};
const response: ApiResponse<typeof formattedShortlink> = {
success: true,
data: formattedShortlink
};
return NextResponse.json(response);
} catch (error) {
console.error('Error fetching shortlink by ID:', error);
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -0,0 +1,143 @@
import { NextRequest, NextResponse } from 'next/server';
import { executeQuery } from '@/lib/clickhouse';
import type { ApiResponse } from '@/lib/types';
export async function GET(request: NextRequest) {
try {
// Get the url from query parameters
const searchParams = request.nextUrl.searchParams;
const url = searchParams.get('url');
if (!url) {
return NextResponse.json({
success: false,
error: 'URL parameter is required'
}, { status: 400 });
}
console.log('Fetching shortlink by URL:', url);
// Query to fetch a single shortlink by shortUrl in attributes
const query = `
SELECT
id,
external_id,
type,
slug,
original_url,
title,
description,
attributes,
schema_version,
creator_id,
creator_email,
creator_name,
created_at,
updated_at,
deleted_at,
projects,
teams,
tags,
qr_codes AS qr_codes,
channels,
favorites,
expires_at,
click_count,
unique_visitors,
domain
FROM shorturl_analytics.shorturl
WHERE JSONHas(attributes, 'shortUrl')
AND JSONExtractString(attributes, 'shortUrl') = '${url}'
AND deleted_at IS NULL
LIMIT 1
`;
console.log('Executing query:', query);
// Execute the query
const result = await executeQuery(query);
// If no shortlink found with the specified URL
if (!Array.isArray(result) || result.length === 0) {
return NextResponse.json({
success: false,
error: 'Shortlink not found'
}, { status: 404 });
}
// Process the shortlink data
const shortlink = result[0];
// Extract shortUrl from attributes
let shortUrl = '';
try {
if (shortlink.attributes && typeof shortlink.attributes === 'string') {
const attributes = JSON.parse(shortlink.attributes);
shortUrl = attributes.shortUrl || '';
}
} catch (e) {
console.error('Error parsing shortlink attributes:', e);
}
// Process teams
let teams = [];
try {
if (shortlink.teams && typeof shortlink.teams === 'string') {
teams = JSON.parse(shortlink.teams);
}
} catch (e) {
console.error('Error parsing teams:', e);
}
// Process tags
let tags = [];
try {
if (shortlink.tags && typeof shortlink.tags === 'string') {
tags = JSON.parse(shortlink.tags);
}
} catch (e) {
console.error('Error parsing tags:', e);
}
// Process projects
let projects = [];
try {
if (shortlink.projects && typeof shortlink.projects === 'string') {
projects = JSON.parse(shortlink.projects);
}
} catch (e) {
console.error('Error parsing projects:', e);
}
// Format the data to match what our store expects
const formattedShortlink = {
id: shortlink.id || '',
externalId: shortlink.external_id || '',
slug: shortlink.slug || '',
originalUrl: shortlink.original_url || '',
title: shortlink.title || '',
shortUrl: shortUrl,
teams: teams,
projects: projects,
tags: tags.map((tag) => tag.tag_name || ''),
createdAt: shortlink.created_at,
domain: shortlink.domain || (shortUrl ? new URL(shortUrl).hostname : '')
};
console.log('Shortlink data formatted with externalId:', shortlink.external_id, 'Final object:', formattedShortlink);
const response: ApiResponse<typeof formattedShortlink> = {
success: true,
data: formattedShortlink
};
return NextResponse.json(response);
} catch (error) {
console.error('Error fetching shortlink by URL:', error);
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -0,0 +1,143 @@
import { NextRequest, NextResponse } from 'next/server';
import { executeQuery } from '@/lib/clickhouse';
import type { ApiResponse } from '@/lib/types';
export async function GET(request: NextRequest) {
try {
// Get the url from query parameters
const searchParams = request.nextUrl.searchParams;
const shortUrl = searchParams.get('shortUrl');
if (!shortUrl) {
return NextResponse.json({
success: false,
error: 'shortUrl parameter is required'
}, { status: 400 });
}
console.log('Fetching shortlink by exact shortUrl:', shortUrl);
// Query to fetch a single shortlink by shortUrl in attributes
const query = `
SELECT
id,
external_id,
type,
slug,
original_url,
title,
description,
attributes,
schema_version,
creator_id,
creator_email,
creator_name,
created_at,
updated_at,
deleted_at,
projects,
teams,
tags,
qr_codes AS qr_codes,
channels,
favorites,
expires_at,
click_count,
unique_visitors,
domain
FROM shorturl_analytics.shorturl
WHERE JSONHas(attributes, 'shortUrl')
AND JSONExtractString(attributes, 'shortUrl') = '${shortUrl}'
AND deleted_at IS NULL
LIMIT 1
`;
console.log('Executing query:', query);
// Execute the query
const result = await executeQuery(query);
// If no shortlink found with the specified URL
if (!Array.isArray(result) || result.length === 0) {
return NextResponse.json({
success: false,
error: 'Shortlink not found'
}, { status: 404 });
}
// Process the shortlink data
const shortlink = result[0] as Record<string, any>;
// Extract shortUrl from attributes
let shortUrlValue = '';
try {
if (shortlink.attributes && typeof shortlink.attributes === 'string') {
const attributes = JSON.parse(shortlink.attributes) as { shortUrl?: string };
shortUrlValue = attributes.shortUrl || '';
}
} catch (e) {
console.error('Error parsing shortlink attributes:', e);
}
// Process teams
let teams: any[] = [];
try {
if (shortlink.teams && typeof shortlink.teams === 'string') {
teams = JSON.parse(shortlink.teams);
}
} catch (e) {
console.error('Error parsing teams:', e);
}
// Process tags
let tags: any[] = [];
try {
if (shortlink.tags && typeof shortlink.tags === 'string') {
tags = JSON.parse(shortlink.tags);
}
} catch (e) {
console.error('Error parsing tags:', e);
}
// Process projects
let projects: any[] = [];
try {
if (shortlink.projects && typeof shortlink.projects === 'string') {
projects = JSON.parse(shortlink.projects);
}
} catch (e) {
console.error('Error parsing projects:', e);
}
// Format the data to match what our store expects
const formattedShortlink = {
id: shortlink.id || '',
externalId: shortlink.external_id || '',
slug: shortlink.slug || '',
originalUrl: shortlink.original_url || '',
title: shortlink.title || '',
shortUrl: shortUrlValue,
teams: teams,
projects: projects,
tags: tags.map((tag: any) => tag.tag_name || ''),
createdAt: shortlink.created_at,
domain: shortlink.domain || (shortUrlValue ? new URL(shortUrlValue).hostname : '')
};
console.log('Formatted shortlink with externalId:', shortlink.external_id);
const response: ApiResponse<typeof formattedShortlink> = {
success: true,
data: formattedShortlink
};
return NextResponse.json(response);
} catch (error) {
console.error('Error fetching shortlink by exact URL:', error);
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

104
app/api/shortlinks/route.ts Normal file
View File

@@ -0,0 +1,104 @@
import { NextResponse } from 'next/server';
import { executeQuery } from '@/lib/clickhouse';
import { NextRequest } from 'next/server';
export async function GET(request: NextRequest) {
try {
// Get pagination and filter parameters from the URL
const searchParams = request.nextUrl.searchParams;
const page = parseInt(searchParams.get('page') || '1', 10);
const pageSize = parseInt(searchParams.get('page_size') || '10', 10);
const search = searchParams.get('search');
const team = searchParams.get('team');
// Calculate OFFSET
const offset = (page - 1) * pageSize;
// Build WHERE conditions
const whereConditions = ['deleted_at IS NULL'];
if (search) {
// Expand search to include more fields: slug, shortUrl in attributes, team name, tag name, original_url
whereConditions.push(`(
slug ILIKE '%${search}%' OR
original_url ILIKE '%${search}%' OR
title ILIKE '%${search}%' OR
JSONHas(attributes, 'shortUrl') AND JSONExtractString(attributes, 'shortUrl') ILIKE '%${search}%' OR
arrayExists(x -> JSONExtractString(x, 'team_name') ILIKE '%${search}%', JSONExtractArrayRaw(teams)) OR
arrayExists(x -> JSONExtractString(x, 'tag_name') ILIKE '%${search}%', JSONExtractArrayRaw(tags))
)`);
}
if (team) {
whereConditions.push(`arrayExists(x -> JSONExtractString(x, 'team_id') = '${team}', JSONExtractArrayRaw(teams))`);
}
const whereClause = whereConditions.join(' AND ');
// First query to get total count
const countQuery = `
SELECT count(*) as total
FROM shorturl_analytics.shorturl
WHERE ${whereClause}
`;
const countResult = await executeQuery(countQuery);
// Handle the result safely by using an explicit type check
const total = Array.isArray(countResult) && countResult.length > 0 && typeof countResult[0] === 'object' && countResult[0] !== null && 'total' in countResult[0]
? Number(countResult[0].total)
: 0;
const totalPages = Math.ceil(total / pageSize);
// Main query with pagination
const query = `
SELECT
id,
external_id,
type,
slug,
original_url,
title,
description,
attributes,
schema_version,
creator_id,
creator_email,
creator_name,
created_at,
updated_at,
deleted_at,
projects,
teams,
tags,
qr_codes AS qr_codes,
channels,
favorites,
expires_at,
click_count,
unique_visitors,
domain
FROM shorturl_analytics.shorturl
WHERE ${whereClause}
ORDER BY created_at DESC
LIMIT ${pageSize} OFFSET ${offset}
`;
// Execute the query using the shared client
const rows = await executeQuery(query);
// Return the data with pagination metadata
return NextResponse.json({
links: rows,
total: total,
total_pages: totalPages,
page: page,
page_size: pageSize
});
} catch (error) {
console.error('Error fetching shortlinks from ClickHouse:', error);
return NextResponse.json(
{ error: 'Failed to fetch shortlinks' },
{ status: 500 }
);
}
}

View File

@@ -0,0 +1,41 @@
import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
import { cookies } from 'next/headers';
import { NextResponse } from 'next/server';
export async function GET() {
try {
const supabase = createRouteHandlerClient({ cookies });
// 获取当前用户
const { data: { user }, error: userError } = await supabase.auth.getUser();
if (userError || !user) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
}
// 获取用户所属的所有团队
const { data: teams, error: teamsError } = await supabase
.from('teams')
.select(`
id,
name,
description,
avatar_url
`)
.innerJoin('team_membership', 'teams.id = team_membership.team_id')
.eq('team_membership.user_id', user.id)
.is('teams.deleted_at', null);
if (teamsError) {
console.error('Error fetching teams:', teamsError);
return NextResponse.json({ error: 'Failed to fetch teams' }, { status: 500 });
}
return NextResponse.json(teams);
} catch (error) {
console.error('Error in /api/teams/list:', error);
return NextResponse.json(
{ error: 'Internal server error' },
{ status: 500 }
);
}
}

168
app/api/types.ts Normal file
View File

@@ -0,0 +1,168 @@
// Event Types
export interface Event {
// 核心事件信息
event_id: string;
event_time: string;
event_type: string;
event_attributes: string;
// 链接信息
link_id: string;
link_slug: string;
link_label: string;
link_title: string;
link_original_url: string;
link_attributes: string;
link_created_at: string;
link_expires_at: string | null;
link_tags: string;
// 用户信息
user_id: string;
user_name: string;
user_email: string;
user_attributes: string;
// 团队信息
team_id: string;
team_name: string;
team_attributes: string;
// 项目信息
project_id: string;
project_name: string;
project_attributes: string;
// 二维码信息
qr_code_id: string;
qr_code_name: string;
qr_code_attributes: string;
// 访问者信息
visitor_id: string;
session_id: string;
ip_address: string;
country: string;
city: string;
device_type: string;
browser: string;
os: string;
user_agent: string;
// 来源信息
referrer: string;
utm_source: string;
utm_medium: string;
utm_campaign: string;
utm_term: string;
utm_content: string;
// 交互信息
time_spent_sec: number;
is_bounce: boolean;
is_qr_scan: boolean;
conversion_type: string;
conversion_value: number;
// 旧接口兼容字段
id?: string;
time?: string;
type?: string;
linkInfo?: {
id: string;
shortUrl: string;
originalUrl: string;
};
visitor?: {
id: string;
browser: string;
os: string;
device: string;
};
location?: {
country: string;
region: string;
city: string;
};
}
// Analytics Types
export interface TimeSeriesData {
timestamp: string;
events: number;
visitors: number;
conversions: number;
}
export interface GeoData {
location: string;
area: string;
visits: number;
visitors: number;
percentage: number;
}
export type DeviceType = 'mobile' | 'desktop' | 'tablet' | 'other';
export interface DeviceAnalytics {
deviceTypes: {
type: string;
count: number;
percentage: number;
}[];
browsers: {
name: string;
count: number;
percentage: number;
}[];
operatingSystems: {
name: string;
count: number;
percentage: number;
}[];
}
export interface EventsSummary {
totalEvents: number;
uniqueVisitors: number;
totalConversions: number;
averageTimeSpent: number;
deviceTypes: {
mobile: number;
desktop: number;
tablet: number;
other: number;
};
browsers: {
name: string;
count: number;
percentage: number;
}[];
operatingSystems: {
name: string;
count: number;
percentage: number;
}[];
}
export interface ConversionStats {
totalConversions: number;
conversionRate: number;
averageValue: number;
byType: {
type: string;
count: number;
percentage: number;
value: number;
}[];
}
export interface EventFilters {
startTime?: string;
endTime?: string;
eventType?: string;
linkId?: string;
linkSlug?: string;
page?: number;
pageSize?: number;
}

View File

@@ -0,0 +1,18 @@
import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
import { cookies } from 'next/headers';
import { NextRequest, NextResponse } from 'next/server';
export async function GET(request: NextRequest) {
const requestUrl = new URL(request.url);
const code = requestUrl.searchParams.get('code');
if (code) {
const cookieStore = cookies();
const supabase = createRouteHandlerClient({ cookies: () => cookieStore });
await supabase.auth.exchangeCodeForSession(code);
}
// URL to redirect to after sign in process completes
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || 'https://main.upj.to';
return NextResponse.redirect(new URL('/analytics', siteUrl));
}

View File

@@ -0,0 +1,45 @@
'use client';
import { useEffect, useState } from 'react';
import { useRouter } from 'next/navigation';
// 这个组件会检查 localStorage 中是否有认证令牌,如果没有则重定向到登录页面
export default function ClientRouteGuard({ children }: { children: React.ReactNode }) {
const router = useRouter();
const [isAuthenticated, setIsAuthenticated] = useState(false);
const [isLoading, setIsLoading] = useState(true);
useEffect(() => {
// 检查 localStorage 中是否有认证令牌
const checkAuth = () => {
// 查找 Supabase 认证令牌
const hasAuthToken = !!localStorage.getItem('sb-mwwvqwevplndzvmqmrxa-auth-token') ||
!!localStorage.getItem('sb-auth-token');
if (!hasAuthToken) {
// 如果没有令牌,重定向到登录页面
router.push('/login');
} else {
setIsAuthenticated(true);
}
setIsLoading(false);
};
checkAuth();
}, [router]);
// 显示加载状态
if (isLoading) {
return (
<div className="flex items-center justify-center min-h-screen">
<div className="text-center">
<div className="animate-spin rounded-full h-16 w-16 border-t-2 border-b-2 border-blue-500 mx-auto"></div>
<p className="mt-4 text-lg text-gray-700 dark:text-gray-300">...</p>
</div>
</div>
);
}
// 只有当用户已认证时才显示子组件
return isAuthenticated ? <>{children}</> : null;
}

111
app/components/Sidebar.tsx Normal file
View File

@@ -0,0 +1,111 @@
'use client';
import { useState } from 'react';
import Link from 'next/link';
import { usePathname } from 'next/navigation';
import {
BarChartIcon,
HomeIcon,
PersonIcon,
ChevronLeftIcon,
ChevronRightIcon
} from '@radix-ui/react-icons';
interface NavItemProps {
href: string;
label: string;
icon: React.ReactNode;
isCollapsed: boolean;
isActive?: boolean;
}
const NavItem = ({ href, label, icon, isCollapsed, isActive }: NavItemProps) => {
return (
<Link
href={href}
className={`flex items-center p-2 rounded-lg ${
isActive
? 'bg-blue-100 text-blue-700'
: 'text-gray-700 hover:bg-gray-100'
} transition-all duration-200 group`}
>
<div className="w-6 h-6 flex items-center justify-center">{icon}</div>
{!isCollapsed && (
<span className={`ml-3 whitespace-nowrap transition-opacity duration-200 ${
isCollapsed ? 'opacity-0 w-0' : 'opacity-100'
}`}>
{label}
</span>
)}
{isCollapsed && (
<span className="sr-only">{label}</span>
)}
</Link>
);
};
export function Sidebar() {
const [isCollapsed, setIsCollapsed] = useState(false);
const pathname = usePathname();
const toggleSidebar = () => {
setIsCollapsed(!isCollapsed);
};
const navigation = [
{ name: 'Dashboard', href: '/dashboard', icon: <HomeIcon className="w-5 h-5" /> },
{ name: 'Analytics', href: '/analytics', icon: <BarChartIcon className="w-5 h-5" /> },
{ name: 'Account', href: '/account', icon: <PersonIcon className="w-5 h-5" /> },
];
return (
<div className={`flex flex-col h-full transition-all duration-300 ${
isCollapsed ? 'w-16' : 'w-64'
} bg-white border-r border-gray-200 relative`}>
{/* 顶部Logo和标题 */}
<div className="flex items-center p-4 border-b border-gray-200">
<div className="flex-shrink-0 flex items-center justify-center w-8 h-8 bg-blue-500 text-white rounded">
<span className="font-bold">S</span>
</div>
{!isCollapsed && (
<span className="ml-3 font-medium text-gray-900 transition-opacity duration-200">
ShortURL Analytics
</span>
)}
</div>
{/* 导航菜单 */}
<div className="flex-grow p-4 overflow-y-auto">
<ul className="space-y-2">
{navigation.map((item) => (
<li key={item.name}>
<NavItem
href={item.href}
label={item.name}
icon={item.icon}
isCollapsed={isCollapsed}
isActive={pathname?.startsWith(item.href)}
/>
</li>
))}
</ul>
</div>
{/* 底部折叠按钮 */}
<div className="border-t border-gray-200 p-4">
<button
onClick={toggleSidebar}
className="w-full flex items-center justify-center p-2 rounded-lg text-gray-500 hover:bg-gray-100"
aria-label={isCollapsed ? "Expand sidebar" : "Collapse sidebar"}
>
{isCollapsed ? (
<ChevronRightIcon className="w-5 h-5" />
) : (
<ChevronLeftIcon className="w-5 h-5" />
)}
{!isCollapsed && <span className="ml-2">Collapse</span>}
</button>
</div>
</div>
);
}

View File

@@ -0,0 +1,52 @@
"use client";
import { DeviceAnalytics as DeviceAnalyticsType } from '@/app/api/types';
interface CategoryItem {
name: string;
count: number;
percentage: number;
}
interface DeviceAnalyticsProps {
data: DeviceAnalyticsType;
}
export default function DeviceAnalytics({ data }: DeviceAnalyticsProps) {
const renderCategory = (items: CategoryItem[], title: string) => (
<div className="bg-white rounded-lg shadow p-6">
<h3 className="text-lg font-semibold text-gray-900 mb-4">{title}</h3>
<div className="space-y-4">
{items.map((item, index) => (
<div key={index}>
<div className="flex justify-between text-sm text-gray-600 mb-1">
<span>{item.name}</span>
<span>{item.percentage.toFixed(1)}% ({item.count})</span>
</div>
<div className="w-full bg-gray-200 rounded-full h-2">
<div
className="bg-blue-600 h-2 rounded-full"
style={{ width: `${item.percentage}%` }}
></div>
</div>
</div>
))}
</div>
</div>
);
// Prepare device types data
const deviceItems = data.deviceTypes.map(item => ({
name: item.type || 'Unknown',
count: item.count,
percentage: item.percentage
}));
return (
<div className="grid grid-cols-1 md:grid-cols-3 gap-6">
{renderCategory(deviceItems, 'Device Types')}
{renderCategory(data.browsers, 'Browsers')}
{renderCategory(data.operatingSystems, 'Operating Systems')}
</div>
);
}

View File

@@ -0,0 +1,274 @@
"use client";
import { useState, useEffect } from 'react';
import { GeoData } from '@/app/api/types';
import { getLocationsFromIPs } from '@/app/utils/ipLocation';
interface GeoAnalyticsProps {
data: GeoData[];
}
// Interface for IP location data in our cache
interface IpLocationDetail {
country: string;
city: string;
region: string;
continent: string;
}
// Cache for IP location data
interface LocationCache {
[key: string]: IpLocationDetail;
}
export default function GeoAnalytics({ data }: GeoAnalyticsProps) {
const [viewMode, setViewMode] = useState<'country' | 'city' | 'region' | 'continent'>('country');
const [locationCache, setLocationCache] = useState<LocationCache>({});
const [isLoading, setIsLoading] = useState(false);
// Track IPs that failed to resolve
const [failedIPs, setFailedIPs] = useState<Set<string>>(new Set());
// 安全地格式化数字
const formatNumber = (value: number | undefined | null): string => {
if (value === undefined || value === null) return '0';
return value.toLocaleString();
};
// 安全地格式化百分比
const formatPercent = (value: number | undefined | null): string => {
if (value === undefined || value === null) return '0';
return value.toFixed(1);
};
const sortedData = [...data].sort((a, b) => (b.visits || 0) - (a.visits || 0));
// Handle tab selection - only change local view mode
const handleViewModeChange = (mode: 'country' | 'city' | 'region' | 'continent') => {
setViewMode(mode);
};
// Load location data for all IPs when the data changes
useEffect(() => {
const fetchLocations = async () => {
if (sortedData.length === 0) return;
setIsLoading(true);
const tempCache: LocationCache = {...locationCache};
const tempFailedIPs = new Set(failedIPs);
// Get all unique IPs that aren't already in the cache and haven't failed
const uniqueIPs = [...new Set(sortedData.map(item => item.location))].filter(ip =>
ip &&
ip !== 'Unknown' &&
!tempCache[ip] &&
!tempFailedIPs.has(ip)
);
if (uniqueIPs.length === 0) {
setIsLoading(false);
return;
}
try {
// Use batch lookup for better performance
const batchResults = await getLocationsFromIPs(uniqueIPs);
// Convert results to our cache format
for (const [ip, data] of Object.entries(batchResults)) {
if (data) {
tempCache[ip] = {
country: data.country_name,
city: data.city,
region: data.region,
continent: data.continent_name
};
} else {
// Mark as failed
tempFailedIPs.add(ip);
}
}
setLocationCache(tempCache);
setFailedIPs(tempFailedIPs);
} catch (error) {
console.error('Error fetching location data:', error);
} finally {
setIsLoading(false);
}
};
fetchLocations();
}, [data]);
// Get the appropriate location value based on the current view mode
const getLocationValue = (item: GeoData): string => {
const ip = item.location || '';
// If there's no IP or it's "Unknown", return that value
if (!ip || ip === 'Unknown') return 'Unknown';
// If IP failed to resolve, return Unknown
if (failedIPs.has(ip)) {
return 'Unknown';
}
// Return from cache if available
if (locationCache[ip]) {
switch (viewMode) {
case 'country':
return locationCache[ip].country || 'Unknown';
case 'city':
return locationCache[ip].city || 'Unknown';
case 'region':
return locationCache[ip].region || 'Unknown';
case 'continent':
return locationCache[ip].continent || 'Unknown';
default:
return ip;
}
}
// Return placeholder if not in cache yet
return `Loading...`;
};
// Get the appropriate area value based on the current view mode
const getAreaValue = (item: GeoData): string => {
const ip = item.location || '';
// If there's no IP or it's "Unknown", return empty string
if (!ip || ip === 'Unknown' || failedIPs.has(ip)) return '';
// Return from cache if available
if (locationCache[ip]) {
switch (viewMode) {
case 'country':
// For country view, show the continent as area
return locationCache[ip].continent || '';
case 'city':
// For city view, show the country and region
return `${locationCache[ip].country}, ${locationCache[ip].region}`;
case 'region':
// For region view, show the country
return locationCache[ip].country || '';
case 'continent':
// For continent view, no additional area needed
return '';
default:
return '';
}
}
// Return empty if not in cache yet
return '';
};
return (
<div>
{/* Tabs for geographic levels */}
<div className="flex border-b mb-6">
<button
onClick={() => handleViewModeChange('country')}
className={`px-4 py-2 ${viewMode === 'country' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Countries
</button>
<button
onClick={() => handleViewModeChange('city')}
className={`px-4 py-2 ${viewMode === 'city' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Cities
</button>
<button
onClick={() => handleViewModeChange('region')}
className={`px-4 py-2 ${viewMode === 'region' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Regions
</button>
<button
onClick={() => handleViewModeChange('continent')}
className={`px-4 py-2 ${viewMode === 'continent' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Continents
</button>
</div>
{/* Loading indicator */}
{isLoading && (
<div className="flex justify-center items-center py-2 mb-4">
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-blue-500 mr-2"></div>
<span className="text-sm text-gray-500">Loading location data...</span>
</div>
)}
{/* Table with added area column */}
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-gray-200">
<thead className="bg-gray-50">
<tr>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
{viewMode === 'country' ? 'Country' :
viewMode === 'city' ? 'City' :
viewMode === 'region' ? 'Region' : 'Continent'}
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
{viewMode === 'country' ? 'Continent' :
viewMode === 'city' ? 'Location' :
viewMode === 'region' ? 'Country' : 'Area'}
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
Visits
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
Unique Visitors
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
% of Total
</th>
</tr>
</thead>
<tbody className="bg-white divide-y divide-gray-200">
{sortedData.length > 0 ? (
sortedData.map((item, index) => (
<tr key={index} className={index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{getLocationValue(item)}
{item.location && (
<div className="text-xs text-gray-500 mt-1">{item.location}</div>
)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
{getAreaValue(item)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{formatNumber(item.visits)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{formatNumber(item.visitors)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
<div className="flex items-center">
<div className="w-24 bg-gray-200 rounded-full h-2">
<div
className="bg-blue-600 h-2 rounded-full"
style={{ width: `${item.percentage || 0}%` }}
/>
</div>
<span className="ml-2">{formatPercent(item.percentage)}%</span>
</div>
</td>
</tr>
))
) : (
<tr>
<td colSpan={5} className="px-6 py-4 text-center text-sm text-gray-500">
No location data available
</td>
</tr>
)}
</tbody>
</table>
</div>
</div>
);
}

View File

@@ -0,0 +1,162 @@
import React, { useState, useEffect } from 'react';
interface PathAnalyticsProps {
startTime: string;
endTime: string;
linkId?: string;
onPathClick?: (path: string) => void;
}
interface PathData {
path: string;
count: number;
percentage: number;
}
const PathAnalytics: React.FC<PathAnalyticsProps> = ({ startTime, endTime, linkId, onPathClick }) => {
const [loading, setLoading] = useState(true);
const [pathData, setPathData] = useState<PathData[]>([]);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
if (!linkId) {
setLoading(false);
return;
}
const fetchPathData = async () => {
try {
const params = new URLSearchParams({
startTime,
endTime,
linkId
});
const response = await fetch(`/api/events/path-analytics?${params.toString()}`);
if (!response.ok) {
throw new Error('Failed to fetch path analytics data');
}
const result = await response.json();
if (result.success && result.data) {
// 自定义处理路径数据,根据是否有子路径来分组
const rawData = result.data;
const pathMap = new Map<string, number>();
let totalClicks = 0;
rawData.forEach((item: PathData) => {
const urlPath = item.path.split('?')[0];
totalClicks += item.count;
// 解析路径,检查是否有子路径
const pathParts = urlPath.split('/').filter(Boolean);
// 基础路径(例如/5seaii或者带有查询参数但没有子路径的路径视为同一个路径
// 子路径(例如/5seaii/bbbbb单独统计
const groupKey = pathParts.length > 1 ? urlPath : `/${pathParts[0]}`;
const currentCount = pathMap.get(groupKey) || 0;
pathMap.set(groupKey, currentCount + item.count);
});
// 转换回数组并排序
const groupedPathData = Array.from(pathMap.entries())
.map(([path, count]) => ({
path,
count,
percentage: totalClicks > 0 ? count / totalClicks : 0,
}))
.sort((a, b) => b.count - a.count);
setPathData(groupedPathData);
} else {
setError(result.error || 'Failed to load path analytics');
}
} catch (err) {
setError(err instanceof Error ? err.message : 'An error occurred');
} finally {
setLoading(false);
}
};
fetchPathData();
}, [startTime, endTime, linkId]);
const handlePathClick = (path: string, e: React.MouseEvent) => {
e.preventDefault();
console.log('====== PATH CLICK DEBUG ======');
console.log('Path value:', path);
console.log('Path type:', typeof path);
console.log('Path length:', path.length);
console.log('Path chars:', Array.from(path).map(c => c.charCodeAt(0)));
console.log('==============================');
if (onPathClick) {
onPathClick(path);
}
};
if (loading) {
return <div className="py-8 flex justify-center">
<div className="animate-spin rounded-full h-8 w-8 border-t-2 border-b-2 border-blue-500" />
</div>;
}
if (error) {
return <div className="py-4 text-red-500">{error}</div>;
}
if (!linkId) {
return <div className="py-4 text-gray-500">Select a specific link to view path analytics.</div>;
}
if (pathData.length === 0) {
return <div className="py-4 text-gray-500">No path data available for this link.</div>;
}
return (
<div>
<div className="text-sm text-gray-500 mb-4">
Note: Paths are grouped by subpath. URLs with different query parameters but the same base path (without subpath) are grouped together.
</div>
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-gray-200">
<thead>
<tr>
<th className="px-6 py-3 bg-gray-50 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Path</th>
<th className="px-6 py-3 bg-gray-50 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">Clicks</th>
<th className="px-6 py-3 bg-gray-50 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">Percentage</th>
</tr>
</thead>
<tbody className="bg-white divide-y divide-gray-200">
{pathData.map((item, index) => (
<tr key={index} className={index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}>
<td className="px-6 py-4 whitespace-nowrap text-sm font-medium text-gray-900">
<a
href="#"
className="hover:text-blue-600 hover:underline cursor-pointer"
onClick={(e) => handlePathClick(item.path, e)}
>
{item.path}
</a>
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500 text-right">{item.count}</td>
<td className="px-6 py-4 whitespace-nowrap text-right">
<div className="flex items-center justify-end">
<span className="text-sm text-gray-500 mr-2">{(item.percentage * 100).toFixed(1)}%</span>
<div className="w-32 bg-gray-200 rounded-full h-2.5">
<div className="bg-blue-600 h-2.5 rounded-full" style={{ width: `${item.percentage * 100}%` }}></div>
</div>
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
);
};
export default PathAnalytics;

View File

@@ -0,0 +1,205 @@
"use client";
import { useState, useEffect } from 'react';
interface UtmData {
utm_value: string;
clicks: number;
visitors: number;
avg_time_spent: number;
bounces: number;
conversions: number;
}
interface UtmAnalyticsProps {
startTime?: string;
endTime?: string;
linkId?: string;
teamIds?: string[];
projectIds?: string[];
tagIds?: string[];
subpath?: string;
}
export default function UtmAnalytics({ startTime, endTime, linkId, teamIds, projectIds, tagIds, subpath }: UtmAnalyticsProps) {
const [activeTab, setActiveTab] = useState<string>('source');
const [utmData, setUtmData] = useState<UtmData[]>([]);
const [isLoading, setIsLoading] = useState<boolean>(false);
const [error, setError] = useState<string | null>(null);
// 加载UTM数据
useEffect(() => {
const fetchUtmData = async () => {
setIsLoading(true);
setError(null);
try {
// 构建URL参数
const params = new URLSearchParams();
if (startTime) params.append('startTime', startTime);
if (endTime) params.append('endTime', endTime);
if (linkId) params.append('linkId', linkId);
if (subpath) params.append('subpath', subpath);
params.append('utmType', activeTab);
// 添加团队ID参数
if (teamIds && teamIds.length > 0) {
teamIds.forEach(id => params.append('teamId', id));
}
// 添加项目ID参数
if (projectIds && projectIds.length > 0) {
projectIds.forEach(id => params.append('projectId', id));
}
// 添加标签名称参数
if (tagIds && tagIds.length > 0) {
tagIds.forEach(tagName => params.append('tagName', tagName));
}
// 发送请求
const response = await fetch(`/api/events/utm?${params}`);
if (!response.ok) {
throw new Error('Failed to fetch UTM data');
}
const result = await response.json();
if (result.success) {
setUtmData(result.data || []);
} else {
throw new Error(result.error || 'Failed to fetch UTM data');
}
} catch (err) {
setError(err instanceof Error ? err.message : 'Unknown error occurred');
console.error('Error fetching UTM data:', err);
} finally {
setIsLoading(false);
}
};
fetchUtmData();
}, [activeTab, startTime, endTime, linkId, teamIds, projectIds, tagIds, subpath]);
// 安全地格式化数字
const formatNumber = (value: number | undefined | null): string => {
if (value === undefined || value === null) return '0';
return value.toLocaleString();
};
return (
<div className="bg-white rounded-lg shadow p-6 mb-8">
<h2 className="text-lg font-semibold text-gray-900 mb-4">UTM Parameters</h2>
<div className="mb-4 border-b">
<div className="flex">
<button
onClick={() => setActiveTab('source')}
className={`px-4 py-2 ${activeTab === 'source' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Source
</button>
<button
onClick={() => setActiveTab('medium')}
className={`px-4 py-2 ${activeTab === 'medium' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Medium
</button>
<button
onClick={() => setActiveTab('campaign')}
className={`px-4 py-2 ${activeTab === 'campaign' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Campaign
</button>
<button
onClick={() => setActiveTab('term')}
className={`px-4 py-2 ${activeTab === 'term' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Term
</button>
<button
onClick={() => setActiveTab('content')}
className={`px-4 py-2 ${activeTab === 'content' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
>
Content
</button>
</div>
</div>
{isLoading ? (
<div className="flex justify-center items-center py-8">
<div className="animate-spin rounded-full h-6 w-6 border-b-2 border-blue-500"></div>
<span className="ml-2 text-gray-500">Loading...</span>
</div>
) : error ? (
<div className="text-red-500 text-center py-8">
Error: {error}
</div>
) : utmData.length === 0 ? (
<div className="text-gray-500 text-center py-8">
No data available
</div>
) : (
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-gray-200">
<thead className="bg-gray-50">
<tr>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
{activeTab === 'source' ? 'Source' :
activeTab === 'medium' ? 'Medium' :
activeTab === 'campaign' ? 'Campaign' :
activeTab === 'term' ? 'Term' : 'Content'}
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
Clicks
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
Visitors
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
Avg. Time
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
Bounce Rate
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
Conversions
</th>
</tr>
</thead>
<tbody className="bg-white divide-y divide-gray-200">
{utmData.map((item, index) => {
const bounceRate = item.clicks > 0 ? (item.bounces / item.clicks) * 100 : 0;
const conversionRate = item.clicks > 0 ? (item.conversions / item.clicks) * 100 : 0;
return (
<tr key={index} className={index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{item.utm_value || 'Unknown'}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{formatNumber(item.clicks)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{formatNumber(item.visitors)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{item.avg_time_spent.toFixed(1)}s
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{bounceRate.toFixed(1)}%
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{formatNumber(item.conversions)} ({conversionRate.toFixed(1)}%)
</td>
</tr>
);
})}
</tbody>
</table>
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,211 @@
'use client';
import { useEffect, useRef } from 'react';
import { DeviceAnalytics } from '@/app/api/types';
import { Chart, PieController, ArcElement, Tooltip, Legend, CategoryScale, LinearScale } from 'chart.js';
// 注册Chart.js组件
Chart.register(PieController, ArcElement, Tooltip, Legend, CategoryScale, LinearScale);
interface DevicePieChartsProps {
data: DeviceAnalytics;
}
// 颜色配置
const COLORS = {
deviceTypes: ['rgba(59, 130, 246, 0.8)', 'rgba(96, 165, 250, 0.8)', 'rgba(147, 197, 253, 0.8)', 'rgba(191, 219, 254, 0.8)', 'rgba(219, 234, 254, 0.8)'],
browsers: ['rgba(16, 185, 129, 0.8)', 'rgba(52, 211, 153, 0.8)', 'rgba(110, 231, 183, 0.8)', 'rgba(167, 243, 208, 0.8)', 'rgba(209, 250, 229, 0.8)'],
os: ['rgba(239, 68, 68, 0.8)', 'rgba(248, 113, 113, 0.8)', 'rgba(252, 165, 165, 0.8)', 'rgba(254, 202, 202, 0.8)', 'rgba(254, 226, 226, 0.8)']
};
export default function DevicePieCharts({ data }: DevicePieChartsProps) {
// 创建图表引用
const deviceTypesChartRef = useRef<HTMLCanvasElement>(null);
const browsersChartRef = useRef<HTMLCanvasElement>(null);
const osChartRef = useRef<HTMLCanvasElement>(null);
// 图表实例引用
const deviceTypesChartInstance = useRef<Chart | null>(null);
const browsersChartInstance = useRef<Chart | null>(null);
const osChartInstance = useRef<Chart | null>(null);
// 初始化和更新图表
useEffect(() => {
if (!data) return;
// 销毁旧的图表实例
if (deviceTypesChartInstance.current) {
deviceTypesChartInstance.current.destroy();
}
if (browsersChartInstance.current) {
browsersChartInstance.current.destroy();
}
if (osChartInstance.current) {
osChartInstance.current.destroy();
}
// 创建设备类型图表
if (deviceTypesChartRef.current && data.deviceTypes.length > 0) {
const ctx = deviceTypesChartRef.current.getContext('2d');
if (ctx) {
deviceTypesChartInstance.current = new Chart(ctx, {
type: 'pie',
data: {
labels: data.deviceTypes.map(item => item.type),
datasets: [{
data: data.deviceTypes.map(item => item.count),
backgroundColor: COLORS.deviceTypes,
borderColor: COLORS.deviceTypes.map(color => color.replace('0.8', '1')),
borderWidth: 1
}]
},
options: {
responsive: true,
plugins: {
legend: {
position: 'bottom',
labels: {
color: 'currentColor'
}
},
tooltip: {
callbacks: {
label: function(context) {
const label = context.label || '';
const value = context.raw as number;
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
const percentage = Math.round((value * 100) / total);
return `${label}: ${value} (${percentage}%)`;
}
}
}
}
}
});
}
}
// 创建浏览器图表
if (browsersChartRef.current && data.browsers.length > 0) {
const ctx = browsersChartRef.current.getContext('2d');
if (ctx) {
browsersChartInstance.current = new Chart(ctx, {
type: 'pie',
data: {
labels: data.browsers.map(item => item.name),
datasets: [{
data: data.browsers.map(item => item.count),
backgroundColor: COLORS.browsers,
borderColor: COLORS.browsers.map(color => color.replace('0.8', '1')),
borderWidth: 1
}]
},
options: {
responsive: true,
plugins: {
legend: {
position: 'bottom',
labels: {
color: 'currentColor'
}
},
tooltip: {
callbacks: {
label: function(context) {
const label = context.label || '';
const value = context.raw as number;
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
const percentage = Math.round((value * 100) / total);
return `${label}: ${value} (${percentage}%)`;
}
}
}
}
}
});
}
}
// 创建操作系统图表
if (osChartRef.current && data.operatingSystems.length > 0) {
const ctx = osChartRef.current.getContext('2d');
if (ctx) {
osChartInstance.current = new Chart(ctx, {
type: 'pie',
data: {
labels: data.operatingSystems.map(item => item.name),
datasets: [{
data: data.operatingSystems.map(item => item.count),
backgroundColor: COLORS.os,
borderColor: COLORS.os.map(color => color.replace('0.8', '1')),
borderWidth: 1
}]
},
options: {
responsive: true,
plugins: {
legend: {
position: 'bottom',
labels: {
color: 'currentColor'
}
},
tooltip: {
callbacks: {
label: function(context) {
const label = context.label || '';
const value = context.raw as number;
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
const percentage = Math.round((value * 100) / total);
return `${label}: ${value} (${percentage}%)`;
}
}
}
}
}
});
}
}
// 清理函数
return () => {
if (deviceTypesChartInstance.current) {
deviceTypesChartInstance.current.destroy();
}
if (browsersChartInstance.current) {
browsersChartInstance.current.destroy();
}
if (osChartInstance.current) {
osChartInstance.current.destroy();
}
};
}, [data]);
return (
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
{/* 设备类型 */}
<div className="bg-white rounded-lg shadow p-6">
<h3 className="text-lg font-semibold text-gray-900 mb-4">Device Types</h3>
<div className="h-64">
<canvas ref={deviceTypesChartRef} />
</div>
</div>
{/* 浏览器 */}
<div className="bg-white rounded-lg shadow p-6">
<h3 className="text-lg font-semibold text-gray-900 mb-4">Browsers</h3>
<div className="h-64">
<canvas ref={browsersChartRef} />
</div>
</div>
{/* 操作系统 */}
<div className="bg-white rounded-lg shadow p-6">
<h3 className="text-lg font-semibold text-gray-900 mb-4">Operating Systems</h3>
<div className="h-64">
<canvas ref={osChartRef} />
</div>
</div>
</div>
);
}

View File

@@ -0,0 +1,189 @@
"use client";
import { useEffect, useRef } from 'react';
import {
Chart as ChartJS,
CategoryScale,
LinearScale,
PointElement,
LineElement,
LineController,
Title,
Tooltip,
Legend,
Filler,
ChartData,
ChartOptions,
TooltipItem
} from 'chart.js';
import { TimeSeriesData } from '@/app/api/types';
// 注册 Chart.js 组件
ChartJS.register(
CategoryScale,
LinearScale,
PointElement,
LineElement,
LineController,
Title,
Tooltip,
Legend,
Filler
);
interface TimeSeriesChartProps {
data: TimeSeriesData[];
}
export default function TimeSeriesChart({ data }: TimeSeriesChartProps) {
const chartRef = useRef<HTMLCanvasElement | null>(null);
const chartInstance = useRef<ChartJS | null>(null);
useEffect(() => {
if (!chartRef.current) return;
// 销毁旧的图表实例
if (chartInstance.current) {
chartInstance.current.destroy();
}
const ctx = chartRef.current.getContext('2d');
if (!ctx) return;
// 准备数据
const labels = data.map(item => {
if (!item || !item.timestamp) return '';
const date = new Date(item.timestamp);
return date.toLocaleDateString();
});
const eventsData = data.map(item => {
if (!item || item.events === undefined || item.events === null) return 0;
return Number(item.events);
});
const visitorsData = data.map(item => {
if (!item || item.visitors === undefined || item.visitors === null) return 0;
return Number(item.visitors);
});
const conversionsData = data.map(item => {
if (!item || item.conversions === undefined || item.conversions === null) return 0;
return Number(item.conversions);
});
// 创建新的图表实例
chartInstance.current = new ChartJS(ctx, {
type: 'line',
data: {
labels,
datasets: [
{
label: 'Events',
data: eventsData,
borderColor: 'rgb(59, 130, 246)', // blue-500
backgroundColor: 'rgba(59, 130, 246, 0.1)',
tension: 0.4,
fill: true
},
{
label: 'Visitors',
data: visitorsData,
borderColor: 'rgb(16, 185, 129)', // green-500
backgroundColor: 'rgba(16, 185, 129, 0.1)',
tension: 0.4,
fill: true
},
{
label: 'Conversions',
data: conversionsData,
borderColor: 'rgb(239, 68, 68)', // red-500
backgroundColor: 'rgba(239, 68, 68, 0.1)',
tension: 0.4,
fill: true
}
]
} as ChartData<'line'>,
options: {
responsive: true,
maintainAspectRatio: false,
interaction: {
mode: 'index',
intersect: false
},
plugins: {
legend: {
position: 'top',
labels: {
usePointStyle: true,
padding: 20,
color: 'rgb(156, 163, 175)' // gray-400
}
},
tooltip: {
mode: 'index',
intersect: false,
backgroundColor: 'rgb(31, 41, 55)', // gray-800
titleColor: 'rgb(229, 231, 235)', // gray-200
bodyColor: 'rgb(229, 231, 235)', // gray-200
borderColor: 'rgb(75, 85, 99)', // gray-600
borderWidth: 1,
padding: 12,
displayColors: true,
callbacks: {
title: (items: TooltipItem<'line'>[]) => {
if (items.length > 0) {
const date = new Date(data[items[0].dataIndex].timestamp);
return date.toLocaleDateString();
}
return '';
},
label: (context) => {
const label = context.dataset.label || '';
const value = context.parsed.y;
return `${label}: ${Math.round(value)}`;
}
}
}
},
scales: {
x: {
grid: {
display: false
},
ticks: {
color: 'rgb(156, 163, 175)' // gray-400
}
},
y: {
beginAtZero: true,
grid: {
color: 'rgb(75, 85, 99, 0.1)' // gray-600 with opacity
},
ticks: {
color: 'rgb(156, 163, 175)', // gray-400
callback: (value: number) => {
if (!value && value !== 0) return '';
if (value >= 1000) {
return `${Math.round(value / 1000)}k`;
}
return Math.round(value);
}
}
}
}
} as ChartOptions<'line'>
});
// 清理函数
return () => {
if (chartInstance.current) {
chartInstance.current.destroy();
}
};
}, [data]);
return (
<canvas ref={chartRef} />
);
}

View File

@@ -0,0 +1,100 @@
"use client";
import { useState, useEffect } from 'react';
import { getLocationFromIP } from '@/app/utils/ipLocation';
interface LocationData {
ip: string;
country_name: string;
country_code: string;
city: string;
region: string;
continent_code: string;
continent_name: string;
latitude: number;
longitude: number;
}
export default function IpLocationTest() {
const [locationData, setLocationData] = useState<LocationData | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const testIp = "120.244.39.90";
useEffect(() => {
async function fetchLocation() {
try {
setLoading(true);
setError(null);
const data = await getLocationFromIP(testIp);
setLocationData(data);
} catch (err) {
setError(err instanceof Error ? err.message : 'Unknown error occurred');
} finally {
setLoading(false);
}
}
fetchLocation();
}, []);
return (
<div className="p-4 bg-white rounded-lg shadow">
<h2 className="text-lg font-semibold mb-4">IP Location Test: {testIp}</h2>
{loading && (
<div className="flex items-center text-gray-500">
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-blue-500 mr-2"></div>
Loading location data...
</div>
)}
{error && (
<div className="text-red-500">
Error: {error}
</div>
)}
{!loading && locationData && (
<div className="space-y-4">
<div>
<h3 className="font-medium">Location Data:</h3>
<pre className="mt-2 p-4 bg-gray-100 rounded overflow-auto">
{JSON.stringify(locationData, null, 2)}
</pre>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="border p-3 rounded">
<h4 className="font-medium">Country</h4>
<div>{locationData.country_name} ({locationData.country_code})</div>
</div>
<div className="border p-3 rounded">
<h4 className="font-medium">City</h4>
<div>{locationData.city || 'N/A'}</div>
</div>
<div className="border p-3 rounded">
<h4 className="font-medium">Region</h4>
<div>{locationData.region || 'N/A'}</div>
</div>
<div className="border p-3 rounded">
<h4 className="font-medium">Continent</h4>
<div>{locationData.continent_name} ({locationData.continent_code})</div>
</div>
<div className="border p-3 rounded col-span-2">
<h4 className="font-medium">Coordinates</h4>
<div>Latitude: {locationData.latitude}, Longitude: {locationData.longitude}</div>
</div>
</div>
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,73 @@
'use client';
import Link from 'next/link';
import { useAuth } from '@/lib/auth';
export default function Header() {
const { user, signOut } = useAuth();
const handleLogout = async () => {
await signOut();
};
return (
<header className="w-full py-4 border-b border-gray-200 bg-white">
<div className="container flex items-center justify-between px-4 mx-auto">
<div className="flex items-center space-x-4">
<Link href="/analytics" className="flex items-center space-x-2">
<svg
className="w-6 h-6 text-blue-500"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
>
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
</svg>
<span className="text-xl font-bold text-gray-900">ShortURL Analytics</span>
</Link>
{user && (
<nav className="ml-6">
<ul className="flex space-x-4">
<li>
<Link href="/analytics" className="text-sm text-gray-700 hover:text-blue-500">
Analytics
</Link>
</li>
<li>
<Link href="/links" className="text-sm text-gray-700 hover:text-blue-500">
Short Links
</Link>
</li>
<li>
<Link href="/create-shorturl" className="text-sm text-gray-700 hover:text-blue-500">
Create Short URL
</Link>
</li>
</ul>
</nav>
)}
</div>
{user && (
<div className="flex items-center space-x-4">
<div className="text-sm text-gray-700">
{user.email}
</div>
<button
onClick={handleLogout}
className="px-4 py-2 text-sm text-white bg-blue-500 rounded hover:bg-blue-600"
>
Logout
</button>
</div>
)}
</div>
</header>
);
}

View File

@@ -1,67 +0,0 @@
'use client';
import Link from 'next/link';
import ThemeToggle from "../ui/ThemeToggle";
export default function Navbar() {
return (
<header className="w-full py-4 border-b border-card-border bg-background">
<div className="container flex items-center justify-between px-4 mx-auto">
<div className="flex items-center space-x-4">
<Link href="/" className="flex items-center space-x-2">
<svg
className="w-6 h-6 text-accent-blue"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
>
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
</svg>
<span className="text-xl font-bold text-foreground">ShortURL</span>
</Link>
<nav className="hidden space-x-4 md:flex">
<Link
href="/links"
className="text-sm text-foreground hover:text-accent-blue transition-colors"
>
Links
</Link>
<Link
href="/analytics"
className="text-sm text-foreground hover:text-accent-blue transition-colors"
>
Analytics
</Link>
</nav>
</div>
<div className="flex items-center space-x-3">
<ThemeToggle />
<button className="p-2 text-sm text-foreground rounded-md gradient-border">
Upgrade
</button>
<button className="p-2 text-sm text-foreground hover:text-accent-blue">
<svg
className="w-5 h-5"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
>
<circle cx="12" cy="12" r="10"></circle>
<circle cx="12" cy="10" r="3"></circle>
<path d="M7 20.662V19a2 2 0 0 1 2-2h6a2 2 0 0 1 2 2v1.662"></path>
</svg>
</button>
</div>
</div>
</header>
);
}

View File

@@ -0,0 +1,84 @@
"use client";
import { useState } from 'react';
import { format } from 'date-fns';
interface DateRange {
from: Date;
to: Date;
}
interface DateRangePickerProps {
value: DateRange;
onChange: (value: DateRange) => void;
className?: string;
}
export function DateRangePicker({
value,
onChange,
className
}: DateRangePickerProps) {
// Internal date state for validation
const [from, setFrom] = useState<string>(
value.from ? format(value.from, 'yyyy-MM-dd') : ''
);
const [to, setTo] = useState<string>(
value.to ? format(value.to, 'yyyy-MM-dd') : ''
);
const handleFromChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newFrom = e.target.value;
setFrom(newFrom);
if (newFrom) {
onChange({
from: new Date(newFrom),
to: value.to
});
}
};
const handleToChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newTo = e.target.value;
setTo(newTo);
if (newTo) {
onChange({
from: value.from,
to: new Date(newTo)
});
}
};
return (
<div className={`flex flex-col space-y-2 sm:flex-row sm:space-y-0 sm:space-x-4 ${className}`}>
<div>
<label htmlFor="from" className="block text-sm font-medium text-gray-500 mb-1">
Start Date
</label>
<input
type="date"
id="from"
value={from}
onChange={handleFromChange}
max={to}
className="block w-full px-3 py-2 bg-white border border-gray-300 rounded-md text-sm text-gray-900 focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
/>
</div>
<div>
<label htmlFor="to" className="block text-sm font-medium text-gray-500 mb-1">
End Date
</label>
<input
type="date"
id="to"
value={to}
onChange={handleToChange}
min={from}
className="block w-full px-3 py-2 bg-white border border-gray-300 rounded-md text-sm text-gray-900 focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
/>
</div>
</div>
);
}

View File

@@ -0,0 +1,363 @@
"use client";
import * as React from 'react';
import { useEffect, useState, useRef } from 'react';
import { getSupabaseClient } from '../../utils/supabase';
import { AuthChangeEvent, Session } from '@supabase/supabase-js';
import { Loader2, X, Check } from 'lucide-react';
import { cn } from '@/lib/utils';
// Define our own Project type
interface Project {
id: string;
name: string;
description?: string | null;
attributes?: Record<string, unknown>;
created_at?: string;
updated_at?: string;
deleted_at?: string | null;
schema_version?: number | null;
creator_id?: string | null;
team_name?: string;
}
// 添加需要的类型定义
interface ProjectWithTeam {
project_id: string;
projects: Project;
teams?: { name: string };
}
// ProjectSelector component with multi-select support
export function ProjectSelector({
value,
onChange,
className,
multiple = false,
teamId,
teamIds,
}: {
value?: string | string[];
onChange?: (projectId: string | string[]) => void;
className?: string;
multiple?: boolean;
teamId?: string; // Optional team ID to filter projects by team
teamIds?: string[]; // Optional array of team IDs to filter projects by multiple teams
}) {
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [projects, setProjects] = useState<Project[]>([]);
const [selectedIds, setSelectedIds] = useState<string[]>([]);
const [isOpen, setIsOpen] = useState(false);
const selectorRef = useRef<HTMLDivElement>(null);
// Normalize team IDs to ensure we're always working with an array
const effectiveTeamIds = React.useMemo(() => {
if (teamIds && teamIds.length > 0) {
return teamIds;
} else if (teamId) {
return [teamId];
}
return undefined;
}, [teamId, teamIds]);
// Initialize selected projects based on value prop
useEffect(() => {
if (value) {
if (Array.isArray(value)) {
setSelectedIds(value);
} else {
setSelectedIds(value ? [value] : []);
}
} else {
setSelectedIds([]);
}
}, [value]);
// Add click outside listener to close dropdown
useEffect(() => {
function handleClickOutside(event: MouseEvent) {
if (selectorRef.current && !selectorRef.current.contains(event.target as Node)) {
setIsOpen(false);
}
}
// Only add the event listener if the dropdown is open
if (isOpen) {
document.addEventListener('mousedown', handleClickOutside);
return () => {
document.removeEventListener('mousedown', handleClickOutside);
};
}
}, [isOpen]);
useEffect(() => {
let isMounted = true;
const fetchProjects = async (userId: string) => {
if (!isMounted) return;
setLoading(true);
setError(null);
try {
const supabase = getSupabaseClient();
if (effectiveTeamIds && effectiveTeamIds.length > 0) {
// If team IDs are provided, get projects for those teams
const { data: projectsData, error: projectsError } = await supabase
.from('team_projects')
.select('project_id, projects:project_id(*), teams:team_id(name)')
.in('team_id', effectiveTeamIds)
.is('projects.deleted_at', null);
if (projectsError) throw projectsError;
if (!projectsData || projectsData.length === 0) {
if (isMounted) setProjects([]);
return;
}
// Extract projects from response with team info
if (isMounted) {
const projectList: Project[] = [];
for (const item of projectsData as ProjectWithTeam[]) {
if (item.projects && typeof item.projects === 'object' && 'id' in item.projects && 'name' in item.projects) {
const project = item.projects as Project;
if (item.teams && 'name' in item.teams) {
project.team_name = item.teams.name;
}
// Avoid duplicate projects from different teams
if (!projectList.some(p => p.id === project.id)) {
projectList.push(project);
}
}
}
setProjects(projectList);
}
} else {
// If no team IDs, get all user's projects
const { data: projectsData, error: projectsError } = await supabase
.from('user_projects')
.select('project_id, projects:project_id(*)')
.eq('user_id', userId)
.is('projects.deleted_at', null);
if (projectsError) throw projectsError;
if (!projectsData || projectsData.length === 0) {
if (isMounted) setProjects([]);
return;
}
// Fetch team info for these projects
const projectIds = projectsData.map(item => item.project_id);
// Get team info for each project
const { data: teamProjectsData, error: teamProjectsError } = await supabase
.from('team_projects')
.select('project_id, teams:team_id(name)')
.in('project_id', projectIds);
if (teamProjectsError) throw teamProjectsError;
// Create project ID to team name mapping
const projectTeamMap: Record<string, string> = {};
if (teamProjectsData) {
teamProjectsData.forEach(item => {
if (item.teams && typeof item.teams === 'object' && 'name' in item.teams) {
projectTeamMap[item.project_id] = (item.teams as { name: string }).name;
}
});
}
// Extract projects with team names
if (isMounted && projectsData) {
const projectList: Project[] = [];
for (const item of projectsData) {
if (item.projects && typeof item.projects === 'object' && 'id' in item.projects && 'name' in item.projects) {
const project = item.projects as Project;
project.team_name = projectTeamMap[project.id];
projectList.push(project);
}
}
setProjects(projectList);
}
}
} catch (err) {
if (isMounted) {
setError(err instanceof Error ? err.message : 'Failed to load projects');
}
} finally {
if (isMounted) {
setLoading(false);
}
}
};
const supabase = getSupabaseClient();
const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent, session: Session | null) => {
if (event === 'SIGNED_IN' && session?.user?.id) {
fetchProjects(session.user.id);
} else if (event === 'SIGNED_OUT') {
setProjects([]);
setError(null);
}
});
supabase.auth.getSession().then(({ data: { session } }) => {
if (session?.user?.id) {
fetchProjects(session.user.id);
}
});
return () => {
isMounted = false;
subscription.unsubscribe();
};
}, [effectiveTeamIds]);
const handleToggle = () => {
if (!loading && !error && projects.length > 0) {
setIsOpen(!isOpen);
}
};
const handleProjectSelect = (projectId: string) => {
let newSelected: string[];
if (multiple) {
// For multi-select: toggle project in/out of selection
if (selectedIds.includes(projectId)) {
newSelected = selectedIds.filter(id => id !== projectId);
} else {
newSelected = [...selectedIds, projectId];
}
} else {
// For single-select: replace selection with the new project
newSelected = [projectId];
setIsOpen(false);
}
setSelectedIds(newSelected);
if (onChange) {
onChange(multiple ? newSelected : newSelected[0] || '');
}
};
const removeProject = (e: React.MouseEvent, projectId: string) => {
e.stopPropagation();
const newSelected = selectedIds.filter(id => id !== projectId);
setSelectedIds(newSelected);
if (onChange) {
onChange(multiple ? newSelected : newSelected[0] || '');
}
};
if (loading) {
return (
<div className={cn(
"flex w-full items-center justify-between rounded-md border px-3 py-2",
className
)}>
<Loader2 className="h-4 w-4 animate-spin" />
</div>
);
}
if (error) {
return (
<div className={cn(
"flex w-full items-center rounded-md border border-destructive bg-destructive/10 px-3 py-2 text-destructive",
className
)}>
{error}
</div>
);
}
if (projects.length === 0) {
return (
<div className={cn(
"flex w-full items-center rounded-md border px-3 py-2 text-muted-foreground",
className
)}>
No projects available
</div>
);
}
const selectedProjects = projects.filter(project => selectedIds.includes(project.id));
return (
<div className="relative" ref={selectorRef}>
<div
className={cn(
"flex w-full min-h-10 items-center flex-wrap rounded-md border p-1 cursor-pointer",
isOpen && "ring-2 ring-offset-2 ring-blue-500",
className
)}
onClick={handleToggle}
>
{selectedProjects.length > 0 ? (
<div className="flex flex-wrap gap-1">
{selectedProjects.map(project => (
<div
key={project.id}
className="flex items-center gap-1 bg-green-100 text-green-800 rounded-md px-2 py-1 text-sm"
>
{project.name}
{multiple && (
<X
size={14}
className="cursor-pointer hover:text-green-900"
onClick={(e) => removeProject(e, project.id)}
/>
)}
</div>
))}
</div>
) : (
<div className="px-2 py-1 text-gray-500">Select a project</div>
)}
</div>
{isOpen && (
<div className="absolute w-full mt-1 max-h-60 overflow-auto rounded-md border bg-white shadow-lg z-10">
{projects.map(project => (
<div
key={project.id}
className={cn(
"px-3 py-2 cursor-pointer hover:bg-gray-100 flex items-center justify-between",
selectedIds.includes(project.id) && "bg-green-50"
)}
onClick={() => handleProjectSelect(project.id)}
>
<span className="flex flex-col">
<span className="font-medium">{project.name}</span>
{project.team_name && (
<span className="text-xs text-gray-500">
{project.team_name}
</span>
)}
{project.description && (
<span className="text-xs text-gray-500 truncate max-w-[250px]">
{project.description}
</span>
)}
</span>
{selectedIds.includes(project.id) && (
<Check className="h-4 w-4 text-green-600" />
)}
</div>
))}
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,88 @@
"use client";
import * as React from 'react';
import { ChevronDown } from 'lucide-react';
interface SelectOption {
value: string;
label: string;
icon?: string;
}
interface SelectProps {
value?: string;
onChange?: (value: string) => void;
options: SelectOption[];
placeholder?: string;
className?: string;
}
export function Select({ value, onChange, options, placeholder, className = '' }: SelectProps) {
const [isOpen, setIsOpen] = React.useState(false);
const containerRef = React.useRef<HTMLDivElement>(null);
React.useEffect(() => {
function handleClickOutside(event: MouseEvent) {
if (containerRef.current && !containerRef.current.contains(event.target as Node)) {
setIsOpen(false);
}
}
document.addEventListener('mousedown', handleClickOutside);
return () => {
document.removeEventListener('mousedown', handleClickOutside);
};
}, []);
const selectedOption = options.find(option => option.value === value);
return (
<div className={`relative ${className}`} ref={containerRef}>
<button
type="button"
onClick={() => setIsOpen(!isOpen)}
className="flex h-10 w-full items-center justify-between rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50"
>
<span className="flex items-center">
{selectedOption?.icon && (
<img
src={selectedOption.icon}
alt=""
className="mr-2 h-4 w-4 rounded-full"
/>
)}
{selectedOption?.label || placeholder}
</span>
<ChevronDown className="h-4 w-4 opacity-50" />
</button>
{isOpen && (
<div className="absolute z-50 mt-1 w-full rounded-md border bg-popover text-popover-foreground shadow-md animate-in fade-in-80">
<div className="p-1">
{options.map((option) => (
<button
key={option.value}
onClick={() => {
onChange?.(option.value);
setIsOpen(false);
}}
className={`relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none hover:bg-accent hover:text-accent-foreground ${
option.value === value ? 'bg-accent text-accent-foreground' : ''
}`}
>
{option.icon && (
<img
src={option.icon}
alt=""
className="mr-2 h-4 w-4 rounded-full"
/>
)}
{option.label}
</button>
))}
</div>
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,321 @@
"use client";
import * as React from 'react';
import { useEffect, useState, useRef } from 'react';
import { getSupabaseClient } from '../../utils/supabase';
import { AuthChangeEvent } from '@supabase/supabase-js';
import { Loader2, X, Check, Tag } from 'lucide-react';
import { cn } from '@/lib/utils';
// Define Tag type based on the database schema
interface Tag {
id: string;
name: string;
type?: string | null;
attributes?: Record<string, unknown>;
created_at?: string;
updated_at?: string;
deleted_at?: string | null;
parent_tag_id?: string | null;
team_id?: string | null;
is_shared?: boolean;
schema_version?: number | null;
is_system?: boolean;
}
// TagSelector component with multi-select support
export function TagSelector({
value,
onChange,
className,
multiple = false,
teamId,
teamIds,
}: {
value?: string | string[];
onChange?: (tagIds: string | string[]) => void;
className?: string;
multiple?: boolean;
teamId?: string; // Optional single team ID
teamIds?: string[]; // Optional array of team IDs
}) {
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [tags, setTags] = useState<Tag[]>([]);
const [selectedIds, setSelectedIds] = useState<string[]>([]);
const [isOpen, setIsOpen] = useState(false);
const selectorRef = useRef<HTMLDivElement>(null);
// Normalize team IDs to ensure we're always working with an array
const effectiveTeamIds = React.useMemo(() => {
if (teamIds && teamIds.length > 0) {
return teamIds;
} else if (teamId) {
return [teamId];
}
return undefined;
}, [teamId, teamIds]);
// 标签名称与ID的映射函数
const getTagIdByName = (name: string): string | undefined => {
const tag = tags.find(t => t.name === name);
return tag?.id;
};
const getTagNameById = (id: string): string | undefined => {
const tag = tags.find(t => t.id === id);
return tag?.name;
};
// 从标签名称转换为标签ID
const nameToId = (nameOrNames: string | string[] | undefined): string[] => {
if (!nameOrNames) return [];
if (Array.isArray(nameOrNames)) {
return nameOrNames
.map(name => getTagIdByName(name))
.filter((id): id is string => !!id);
}
const id = getTagIdByName(nameOrNames);
return id ? [id] : [];
};
// 从标签ID转换为标签名称
const idToName = (idOrIds: string | string[] | undefined): string[] => {
if (!idOrIds) return [];
if (Array.isArray(idOrIds)) {
return idOrIds
.map(id => getTagNameById(id))
.filter((name): name is string => !!name);
}
const name = getTagNameById(idOrIds);
return name ? [name] : [];
};
// 初始化已选择的标签 - 从传入的名称转换为ID
useEffect(() => {
if (tags.length > 0 && value) {
setSelectedIds(nameToId(value));
}
}, [value, tags]);
// Add click outside listener to close dropdown
useEffect(() => {
function handleClickOutside(event: MouseEvent) {
if (selectorRef.current && !selectorRef.current.contains(event.target as Node)) {
setIsOpen(false);
}
}
// Only add the event listener if the dropdown is open
if (isOpen) {
document.addEventListener('mousedown', handleClickOutside);
return () => {
document.removeEventListener('mousedown', handleClickOutside);
};
}
}, [isOpen]);
useEffect(() => {
let isMounted = true;
const fetchTags = async () => {
if (!isMounted) return;
setLoading(true);
setError(null);
try {
const supabase = getSupabaseClient();
let query = supabase.from('tags').select('*').is('deleted_at', null);
// Filter by team if teamId is provided
if (effectiveTeamIds) {
query = query.in('team_id', effectiveTeamIds);
}
const { data: tagsData, error: tagsError } = await query;
if (tagsError) throw tagsError;
if (!tagsData || tagsData.length === 0) {
if (isMounted) setTags([]);
return;
}
if (isMounted) {
setTags(tagsData as Tag[]);
}
} catch (err) {
if (isMounted) {
setError(err instanceof Error ? err.message : 'Failed to load tags');
}
} finally {
if (isMounted) {
setLoading(false);
}
}
};
const supabase = getSupabaseClient();
const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent) => {
if (event === 'SIGNED_IN') {
fetchTags();
} else if (event === 'SIGNED_OUT') {
setTags([]);
setError(null);
}
});
supabase.auth.getSession().then(() => {
fetchTags();
});
return () => {
isMounted = false;
subscription.unsubscribe();
};
}, [effectiveTeamIds]);
const handleToggle = () => {
if (!loading && !error && tags.length > 0) {
setIsOpen(!isOpen);
}
};
const handleTagSelect = (tagId: string) => {
let newSelected: string[];
if (multiple) {
// For multi-select: toggle tag in/out of selection
if (selectedIds.includes(tagId)) {
newSelected = selectedIds.filter(id => id !== tagId);
} else {
newSelected = [...selectedIds, tagId];
}
} else {
// For single-select: replace selection with the new tag
newSelected = [tagId];
setIsOpen(false);
}
setSelectedIds(newSelected);
// 传递标签名称而不是ID
if (onChange) {
const tagNames = idToName(newSelected);
onChange(multiple ? tagNames : tagNames[0] || '');
}
};
const removeTag = (e: React.MouseEvent, tagId: string) => {
e.stopPropagation();
const newSelected = selectedIds.filter(id => id !== tagId);
setSelectedIds(newSelected);
// 传递标签名称而不是ID
if (onChange) {
const tagNames = idToName(newSelected);
onChange(multiple ? tagNames : tagNames[0] || '');
}
};
if (loading) {
return (
<div className={cn(
"flex w-full items-center justify-between rounded-md border px-3 py-2",
className
)}>
<Loader2 className="h-4 w-4 animate-spin" />
</div>
);
}
if (error) {
return (
<div className={cn(
"flex w-full items-center rounded-md border border-destructive bg-destructive/10 px-3 py-2 text-destructive",
className
)}>
{error}
</div>
);
}
if (tags.length === 0) {
return (
<div className={cn(
"flex w-full items-center rounded-md border px-3 py-2 text-muted-foreground",
className
)}>
No tags available
</div>
);
}
// 根据已选择的ID筛选出已选择的标签
const selectedTags = tags.filter(tag => selectedIds.includes(tag.id));
return (
<div className="relative" ref={selectorRef}>
<div
className={cn(
"flex w-full min-h-10 items-center flex-wrap rounded-md border p-1 cursor-pointer",
isOpen && "ring-2 ring-offset-2 ring-purple-500",
className
)}
onClick={handleToggle}
>
{selectedTags.length > 0 ? (
<div className="flex flex-wrap gap-1">
{selectedTags.map(tag => (
<div
key={tag.id}
className="flex items-center gap-1 bg-purple-100 text-purple-800 rounded-md px-2 py-1 text-sm"
>
{tag.name}
{multiple && (
<X
size={14}
className="cursor-pointer hover:text-purple-900"
onClick={(e) => removeTag(e, tag.id)}
/>
)}
</div>
))}
</div>
) : (
<div className="px-2 py-1 text-gray-500">Select tags</div>
)}
</div>
{isOpen && (
<div className="absolute w-full mt-1 max-h-60 overflow-auto rounded-md border bg-white shadow-lg z-10">
{tags.map(tag => (
<div
key={tag.id}
className={cn(
"px-3 py-2 cursor-pointer hover:bg-gray-100 flex items-center justify-between",
selectedIds.includes(tag.id) && "bg-purple-50"
)}
onClick={() => handleTagSelect(tag.id)}
>
<span className="flex items-center gap-2">
<Tag className="h-4 w-4 text-purple-500" />
<span>{tag.name}</span>
{tag.type && (
<span className="text-xs text-gray-500 bg-gray-100 px-1.5 py-0.5 rounded">
{tag.type}
</span>
)}
</span>
{selectedIds.includes(tag.id) && (
<Check className="h-4 w-4 text-purple-600" />
)}
</div>
))}
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,267 @@
"use client";
import * as React from 'react';
import { useEffect, useState, useRef } from 'react';
import type { Database } from '@/types/supabase';
import { getSupabaseClient } from '../../utils/supabase';
import { AuthChangeEvent, Session } from '@supabase/supabase-js';
import { Loader2, X, Check } from 'lucide-react';
import { cn } from '@/lib/utils';
import { limqRequest } from '@/lib/api';
type Team = Database['limq']['Tables']['teams']['Row'];
// TeamSelector component with multi-select support
export function TeamSelector({
value,
onChange,
className,
multiple = false,
}: {
value?: string | string[];
onChange?: (teamId: string | string[]) => void;
className?: string;
multiple?: boolean;
}) {
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [teams, setTeams] = useState<Team[]>([]);
const [selectedIds, setSelectedIds] = useState<string[]>([]);
const [isOpen, setIsOpen] = useState(false);
const selectorRef = useRef<HTMLDivElement>(null);
// Initialize selected teams based on value prop
useEffect(() => {
if (value) {
if (Array.isArray(value)) {
setSelectedIds(value);
} else {
setSelectedIds(value ? [value] : []);
}
} else {
setSelectedIds([]);
}
}, [value]);
// Add click outside listener to close dropdown
useEffect(() => {
function handleClickOutside(event: MouseEvent) {
if (selectorRef.current && !selectorRef.current.contains(event.target as Node)) {
setIsOpen(false);
}
}
// Only add the event listener if the dropdown is open
if (isOpen) {
document.addEventListener('mousedown', handleClickOutside);
return () => {
document.removeEventListener('mousedown', handleClickOutside);
};
}
}, [isOpen]);
useEffect(() => {
let isMounted = true;
const fetchTeams = async (userId: string) => {
if (!isMounted) return;
setLoading(true);
setError(null);
try {
const supabase = getSupabaseClient();
// 尝试创建默认团队和项目(如果用户还没有)
try {
const response = await limqRequest('team/create-default', 'POST');
console.log('Default team creation response:', response);
} catch (teamError) {
console.error('Error creating default team:', teamError);
}
const { data: memberships, error: membershipError } = await supabase
.from('team_membership')
.select('team_id')
.eq('user_id', userId);
if (membershipError) throw membershipError;
if (!memberships || memberships.length === 0) {
if (isMounted) setTeams([]);
return;
}
const teamIds = memberships.map(m => m.team_id);
const { data: teamsData, error: teamsError } = await supabase
.from('teams')
.select('*')
.in('id', teamIds)
.is('deleted_at', null);
if (teamsError) throw teamsError;
if (isMounted && teamsData) {
setTeams(teamsData);
}
} catch (err) {
if (isMounted) {
setError(err instanceof Error ? err.message : 'Failed to load teams');
}
} finally {
if (isMounted) {
setLoading(false);
}
}
};
const supabase = getSupabaseClient();
const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent, session: Session | null) => {
if (event === 'SIGNED_IN' && session?.user?.id) {
fetchTeams(session.user.id);
} else if (event === 'SIGNED_OUT') {
setTeams([]);
setError(null);
}
});
supabase.auth.getSession().then(({ data: { session } }) => {
if (session?.user?.id) {
fetchTeams(session.user.id);
}
});
return () => {
isMounted = false;
subscription.unsubscribe();
};
}, []);
const handleToggle = () => {
if (!loading && !error && teams.length > 0) {
setIsOpen(!isOpen);
}
};
const handleTeamSelect = (teamId: string) => {
let newSelected: string[];
if (multiple) {
// For multi-select: toggle team in/out of selection
if (selectedIds.includes(teamId)) {
newSelected = selectedIds.filter(id => id !== teamId);
} else {
newSelected = [...selectedIds, teamId];
}
} else {
// For single-select: replace selection with the new team
newSelected = [teamId];
setIsOpen(false);
}
setSelectedIds(newSelected);
if (onChange) {
onChange(multiple ? newSelected : newSelected[0] || '');
}
};
const removeTeam = (e: React.MouseEvent, teamId: string) => {
e.stopPropagation();
const newSelected = selectedIds.filter(id => id !== teamId);
setSelectedIds(newSelected);
if (onChange) {
onChange(multiple ? newSelected : newSelected[0] || '');
}
};
if (loading) {
return (
<div className={cn(
"flex w-full items-center justify-between rounded-md border px-3 py-2",
className
)}>
<Loader2 className="h-4 w-4 animate-spin" />
</div>
);
}
if (error) {
return (
<div className={cn(
"flex w-full items-center rounded-md border border-destructive bg-destructive/10 px-3 py-2 text-destructive",
className
)}>
{error}
</div>
);
}
if (teams.length === 0) {
return (
<div className={cn(
"flex w-full items-center rounded-md border px-3 py-2 text-muted-foreground",
className
)}>
No teams available
</div>
);
}
const selectedTeams = teams.filter(team => selectedIds.includes(team.id));
return (
<div className="relative" ref={selectorRef}>
<div
className={cn(
"flex w-full min-h-10 items-center flex-wrap rounded-md border p-1 cursor-pointer",
isOpen && "ring-2 ring-offset-2 ring-blue-500",
className
)}
onClick={handleToggle}
>
{selectedTeams.length > 0 ? (
<div className="flex flex-wrap gap-1">
{selectedTeams.map(team => (
<div
key={team.id}
className="flex items-center gap-1 bg-blue-100 text-blue-800 rounded-md px-2 py-1 text-sm"
>
{team.name}
{multiple && (
<X
size={14}
className="cursor-pointer hover:text-blue-900"
onClick={(e) => removeTeam(e, team.id)}
/>
)}
</div>
))}
</div>
) : (
<div className="px-2 py-1 text-gray-500">Select a team</div>
)}
</div>
{isOpen && (
<div className="absolute w-full mt-1 max-h-60 overflow-auto rounded-md border bg-white shadow-lg z-10">
{teams.map(team => (
<div
key={team.id}
className={cn(
"px-3 py-2 cursor-pointer hover:bg-gray-100 flex items-center justify-between",
selectedIds.includes(team.id) && "bg-blue-50"
)}
onClick={() => handleTeamSelect(team.id)}
>
<span>{team.name}</span>
{selectedIds.includes(team.id) && (
<Check className="h-4 w-4 text-blue-600" />
)}
</div>
))}
</div>
)}
</div>
);
}

View File

@@ -1,64 +0,0 @@
"use client";
import { useState, useEffect } from 'react';
export default function ThemeToggle() {
const [darkMode, setDarkMode] = useState(false);
// Initialize theme on component mount
useEffect(() => {
const isDarkMode = localStorage.getItem('darkMode') === 'true';
setDarkMode(isDarkMode);
if (isDarkMode) {
document.documentElement.classList.add('dark');
} else {
document.documentElement.classList.remove('dark');
}
}, []);
// Update theme when darkMode state changes
const toggleTheme = () => {
const newDarkMode = !darkMode;
setDarkMode(newDarkMode);
localStorage.setItem('darkMode', newDarkMode.toString());
if (newDarkMode) {
document.documentElement.classList.add('dark');
} else {
document.documentElement.classList.remove('dark');
}
};
return (
<button
onClick={toggleTheme}
className="p-2 rounded-md bg-card-bg border border-card-border hover:bg-card-bg/80 transition-colors"
aria-label={darkMode ? "Switch to light mode" : "Switch to dark mode"}
>
{darkMode ? (
<svg
className="w-5 h-5 text-accent-yellow"
fill="currentColor"
viewBox="0 0 20 20"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
d="M10 2a1 1 0 011 1v1a1 1 0 11-2 0V3a1 1 0 011-1zm4 8a4 4 0 11-8 0 4 4 0 018 0zm-.464 4.95l.707.707a1 1 0 001.414-1.414l-.707-.707a1 1 0 00-1.414 1.414zm2.12-10.607a1 1 0 010 1.414l-.706.707a1 1 0 11-1.414-1.414l.707-.707a1 1 0 011.414 0zM17 11a1 1 0 100-2h-1a1 1 0 100 2h1zm-7 4a1 1 0 011 1v1a1 1 0 11-2 0v-1a1 1 0 011-1zM5.05 6.464A1 1 0 106.465 5.05l-.708-.707a1 1 0 00-1.414 1.414l.707.707zm1.414 8.486l-.707.707a1 1 0 01-1.414-1.414l.707-.707a1 1 0 011.414 1.414zM4 11a1 1 0 100-2H3a1 1 0 000 2h1z"
clipRule="evenodd"
/>
</svg>
) : (
<svg
className="w-5 h-5 text-foreground"
fill="currentColor"
viewBox="0 0 20 20"
xmlns="http://www.w3.org/2000/svg"
>
<path d="M17.293 13.293A8 8 0 016.707 2.707a8.001 8.001 0 1010.586 10.586z" />
</svg>
)}
</button>
);
}

View File

@@ -0,0 +1,403 @@
'use client';
import { useState, useEffect } from 'react';
import { useRouter } from 'next/navigation';
import { useAuth } from '@/lib/auth';
import { limqRequest } from '@/lib/api';
import { TeamSelector } from '@/app/components/ui/TeamSelector';
import { ProjectSelector } from '@/app/components/ui/ProjectSelector';
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
interface ShortUrlData {
originalUrl: string;
customSlug?: string;
title: string;
description?: string;
tags?: string[];
teamId: string;
projectId: string;
domain: string;
}
export default function CreateShortUrlPage() {
return (
<ClientRouteGuard>
<CreateShortUrlForm />
</ClientRouteGuard>
);
}
function CreateShortUrlForm() {
const router = useRouter();
const { user } = useAuth();
const [formData, setFormData] = useState<ShortUrlData>({
originalUrl: '',
customSlug: '',
title: '',
description: '',
tags: [],
teamId: '',
projectId: '',
domain: 'googleads.link',
});
const [tagInput, setTagInput] = useState('');
const [isSubmitting, setIsSubmitting] = useState(false);
const [error, setError] = useState<string | null>(null);
const [success, setSuccess] = useState(false);
// Use useEffect to add user information to form data on load
useEffect(() => {
if (user) {
console.log('Current user:', user.email);
// Can add user-related data to the form here
}
}, [user]);
const handleChange = (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
const { name, value } = e.target;
setFormData(prev => ({
...prev,
[name]: value
}));
};
const handleTagKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
if (e.key === 'Enter' && tagInput.trim()) {
e.preventDefault();
addTag();
}
};
const addTag = () => {
if (tagInput.trim() && !formData.tags?.includes(tagInput.trim())) {
setFormData(prev => ({
...prev,
tags: [...(prev.tags || []), tagInput.trim()]
}));
setTagInput('');
}
};
const removeTag = (tagToRemove: string) => {
setFormData(prev => ({
...prev,
tags: prev.tags?.filter(tag => tag !== tagToRemove)
}));
};
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
setIsSubmitting(true);
setError(null);
try {
// Validate required fields
if (!formData.originalUrl) {
throw new Error('Original URL is required');
}
if (!formData.title) {
throw new Error('Title is required');
}
if (!formData.teamId) {
throw new Error('Team is required');
}
if (!formData.projectId) {
throw new Error('Project is required');
}
if (!formData.domain) {
throw new Error('Domain is required');
}
// Construct request data according to API requirements
const requestData = {
type: "shorturl",
attributes: {
// Can add any additional attributes, but attributes cannot be empty
icon: ""
},
shortUrl: {
url: formData.originalUrl,
slug: formData.customSlug || undefined,
title: formData.title,
name: formData.title,
description: formData.description || "",
domain: formData.domain
},
teamId: formData.teamId,
projectId: formData.projectId,
tagIds: formData.tags && formData.tags.length > 0 ? formData.tags : undefined
};
// Call API to create shorturl resource
const response = await limqRequest('resource/shorturl', 'POST', requestData as unknown as Record<string, unknown>);
console.log('Created successfully:', response);
setSuccess(true);
// Redirect to links list page after 2 seconds
setTimeout(() => {
router.push('/links');
}, 2000);
} catch (err) {
console.error('Failed to create short URL:', err);
setError(err instanceof Error ? err.message : 'Failed to create short URL, please try again later');
} finally {
setIsSubmitting(false);
}
};
return (
<div className="container mx-auto px-4 py-8 max-w-3xl">
<div className="bg-white rounded-lg shadow-md overflow-hidden">
<div className="border-b border-gray-200 bg-blue-50 px-6 py-4">
<h1 className="text-xl font-medium text-gray-900">Create Short URL</h1>
<p className="mt-1 text-sm text-gray-600">
Create a new short URL resource for tracking and analytics
</p>
</div>
{error && (
<div className="bg-red-50 border-l-4 border-red-500 p-4 m-6">
<div className="flex">
<div className="flex-shrink-0">
<svg className="h-5 w-5 text-red-500" fill="currentColor" viewBox="0 0 20 20">
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
</svg>
</div>
<div className="ml-3">
<p className="text-sm text-red-700">{error}</p>
</div>
</div>
</div>
)}
{success && (
<div className="bg-green-50 border-l-4 border-green-500 p-4 m-6">
<div className="flex">
<div className="flex-shrink-0">
<svg className="h-5 w-5 text-green-500" fill="currentColor" viewBox="0 0 20 20">
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clipRule="evenodd" />
</svg>
</div>
<div className="ml-3">
<p className="text-sm text-green-700">
Short URL created successfully! Redirecting...
</p>
</div>
</div>
</div>
)}
<form onSubmit={handleSubmit} className="p-6 space-y-6">
{/* Title */}
<div>
<label htmlFor="title" className="block text-sm font-medium text-gray-700">
Title <span className="text-red-500">*</span>
</label>
<input
type="text"
id="title"
name="title"
value={formData.title}
onChange={handleChange}
placeholder="e.g., Product Launch Campaign"
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
required
/>
</div>
{/* Original URL */}
<div>
<label htmlFor="originalUrl" className="block text-sm font-medium text-gray-700">
Original URL <span className="text-red-500">*</span>
</label>
<input
type="url"
id="originalUrl"
name="originalUrl"
value={formData.originalUrl}
onChange={handleChange}
placeholder="https://example.com/your-long-url"
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
required
/>
</div>
{/* Custom Short Link */}
<div>
<label htmlFor="customSlug" className="block text-sm font-medium text-gray-700">
Custom Short Link <span className="text-gray-500">(Optional)</span>
</label>
<div className="flex mt-1 rounded-md shadow-sm">
<span className="inline-flex items-center px-3 py-2 text-sm text-gray-500 border border-r-0 border-gray-300 rounded-l-md bg-gray-50">
{formData.domain}/
</span>
<input
type="text"
id="customSlug"
name="customSlug"
value={formData.customSlug}
onChange={handleChange}
placeholder="custom-slug"
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-none rounded-r-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
/>
</div>
<p className="mt-1 text-xs text-gray-500">
Leave blank to generate a random short link
</p>
</div>
{/* Domain */}
<div>
<label htmlFor="domain" className="block text-sm font-medium text-gray-700">
Domain <span className="text-red-500">*</span>
</label>
<input
type="text"
id="domain"
name="domain"
value={formData.domain}
onChange={handleChange}
placeholder="e.g., googleads.link"
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
required
/>
</div>
{/* Team Selection */}
<div>
<label htmlFor="teamId" className="block text-sm font-medium text-gray-700">
Team <span className="text-red-500">*</span>
</label>
<div className="mt-1">
<TeamSelector
value={formData.teamId}
onChange={(teamId) => {
setFormData(prev => ({
...prev,
teamId: teamId as string,
// Clear selected project when team changes
projectId: ''
}));
}}
/>
</div>
</div>
{/* Project Selection */}
<div>
<label htmlFor="projectId" className="block text-sm font-medium text-gray-700">
Project <span className="text-red-500">*</span>
</label>
<div className="mt-1">
<ProjectSelector
teamId={formData.teamId}
value={formData.projectId}
onChange={(projectId) => {
setFormData(prev => ({
...prev,
projectId: projectId as string
}));
}}
/>
</div>
</div>
{/* Description */}
<div>
<label htmlFor="description" className="block text-sm font-medium text-gray-700">
Description <span className="text-gray-500">(Optional)</span>
</label>
<textarea
id="description"
name="description"
value={formData.description}
onChange={handleChange}
rows={3}
placeholder="A brief description of this link"
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
/>
</div>
{/* Tags */}
<div>
<label htmlFor="tagInput" className="block text-sm font-medium text-gray-700">
Tags <span className="text-gray-500">(Optional)</span>
</label>
<div className="flex mt-1 rounded-md shadow-sm">
<input
type="text"
id="tagInput"
value={tagInput}
onChange={(e) => setTagInput(e.target.value)}
onKeyDown={handleTagKeyDown}
placeholder="Add a tag and press Enter"
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-l-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
/>
<button
type="button"
onClick={addTag}
className="inline-flex items-center px-3 py-2 text-sm font-medium text-white border border-transparent rounded-r-md shadow-sm bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
>
Add
</button>
</div>
{formData.tags && formData.tags.length > 0 && (
<div className="flex flex-wrap gap-2 mt-2">
{formData.tags.map(tag => (
<span key={tag} className="inline-flex items-center px-2 py-0.5 text-xs font-medium bg-blue-100 rounded-full text-blue-800">
{tag}
<button
type="button"
onClick={() => removeTag(tag)}
className="flex-shrink-0 ml-1 text-blue-500 rounded-full hover:text-blue-700 focus:outline-none"
>
<span className="sr-only">Remove tag {tag}</span>
<svg className="w-3 h-3" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
</svg>
</button>
</span>
))}
</div>
)}
</div>
{/* Submit Button */}
<div className="flex justify-end pt-4 border-t border-gray-200">
<button
type="button"
onClick={() => router.back()}
className="px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 mr-3"
>
Cancel
</button>
<button
type="submit"
disabled={isSubmitting}
className="inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
>
{isSubmitting ? (
<>
<svg className="w-5 h-5 mr-2 -ml-1 animate-spin" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
Processing...
</>
) : 'Create Short URL'}
</button>
</div>
</form>
</div>
</div>
);
}

View File

@@ -30,30 +30,6 @@
--gradient-red: linear-gradient(135deg, #f43f5e, #e11d48);
}
.dark {
/* Dark Mode */
--background: #0f172a;
--foreground: #ffffff;
/* Card colors */
--card-bg: #1e293b;
--card-border: #334155;
/* Vibrant accent colors */
--accent-blue: #3b82f6;
--accent-green: #10b981;
--accent-red: #f43f5e;
--accent-yellow: #f59e0b;
--accent-purple: #8b5cf6;
--accent-pink: #ec4899;
--accent-teal: #14b8a6;
--accent-orange: #f97316;
/* UI colors */
--text-secondary: #94a3b8;
--progress-bg: #334155;
}
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);

View File

@@ -1,21 +1,13 @@
import './globals.css';
import '@radix-ui/themes/styles.css';
import type { Metadata } from 'next';
import { Geist, Geist_Mono } from "next/font/google";
import Navbar from "./components/layout/Navbar";
const geistSans = Geist({
variable: "--font-geist-sans",
subsets: ["latin"],
});
const geistMono = Geist_Mono({
variable: "--font-geist-mono",
subsets: ["latin"],
});
import { AuthProvider } from '@/lib/auth';
import { Theme } from '@radix-ui/themes';
import Header from '@/app/components/layout/Header';
export const metadata: Metadata = {
title: 'ShortURL Analytics',
description: 'Analytics dashboard for short URL management',
description: 'Track and analyze shortened links',
};
export default function RootLayout({
@@ -25,14 +17,14 @@ export default function RootLayout({
}>) {
return (
<html lang="en" suppressHydrationWarning>
<body
className={`${geistSans.variable} ${geistMono.variable} antialiased bg-background`}
>
<Navbar />
<main className="min-h-screen px-4 py-6">
{children}
</main>
<body>
<Theme>
<AuthProvider>
<Header />
{children}
</AuthProvider>
</Theme>
</body>
</html>
);
}
}

View File

@@ -1,21 +0,0 @@
import './globals.css';
import type { Metadata } from 'next';
export const metadata: Metadata = {
title: 'Link Management & Analytics',
description: 'Track and analyze shortened links',
};
export default function RootLayout({
children,
}: Readonly<{
children: React.ReactNode;
}>) {
return (
<html lang="en" suppressHydrationWarning>
<body>
{children}
</body>
</html>
);
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

207
app/login/page.tsx Normal file
View File

@@ -0,0 +1,207 @@
'use client';
import { useState, useEffect, Suspense } from 'react';
import { useRouter, useSearchParams } from 'next/navigation';
import Link from 'next/link';
import { useAuth } from '@/lib/auth';
// Separate component for message handling to isolate useSearchParams
function MessageHandler({ setMessage }: { setMessage: (message: { type: string, content: string }) => void }) {
const searchParams = useSearchParams();
useEffect(() => {
const messageParam = searchParams.get('message');
if (messageParam) {
setMessage({ type: 'info', content: messageParam });
}
}, [searchParams, setMessage]);
return null;
}
export default function LoginPage() {
const router = useRouter();
const { signIn, signInWithGoogle, user } = useAuth();
const [email, setEmail] = useState('');
const [password, setPassword] = useState('');
const [isLoading, setIsLoading] = useState(false);
const [message, setMessage] = useState({ type: '', content: '' });
// 如果用户已登录,重定向到首页
useEffect(() => {
if (user) {
router.push('/');
}
}, [user, router]);
const handleEmailSignIn = async (e: React.FormEvent) => {
e.preventDefault();
if (!email || !password) {
setMessage({
type: 'error',
content: 'Please enter both email and password'
});
return;
}
try {
setIsLoading(true);
setMessage({ type: '', content: '' });
const { error } = await signIn(email, password);
if (error) {
throw new Error(error.message);
}
// 登录成功后,会通过 useEffect 重定向
} catch (error) {
console.error('Login error:', error);
setMessage({
type: 'error',
content: error instanceof Error ? error.message : 'Failed to sign in'
});
setIsLoading(false);
}
};
const handleGoogleSignIn = async () => {
try {
setIsLoading(true);
setMessage({ type: '', content: '' });
const { error } = await signInWithGoogle();
if (error) {
throw new Error(error.message);
}
// Google OAuth will handle the redirect
} catch (error) {
console.error('Google login error:', error);
setMessage({
type: 'error',
content: error instanceof Error ? error.message : 'Failed to sign in with Google'
});
setIsLoading(false);
}
};
return (
<div className="flex items-center justify-center min-h-screen bg-gray-100">
{/* Wrap the component using useSearchParams in Suspense */}
<Suspense fallback={null}>
<MessageHandler setMessage={setMessage} />
</Suspense>
<div className="w-full max-w-md p-8 space-y-8 bg-white rounded-lg shadow-md">
<div className="text-center">
<h1 className="text-2xl font-bold text-gray-900">Login</h1>
<p className="mt-2 text-sm text-gray-600">
Sign in to your account to access analytics
</p>
<div className="mt-2 text-xs text-gray-500">
Welcome to ShortURL Analytics
</div>
</div>
{/* Message display */}
{message.content && (
<div className={`p-4 mb-4 text-sm rounded-lg ${
message.type === 'error'
? 'text-red-700 bg-red-100 border border-red-200'
: 'text-blue-700 bg-blue-50 border border-blue-200'
}`}>
{message.type === 'error' ? (
<span className="font-medium">Error: </span>
) : (
<span className="font-medium">Notice: </span>
)}
{message.content}
</div>
)}
{/* Google Sign In Button */}
<button
type="button"
onClick={handleGoogleSignIn}
disabled={isLoading}
className="w-full flex items-center justify-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
>
<svg className="h-5 w-5 mr-2" viewBox="0 0 24 24" width="24" height="24">
<g transform="matrix(1, 0, 0, 1, 27.009001, -39.238998)">
<path fill="#4285F4" d="M -3.264 51.509 C -3.264 50.719 -3.334 49.969 -3.454 49.239 L -14.754 49.239 L -14.754 53.749 L -8.284 53.749 C -8.574 55.229 -9.424 56.479 -10.684 57.329 L -10.684 60.329 L -6.824 60.329 C -4.564 58.239 -3.264 55.159 -3.264 51.509 Z"/>
<path fill="#34A853" d="M -14.754 63.239 C -11.514 63.239 -8.804 62.159 -6.824 60.329 L -10.684 57.329 C -11.764 58.049 -13.134 58.489 -14.754 58.489 C -17.884 58.489 -20.534 56.379 -21.484 53.529 L -25.464 53.529 L -25.464 56.619 C -23.494 60.539 -19.444 63.239 -14.754 63.239 Z"/>
<path fill="#FBBC05" d="M -21.484 53.529 C -21.734 52.809 -21.864 52.039 -21.864 51.239 C -21.864 50.439 -21.724 49.669 -21.484 48.949 L -21.484 45.859 L -25.464 45.859 C -26.284 47.479 -26.754 49.299 -26.754 51.239 C -26.754 53.179 -26.284 54.999 -25.464 56.619 L -21.484 53.529 Z"/>
<path fill="#EA4335" d="M -14.754 43.989 C -12.984 43.989 -11.404 44.599 -10.154 45.789 L -6.734 42.369 C -8.804 40.429 -11.514 39.239 -14.754 39.239 C -19.444 39.239 -23.494 41.939 -25.464 45.859 L -21.484 48.949 C -20.534 46.099 -17.884 43.989 -14.754 43.989 Z"/>
</g>
</svg>
{isLoading ? 'Signing in...' : 'Sign in with Google'}
</button>
<div className="mt-6 flex items-center justify-center">
<div className="border-t border-gray-300 flex-grow mr-3"></div>
<span className="text-sm text-gray-500">or</span>
<div className="border-t border-gray-300 flex-grow ml-3"></div>
</div>
<form onSubmit={handleEmailSignIn} className="mt-6 space-y-6">
<div>
<label htmlFor="email" className="block text-sm font-medium text-gray-700">
Email address
</label>
<input
id="email"
name="email"
type="email"
autoComplete="email"
required
value={email}
onChange={(e) => setEmail(e.target.value)}
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm placeholder-gray-400 focus:outline-none focus:ring-blue-500 focus:border-blue-500"
placeholder="your@email.com"
disabled={isLoading}
/>
</div>
<div>
<label htmlFor="password" className="block text-sm font-medium text-gray-700">
Password
</label>
<input
id="password"
name="password"
type="password"
autoComplete="current-password"
required
value={password}
onChange={(e) => setPassword(e.target.value)}
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm placeholder-gray-400 focus:outline-none focus:ring-blue-500 focus:border-blue-500"
placeholder="••••••••"
disabled={isLoading}
/>
</div>
<div>
<button
type="submit"
disabled={isLoading}
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
>
{isLoading ? 'Signing in...' : 'Sign in with Email'}
</button>
</div>
</form>
<p className="text-sm mt-6 text-gray-600">
Don&apos;t have an account?{' '}
<Link href="/register" className="font-medium text-blue-600 hover:text-blue-500">
Register
</Link>
</p>
</div>
</div>
);
}

View File

@@ -1,5 +1,5 @@
import { redirect } from 'next/navigation';
export default function Home() {
redirect('/links');
}
redirect('/analytics');
}

195
app/register/page.tsx Normal file
View File

@@ -0,0 +1,195 @@
'use client';
import { useState, FormEvent } from 'react';
import Link from 'next/link';
import { useAuth } from '@/lib/auth';
export default function RegisterPage() {
const [email, setEmail] = useState('');
const [password, setPassword] = useState('');
const [confirmPassword, setConfirmPassword] = useState('');
const [error, setError] = useState<string | null>(null);
const [isLoading, setIsLoading] = useState(false);
const { signUp, signInWithGoogle } = useAuth();
// Handle registration form submission
const handleSubmit = async (e: FormEvent) => {
e.preventDefault();
setError(null);
// Validate passwords
if (password !== confirmPassword) {
setError('Passwords do not match');
return;
}
// Password strength validation
if (password.length < 6) {
setError('Password must be at least 6 characters');
return;
}
setIsLoading(true);
try {
await signUp(email, password);
// After successful registration, redirect to login page with email verification prompt
} catch (error) {
console.error('Registration error:', error);
setError('Registration failed. Please try again later or use a different email');
} finally {
setIsLoading(false);
}
};
// Handle Google registration/login
const handleGoogleSignIn = async () => {
setError(null);
try {
await signInWithGoogle();
// Login flow will redirect to Google and then back to the application
} catch (error) {
console.error('Google sign in error:', error);
setError('Google login failed. Please try again later');
}
};
return (
<div className="flex items-center justify-center min-h-screen bg-gray-100 dark:bg-gray-900">
<div className="w-full max-w-md p-8 space-y-8 bg-white dark:bg-gray-800 rounded-lg shadow-md">
<div className="text-center">
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">Register</h1>
<p className="mt-2 text-sm text-gray-600 dark:text-gray-400">
Create your account to access the analytics dashboard
</p>
</div>
{/* Error message */}
{error && (
<div className="p-4 mb-4 text-sm text-red-700 bg-red-100 dark:bg-red-900 dark:text-red-200 rounded-lg">
{error}
</div>
)}
<form className="mt-8 space-y-6" onSubmit={handleSubmit}>
<div className="space-y-4">
<div>
<label htmlFor="email" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
Email Address
</label>
<input
id="email"
name="email"
type="email"
autoComplete="email"
required
value={email}
onChange={(e) => setEmail(e.target.value)}
className="mt-1 block w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm placeholder-gray-400 dark:placeholder-gray-500 dark:bg-gray-700 dark:text-white focus:outline-none focus:ring-blue-500 focus:border-blue-500"
placeholder="your@email.com"
/>
</div>
<div>
<label htmlFor="password" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
Password
</label>
<input
id="password"
name="password"
type="password"
autoComplete="new-password"
required
value={password}
onChange={(e) => setPassword(e.target.value)}
className="mt-1 block w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm placeholder-gray-400 dark:placeholder-gray-500 dark:bg-gray-700 dark:text-white focus:outline-none focus:ring-blue-500 focus:border-blue-500"
placeholder="********"
/>
</div>
<div>
<label htmlFor="confirmPassword" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
Confirm Password
</label>
<input
id="confirmPassword"
name="confirmPassword"
type="password"
autoComplete="new-password"
required
value={confirmPassword}
onChange={(e) => setConfirmPassword(e.target.value)}
className="mt-1 block w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm placeholder-gray-400 dark:placeholder-gray-500 dark:bg-gray-700 dark:text-white focus:outline-none focus:ring-blue-500 focus:border-blue-500"
placeholder="********"
/>
</div>
</div>
<div>
<button
type="submit"
disabled={isLoading}
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
>
{isLoading ? 'Registering...' : 'Register'}
</button>
</div>
<div className="relative">
<div className="absolute inset-0 flex items-center">
<div className="w-full border-t border-gray-300 dark:border-gray-600"></div>
</div>
<div className="relative flex justify-center text-sm">
<span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">or</span>
</div>
</div>
<div>
<button
type="button"
onClick={handleGoogleSignIn}
className="w-full flex justify-center items-center py-2 px-4 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm text-sm font-medium text-gray-700 dark:text-gray-300 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
>
<svg
className="h-5 w-5 mr-2"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
width="24"
height="24"
>
<g transform="matrix(1, 0, 0, 1, 27.009001, -39.238998)">
<path
fill="#4285F4"
d="M -3.264 51.509 C -3.264 50.719 -3.334 49.969 -3.454 49.239 L -14.754 49.239 L -14.754 53.749 L -8.284 53.749 C -8.574 55.229 -9.424 56.479 -10.684 57.329 L -10.684 60.329 L -6.824 60.329 C -4.564 58.239 -3.264 55.159 -3.264 51.509 Z"
/>
<path
fill="#34A853"
d="M -14.754 63.239 C -11.514 63.239 -8.804 62.159 -6.824 60.329 L -10.684 57.329 C -11.764 58.049 -13.134 58.489 -14.754 58.489 C -17.884 58.489 -20.534 56.379 -21.484 53.529 L -25.464 53.529 L -25.464 56.619 C -23.494 60.539 -19.444 63.239 -14.754 63.239 Z"
/>
<path
fill="#FBBC05"
d="M -21.484 53.529 C -21.734 52.809 -21.864 52.039 -21.864 51.239 C -21.864 50.439 -21.724 49.669 -21.484 48.949 L -21.484 45.859 L -25.464 45.859 C -26.284 47.479 -26.754 49.299 -26.754 51.239 C -26.754 53.179 -26.284 54.999 -25.464 56.619 L -21.484 53.529 Z"
/>
<path
fill="#EA4335"
d="M -14.754 43.989 C -12.984 43.989 -11.404 44.599 -10.154 45.789 L -6.734 42.369 C -8.804 40.429 -11.514 39.239 -14.754 39.239 C -19.444 39.239 -23.494 41.939 -25.464 45.859 L -21.484 48.949 C -20.534 46.099 -17.884 43.989 -14.754 43.989 Z"
/>
</g>
</svg>
Sign up with Google
</button>
</div>
</form>
<div className="mt-6 text-center">
<p className="text-sm text-gray-600 dark:text-gray-400">
Already have an account?{' '}
<Link
href="/login"
className="font-medium text-blue-600 hover:text-blue-500 dark:text-blue-400 dark:hover:text-blue-300"
>
Log in
</Link>
</p>
</div>
</div>
</div>
);
}

484
app/utils/ipLocation.ts Normal file
View File

@@ -0,0 +1,484 @@
interface IpLocationData {
ip: string;
country_name: string;
country_code: string;
city: string;
region: string;
continent_code: string;
continent_name: string;
latitude: number;
longitude: number;
timestamp?: number; // When this data was fetched
}
// In-memory cache
let locationCache: Record<string, IpLocationData> = {};
// Blacklist for IPs that failed to resolve multiple times
let failedIPs: Record<string, { attempts: number, lastAttempt: number }> = {};
// Cache expiration time (30 days in milliseconds)
const CACHE_EXPIRATION = 30 * 24 * 60 * 60 * 1000;
// Max retries for a failed IP
const MAX_RETRY_ATTEMPTS = 3;
// Retry timeout (24 hours in milliseconds)
const RETRY_TIMEOUT = 24 * 60 * 60 * 1000;
// Max number of IPs to batch in a single request
const MAX_BATCH_SIZE = 10;
/**
* Initialize cache from localStorage
*/
const initializeCache = () => {
if (typeof window === 'undefined') return;
try {
// Load location cache
const cachedData = localStorage.getItem('ip-location-cache');
if (cachedData) {
const parsedCache = JSON.parse(cachedData);
// Filter out expired entries
const now = Date.now();
const validEntries = Object.entries(parsedCache).filter(([, data]) => {
const entry = data as IpLocationData;
return entry.timestamp && now - entry.timestamp < CACHE_EXPIRATION;
});
locationCache = Object.fromEntries(validEntries) as Record<string, IpLocationData>;
console.log(`Loaded ${validEntries.length} IP locations from cache`);
}
// Load failed IPs
const failedIPsData = localStorage.getItem('ip-location-blacklist');
if (failedIPsData) {
const parsedFailedIPs = JSON.parse(failedIPsData);
// Filter out expired blacklist entries
const now = Date.now();
const validFailedEntries = Object.entries(parsedFailedIPs).filter(([, data]) => {
const entry = data as { attempts: number, lastAttempt: number };
// Keep entries that have max attempts or haven't timed out yet
return entry.attempts >= MAX_RETRY_ATTEMPTS ||
now - entry.lastAttempt < RETRY_TIMEOUT;
});
failedIPs = Object.fromEntries(validFailedEntries) as Record<string, { attempts: number, lastAttempt: number }>;
console.log(`Loaded ${validFailedEntries.length} blacklisted IPs`);
}
} catch (error) {
console.error('Failed to load IP location cache:', error);
// Reset cache if corrupted
localStorage.removeItem('ip-location-cache');
localStorage.removeItem('ip-location-blacklist');
locationCache = {};
failedIPs = {};
}
};
/**
* Save cache to localStorage
*/
const saveCache = () => {
if (typeof window === 'undefined') return;
try {
localStorage.setItem('ip-location-cache', JSON.stringify(locationCache));
} catch (error) {
console.error('Failed to save IP location cache:', error);
// If localStorage is full, clear old entries
if (error instanceof DOMException && error.name === 'QuotaExceededError') {
// Clear older entries - keep newest 100
const entries = Object.entries(locationCache)
.sort((a, b) => {
const timestampA = (a[1].timestamp || 0);
const timestampB = (b[1].timestamp || 0);
return timestampB - timestampA;
})
.slice(0, 100);
locationCache = Object.fromEntries(entries);
localStorage.setItem('ip-location-cache', JSON.stringify(locationCache));
}
}
};
/**
* Save failed IPs to localStorage
*/
const saveFailedIPs = () => {
if (typeof window === 'undefined') return;
try {
localStorage.setItem('ip-location-blacklist', JSON.stringify(failedIPs));
} catch (error) {
console.error('Failed to save IP blacklist:', error);
// If localStorage is full, limit the size
if (error instanceof DOMException && error.name === 'QuotaExceededError') {
// Keep only IPs with max attempts
const entries = Object.entries(failedIPs)
.filter(([, data]) => data.attempts >= MAX_RETRY_ATTEMPTS);
failedIPs = Object.fromEntries(entries);
localStorage.setItem('ip-location-blacklist', JSON.stringify(failedIPs));
}
}
};
/**
* Check if IP is a private/local address
*/
const isPrivateIP = (ip: string): boolean => {
return (
ip.startsWith('10.') ||
ip.startsWith('192.168.') ||
ip.startsWith('172.16.') ||
ip.startsWith('172.17.') ||
ip.startsWith('172.18.') ||
ip.startsWith('172.19.') ||
ip.startsWith('172.20.') ||
ip.startsWith('172.21.') ||
ip.startsWith('172.22.') ||
ip.startsWith('127.') ||
ip === 'localhost' ||
ip === '::1'
);
};
/**
* Check if an IP should be skipped (blacklisted)
*/
const shouldSkipIP = (ip: string): boolean => {
// If not in failed list, don't skip
if (!failedIPs[ip]) return false;
const now = Date.now();
// If reached max attempts, skip
if (failedIPs[ip].attempts >= MAX_RETRY_ATTEMPTS) {
return true;
}
// If hasn't been long enough since last attempt, skip
if (now - failedIPs[ip].lastAttempt < RETRY_TIMEOUT) {
return true;
}
// Otherwise, we can try again
return false;
};
/**
* Mark IP as failed
*/
const markIPAsFailed = (ip: string): void => {
const now = Date.now();
if (failedIPs[ip]) {
failedIPs[ip] = {
attempts: failedIPs[ip].attempts + 1,
lastAttempt: now
};
} else {
failedIPs[ip] = {
attempts: 1,
lastAttempt: now
};
}
saveFailedIPs();
};
/**
* Get location data for a single IP address
*/
const fetchSingleIP = async (ip: string): Promise<IpLocationData | null> => {
// Skip blacklisted IPs
if (shouldSkipIP(ip)) {
console.log(`Skipping blacklisted IP: ${ip}`);
return null;
}
try {
const response = await fetch(`https://ipapi.co/${ip}/json/`);
if (!response.ok) {
console.error(`Error fetching location for IP ${ip}: ${response.statusText}`);
markIPAsFailed(ip);
return null;
}
const data = await response.json();
if (data.error) {
console.error(`Error fetching location for IP ${ip}: ${data.reason}`);
markIPAsFailed(ip);
return null;
}
// Reset failed attempts if successful
if (failedIPs[ip]) {
delete failedIPs[ip];
saveFailedIPs();
}
const locationData: IpLocationData = {
ip: data.ip,
country_name: data.country_name || 'Unknown',
country_code: data.country_code || 'UN',
city: data.city || 'Unknown',
region: data.region || 'Unknown',
continent_code: data.continent_code || 'UN',
continent_name: getContinentName(data.continent_code) || 'Unknown',
latitude: data.latitude || 0,
longitude: data.longitude || 0,
timestamp: Date.now()
};
return locationData;
} catch (error) {
console.error(`Error fetching location for IP ${ip}:`, error);
markIPAsFailed(ip);
return null;
}
};
/**
* Batch process multiple IPs at once using our own API endpoint
* This is a placeholder - we'll create a server API route for this
*/
const fetchBatchIPs = async (ips: string[]): Promise<Record<string, IpLocationData | null>> => {
try {
// Filter out blacklisted IPs
const validIPs = ips.filter(ip => !shouldSkipIP(ip));
if (validIPs.length === 0) {
return {};
}
const response = await fetch('/api/geo/batch', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ ips: validIPs }),
});
if (!response.ok) {
throw new Error(`Batch request failed: ${response.statusText}`);
}
const results = await response.json();
// Mark failed IPs from results
for (const [ip, data] of Object.entries(results.data)) {
if (!data) {
markIPAsFailed(ip);
} else if (failedIPs[ip]) {
// Reset failed attempts if successful
delete failedIPs[ip];
}
}
saveFailedIPs();
return results.data;
} catch (error) {
console.error('Error in batch IP lookup:', error);
// Fallback to individual requests
const results: Record<string, IpLocationData | null> = {};
for (const ip of ips) {
// Skip blacklisted IPs
if (shouldSkipIP(ip)) {
results[ip] = null;
continue;
}
// Add delays between requests to avoid rate limiting
await new Promise(resolve => setTimeout(resolve, 200));
results[ip] = await fetchSingleIP(ip);
}
return results;
}
};
/**
* Handle private IP addresses
*/
const getPrivateIPData = (ip: string): IpLocationData => ({
ip,
country_name: 'Local Network',
country_code: 'LO',
city: 'Local',
region: 'Local',
continent_code: 'LO',
continent_name: 'Local',
latitude: 0,
longitude: 0,
timestamp: Date.now()
});
/**
* Convert an IP address to location information
* Individual lookup for a single IP
*/
export async function getLocationFromIP(ip: string): Promise<IpLocationData | null> {
// Initialize cache from localStorage if needed
if (Object.keys(locationCache).length === 0) {
initializeCache();
}
// Handle private IP addresses
if (isPrivateIP(ip)) {
const privateIPData = getPrivateIPData(ip);
locationCache[ip] = privateIPData;
return privateIPData;
}
// Skip blacklisted IPs
if (shouldSkipIP(ip)) {
console.log(`Skipping blacklisted IP: ${ip}`);
return null;
}
// Return from cache if available and not expired
if (locationCache[ip]) {
const cachedData = locationCache[ip];
const now = Date.now();
// Return cached data if not expired
if (cachedData.timestamp && now - cachedData.timestamp < CACHE_EXPIRATION) {
return cachedData;
}
}
// Fetch new data
const locationData = await fetchSingleIP(ip);
// Save to cache if successful
if (locationData) {
locationCache[ip] = locationData;
saveCache();
}
return locationData;
}
/**
* Batch lookup for multiple IPs at once
* More efficient than calling getLocationFromIP multiple times
*/
export async function getLocationsFromIPs(ips: string[]): Promise<Record<string, IpLocationData | null>> {
// Initialize cache from localStorage if needed
if (Object.keys(locationCache).length === 0) {
initializeCache();
}
// Filter out IPs that are already in cache and not expired
const now = Date.now();
const cachedResults: Record<string, IpLocationData> = {};
const ipsToFetch: string[] = [];
for (const ip of ips) {
// Handle private IPs
if (isPrivateIP(ip)) {
cachedResults[ip] = getPrivateIPData(ip);
continue;
}
// Skip blacklisted IPs
if (shouldSkipIP(ip)) {
console.log(`Skipping blacklisted IP: ${ip}`);
continue;
}
// Check cache
if (locationCache[ip] && locationCache[ip].timestamp &&
now - locationCache[ip].timestamp < CACHE_EXPIRATION) {
cachedResults[ip] = locationCache[ip];
} else {
ipsToFetch.push(ip);
}
}
// If all IPs were cached or blacklisted, return immediately
if (ipsToFetch.length === 0) {
return cachedResults;
}
// Process IPs in batches to avoid overwhelming the API
const results: Record<string, IpLocationData | null> = { ...cachedResults };
// Process in smaller batches (e.g., 10 IPs at a time)
for (let i = 0; i < ipsToFetch.length; i += MAX_BATCH_SIZE) {
const batchIPs = ipsToFetch.slice(i, i + MAX_BATCH_SIZE);
// Batch request
const batchResults = await fetchBatchIPs(batchIPs);
// Update results and cache
for (const [ip, data] of Object.entries(batchResults)) {
results[ip] = data;
if (data) {
locationCache[ip] = data;
}
}
// Save updated cache
saveCache();
// Add delay between batches
if (i + MAX_BATCH_SIZE < ipsToFetch.length) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
return results;
}
/**
* Get continent name from continent code
*/
function getContinentName(code?: string): string {
if (!code) return 'Unknown';
const continents: Record<string, string> = {
'AF': 'Africa',
'AN': 'Antarctica',
'AS': 'Asia',
'EU': 'Europe',
'NA': 'North America',
'OC': 'Oceania',
'SA': 'South America'
};
return continents[code] || 'Unknown';
}
/**
* Get location information based on view mode
*/
export function getLocationByType(
locationData: IpLocationData | null,
viewMode: 'country' | 'city' | 'region' | 'continent'
): string {
if (!locationData) return 'Unknown';
switch (viewMode) {
case 'country':
return locationData.country_name || 'Unknown';
case 'city':
return locationData.city || 'Unknown';
case 'region':
return locationData.region || 'Unknown';
case 'continent':
return locationData.continent_name || 'Unknown';
default:
return 'Unknown';
}
}

52
app/utils/store.ts Normal file
View File

@@ -0,0 +1,52 @@
import { create } from 'zustand';
import { persist } from 'zustand/middleware';
// Define interface for team, project and tag objects
interface TeamData {
team_id: string;
team_name: string;
[key: string]: unknown;
}
interface ProjectData {
project_id: string;
project_name: string;
[key: string]: unknown;
}
// 定义 ShortUrl 数据类型
export interface ShortUrlData {
id: string;
externalId: string;
slug: string;
originalUrl: string;
title?: string;
shortUrl: string;
teams?: TeamData[];
projects?: ProjectData[];
tags?: string[];
createdAt?: string;
domain?: string;
}
// 定义 store 类型
interface ShortUrlStore {
selectedShortUrl: ShortUrlData | null;
setSelectedShortUrl: (shortUrl: ShortUrlData | null) => void;
clearSelectedShortUrl: () => void;
}
// 创建 store 并使用 persist 中间件保存到 localStorage
export const useShortUrlStore = create<ShortUrlStore>()(
persist(
(set) => ({
selectedShortUrl: null,
setSelectedShortUrl: (shortUrl) => set({ selectedShortUrl: shortUrl }),
clearSelectedShortUrl: () => set({ selectedShortUrl: null }),
}),
{
name: 'shorturl-storage', // localStorage 中的 key 名称
partialize: (state) => ({ selectedShortUrl: state.selectedShortUrl }), // 只持久化 selectedShortUrl
}
)
);

59
app/utils/supabase.ts Normal file
View File

@@ -0,0 +1,59 @@
import { createClient, SupabaseClient } from "@supabase/supabase-js";
import type { Database } from "@/types/supabase";
let supabase: SupabaseClient<Database> | null = null;
// 简单的存储适配器使用localStorage
const storageAdapter = {
getItem: async (key: string) => {
try {
const item = localStorage.getItem(key);
return item;
} catch (error) {
console.error("Storage get error:", error);
return null;
}
},
setItem: async (key: string, value: string) => {
try {
localStorage.setItem(key, value);
} catch (error) {
console.error("Storage set error:", error);
}
},
removeItem: async (key: string) => {
try {
localStorage.removeItem(key);
} catch (error) {
console.error("Storage remove error:", error);
}
},
};
export const getSupabaseClient = (): SupabaseClient<Database> => {
if (!supabase) {
if (!process.env.NEXT_PUBLIC_SUPABASE_URL || !process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY) {
throw new Error('Missing Supabase environment variables');
}
supabase = createClient<Database>(
process.env.NEXT_PUBLIC_SUPABASE_URL,
process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY,
{
db: { schema: "limq" },
auth: {
storage: storageAdapter,
persistSession: true,
autoRefreshToken: true,
},
}
);
}
return supabase;
};
export const clearSupabaseInstance = () => {
supabase = null;
};

151
docs/swagger-setup.md Normal file
View File

@@ -0,0 +1,151 @@
# Setting up Swagger UI in Next.js
This guide explains how to set up Swagger UI in a Next.js application using route groups.
## Directory Structure
The recommended directory structure for Swagger documentation:
```
app/
(swagger)/ # Route group for swagger-related pages
swagger/ # Actual swagger route
page.tsx # Swagger UI component
```
## Installation
1. Add Swagger UI dependencies to your project:
```json
{
"dependencies": {
"swagger-ui-react": "^5.12.0",
"swagger-ui-dist": "^5.12.0"
},
"devDependencies": {
"@types/swagger-ui-react": "^4.18.3"
}
}
```
2. Install webpack style loaders for handling Swagger UI CSS:
```bash
pnpm add -D style-loader css-loader
```
## Next.js Configuration
Create or update `next.config.js` to handle Swagger UI CSS:
```javascript
/** @type {import('next').NextConfig} */
const nextConfig = {
transpilePackages: ['swagger-ui-react'],
webpack: (config) => {
config.module.rules.push({
test: /\.css$/,
use: ['style-loader', 'css-loader'],
});
return config;
},
};
module.exports = nextConfig;
```
## Swagger UI Component
Create `app/(swagger)/swagger/page.tsx`:
```typescript
"use client";
import { useEffect } from 'react';
import SwaggerUI from 'swagger-ui-react';
import 'swagger-ui-react/swagger-ui.css';
export default function SwaggerPage() {
useEffect(() => {
document.title = 'API Documentation - ShortURL Analytics';
}, []);
const swaggerConfig = {
openapi: '3.0.0',
info: {
title: 'Your API Title',
version: '1.0.0',
description: 'API documentation',
contact: {
name: 'API Support',
email: 'support@example.com',
},
license: {
name: 'MIT',
url: 'https://opensource.org/licenses/MIT',
},
},
// ... your API configuration
};
return (
<div className="container mx-auto px-4 py-8">
<div className="mb-8">
<h1 className="text-3xl font-bold mb-2">API Documentation</h1>
<p className="text-gray-600">
Explore and test the API endpoints using the interactive documentation below.
</p>
</div>
<SwaggerUI spec={swaggerConfig} />
</div>
);
}
```
## Best Practices
1. **Route Groups**: Use route groups `(groupname)` to organize related pages without affecting the URL structure.
2. **API Documentation**:
- Add detailed descriptions for all endpoints
- Include parameter descriptions and constraints
- Define response schemas
- Document error responses
- Use appropriate data formats (UUID, URI, etc.)
- Group related endpoints using tags
3. **Swagger Configuration**:
- Add contact information
- Include license details
- Set appropriate servers configuration
- Define required fields
- Add parameter validations (min/max values)
## Common Issues
1. **Route Conflicts**: Avoid parallel routes that resolve to the same path. For example, don't have both `app/swagger/page.tsx` and `app/(group)/swagger/page.tsx` as they would conflict.
2. **CSS Loading**: Make sure to:
- Import Swagger UI CSS
- Configure webpack in `next.config.js`
- Use the `"use client"` directive as Swagger UI is a client-side component
3. **React Version Compatibility**: Be aware of potential peer dependency warnings between Swagger UI React and your React version. You might need to use `--legacy-peer-deps` or adjust your React version accordingly.
## Accessing the Documentation
After setup, your Swagger documentation will be available at `/swagger` in your application. The UI provides:
- Interactive API documentation
- Request/response examples
- Try-it-out functionality
- Schema definitions
- Error responses
## Maintenance
Keep your Swagger documentation up-to-date by:
- Updating the OpenAPI specification when adding or modifying endpoints
- Maintaining accurate parameter descriptions
- Keeping example values relevant
- Updating response schemas when data structures change

View File

@@ -1,8 +1,16 @@
import { executeQuery, executeQuerySingle, buildFilter, buildPagination, buildOrderBy } from './clickhouse';
import type { Event, EventsSummary, TimeSeriesData, GeoData, DeviceAnalytics, DeviceType } from './types';
// 获取事件列表
export async function getEvents(params: {
// 时间粒度枚举
export enum TimeGranularity {
HOUR = 'hour',
DAY = 'day',
WEEK = 'week',
MONTH = 'month'
}
// 事件查询参数类型
export interface EventsQueryParams {
startTime?: string;
endTime?: string;
eventType?: string;
@@ -11,11 +19,18 @@ export async function getEvents(params: {
userId?: string;
teamId?: string;
projectId?: string;
teamIds?: string[];
projectIds?: string[];
tagIds?: string[];
subpath?: string;
page?: number;
pageSize?: number;
sortBy?: string;
sortOrder?: 'asc' | 'desc';
}): Promise<{ events: Event[]; total: number }> {
}
// 获取事件列表
export async function getEvents(params: EventsQueryParams): Promise<{ events: Event[]; total: number }> {
const filter = buildFilter(params);
const pagination = buildPagination(params.page, params.pageSize);
const orderBy = buildOrderBy(params.sortBy, params.sortOrder);
@@ -49,14 +64,20 @@ export async function getEventsSummary(params: {
startTime?: string;
endTime?: string;
linkId?: string;
teamIds?: string[];
projectIds?: string[];
tagIds?: string[];
subpath?: string;
}): Promise<EventsSummary> {
console.log('getEventsSummary received params:', params);
const filter = buildFilter(params);
console.log('getEventsSummary built filter:', filter);
// 获取基本统计数据
const baseQuery = `
SELECT
count() as totalEvents,
uniq(visitor_id) as uniqueVisitors,
uniq(ip_address) as uniqueVisitors,
countIf(event_type = 'conversion') as totalConversions,
avg(time_spent_sec) as averageTimeSpent,
@@ -91,57 +112,71 @@ export async function getEventsSummary(params: {
ORDER BY count DESC
`;
const [baseResult, browserResults, osResults] = await Promise.all([
executeQuerySingle<{
totalEvents: number;
uniqueVisitors: number;
totalConversions: number;
averageTimeSpent: number;
mobileCount: number;
desktopCount: number;
tabletCount: number;
otherCount: number;
}>(baseQuery),
executeQuery<{ name: string; count: number }>(browserQuery),
executeQuery<{ name: string; count: number }>(osQuery)
]);
if (!baseResult) {
throw new Error('Failed to get events summary');
try {
const [baseResult, browserResults, osResults] = await Promise.all([
executeQuerySingle<{
totalEvents: number;
uniqueVisitors: number;
totalConversions: number;
averageTimeSpent: number;
mobileCount: number;
desktopCount: number;
tabletCount: number;
otherCount: number;
}>(baseQuery),
executeQuery<{ name: string; count: number }>(browserQuery),
executeQuery<{ name: string; count: number }>(osQuery)
]);
if (!baseResult) {
throw new Error('Failed to get events summary');
}
// 安全转换数字类型
const safeNumber = (value: any): number => {
if (value === null || value === undefined) return 0;
const num = Number(value);
return isNaN(num) ? 0 : num;
};
// 计算百分比
const calculatePercentage = (count: number, total: number) => {
if (!total) return 0; // 防止除以零
return Number(((count / total) * 100).toFixed(2));
};
// 处理浏览器数据
const browsers = browserResults.map(item => ({
name: item.name || 'Unknown',
count: safeNumber(item.count),
percentage: calculatePercentage(safeNumber(item.count), safeNumber(baseResult.totalEvents))
}));
// 处理操作系统数据
const operatingSystems = osResults.map(item => ({
name: item.name || 'Unknown',
count: safeNumber(item.count),
percentage: calculatePercentage(safeNumber(item.count), safeNumber(baseResult.totalEvents))
}));
return {
totalEvents: safeNumber(baseResult.totalEvents),
uniqueVisitors: safeNumber(baseResult.uniqueVisitors),
totalConversions: safeNumber(baseResult.totalConversions),
averageTimeSpent: baseResult.averageTimeSpent ? Number(baseResult.averageTimeSpent.toFixed(2)) : 0,
deviceTypes: {
mobile: safeNumber(baseResult.mobileCount),
desktop: safeNumber(baseResult.desktopCount),
tablet: safeNumber(baseResult.tabletCount),
other: safeNumber(baseResult.otherCount)
},
browsers,
operatingSystems
};
} catch (error) {
console.error('Error in getEventsSummary:', error);
throw error;
}
// 计算百分比
const calculatePercentage = (count: number, total: number) =>
Number(((count / total) * 100).toFixed(2));
// 处理浏览器数据
const browsers = browserResults.map(item => ({
name: item.name,
count: item.count,
percentage: calculatePercentage(item.count, baseResult.totalEvents)
}));
// 处理操作系统数据
const operatingSystems = osResults.map(item => ({
name: item.name,
count: item.count,
percentage: calculatePercentage(item.count, baseResult.totalEvents)
}));
return {
totalEvents: baseResult.totalEvents,
uniqueVisitors: baseResult.uniqueVisitors,
totalConversions: baseResult.totalConversions,
averageTimeSpent: Number(baseResult.averageTimeSpent.toFixed(2)),
deviceTypes: {
mobile: baseResult.mobileCount,
desktop: baseResult.desktopCount,
tablet: baseResult.tabletCount,
other: baseResult.otherCount
},
browsers,
operatingSystems
};
}
// 获取时间序列数据
@@ -150,6 +185,10 @@ export async function getTimeSeriesData(params: {
endTime: string;
linkId?: string;
granularity: 'hour' | 'day' | 'week' | 'month';
teamIds?: string[];
projectIds?: string[];
tagIds?: string[];
subpath?: string;
}): Promise<TimeSeriesData[]> {
const filter = buildFilter(params);
@@ -165,7 +204,7 @@ export async function getTimeSeriesData(params: {
SELECT
toStartOfInterval(event_time, INTERVAL ${interval}) as timestamp,
count() as events,
uniq(visitor_id) as visitors,
uniq(ip_address) as visitors,
countIf(event_type = 'conversion') as conversions
FROM events
${filter}
@@ -181,23 +220,34 @@ export async function getGeoAnalytics(params: {
startTime?: string;
endTime?: string;
linkId?: string;
groupBy?: 'country' | 'city';
groupBy?: 'country' | 'city' | 'region' | 'continent';
teamIds?: string[];
projectIds?: string[];
tagIds?: string[];
subpath?: string;
}): Promise<GeoData[]> {
const filter = buildFilter(params);
const groupByField = 'ip_address'; // 暂时按 IP 地址分组
// Choose grouping field based on selected view
let groupByField = 'country';
if (params.groupBy === 'city') groupByField = 'city';
else if (params.groupBy === 'region') groupByField = 'region';
else if (params.groupBy === 'continent') groupByField = 'continent';
else if (!params.groupBy) groupByField = 'ip_address'; // Default to IP address if no groupBy is specified
const query = `
SELECT
${groupByField} as location,
COALESCE(${groupByField}, 'Unknown') as location,
'' as area, /* Area column - empty for now */
count() as visits,
uniq(visitor_id) as visitors,
uniq(ip_address) as visitors,
count() * 100.0 / sum(count()) OVER () as percentage
FROM events
${filter}
GROUP BY ${groupByField}
GROUP BY location
HAVING location != ''
ORDER BY visits DESC
LIMIT 10
LIMIT 20
`;
return executeQuery<GeoData>(query);
@@ -208,6 +258,10 @@ export async function getDeviceAnalytics(params: {
startTime?: string;
endTime?: string;
linkId?: string;
teamIds?: string[];
projectIds?: string[];
tagIds?: string[];
subpath?: string;
}): Promise<DeviceAnalytics> {
const filter = buildFilter(params);
@@ -263,8 +317,10 @@ export async function getDeviceAnalytics(params: {
}
// 计算百分比
const calculatePercentage = (count: number) =>
Number(((count / totalResult.total) * 100).toFixed(2));
const calculatePercentage = (count: number) => {
if (!totalResult || totalResult.total === 0) return 0;
return Number(((count / totalResult.total) * 100).toFixed(2));
};
return {
deviceTypes: deviceTypes.map(item => ({

50
lib/api.ts Normal file
View File

@@ -0,0 +1,50 @@
import supabase from './supabase';
// Define response type for API
export interface ApiResponse<T = unknown> {
success: boolean;
data?: T;
error?: string;
message?: string;
}
// Common function for authenticated API requests to LIMQ
export async function limqRequest<T = unknown>(
endpoint: string,
method: 'GET' | 'POST' | 'PUT' | 'DELETE' = 'GET',
data?: Record<string, unknown>
): Promise<ApiResponse<T>> {
// Get current session
const { data: { session } } = await supabase.auth.getSession();
if (!session) {
throw new Error('No active session. User must be authenticated.');
}
const baseUrl = process.env.NEXT_PUBLIC_LIMQ_API;
const url = `${baseUrl}${endpoint.startsWith('/') ? endpoint : '/' + endpoint}`;
const options: RequestInit = {
method,
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${session.access_token}`
},
mode: 'cors'
};
if (data && (method === 'POST' || method === 'PUT')) {
options.body = JSON.stringify(data);
}
const response = await fetch(url, options);
if (!response.ok) {
const errorData = await response.json().catch(() => null);
throw new Error(
errorData?.error || `Request failed with status ${response.status}`
);
}
return response.json();
}

266
lib/auth.tsx Normal file
View File

@@ -0,0 +1,266 @@
'use client';
import React, { createContext, useContext, useEffect, useState } from 'react';
import { useRouter } from 'next/navigation';
import { Session, User } from '@supabase/supabase-js';
import supabase from './supabase';
import { limqRequest } from './api';
// 定义用户类型
export type AuthUser = User | null;
// 定义验证上下文类型
export type AuthContextType = {
user: AuthUser;
session: Session | null;
isLoading: boolean;
signIn: (email: string, password: string) => Promise<{ error?: unknown }>;
signInWithGoogle: () => Promise<{ error?: unknown }>;
signInWithGitHub: () => Promise<{ error?: unknown }>;
signUp: (email: string, password: string) => Promise<void>;
signOut: () => Promise<void>;
};
// 创建验证上下文
const AuthContext = createContext<AuthContextType | undefined>(undefined);
// 验证提供者组件
export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
const [user, setUser] = useState<AuthUser>(null);
const [session, setSession] = useState<Session | null>(null);
const [isLoading, setIsLoading] = useState(true);
const router = useRouter();
// 初始化验证状态
useEffect(() => {
const getSession = async () => {
setIsLoading(true);
try {
// 尝试从Supabase获取会话
const { data: { session }, error } = await supabase.auth.getSession();
if (error) {
console.error('Error getting session:', error);
return;
}
setSession(session);
setUser(session?.user || null);
} catch (error) {
console.error('Unexpected error during getSession:', error);
} finally {
setIsLoading(false);
}
};
getSession();
// 监听验证状态变化
const { data: { subscription } } = supabase.auth.onAuthStateChange((_event, session) => {
setSession(session);
setUser(session?.user || null);
});
// 清理函数
return () => {
subscription.unsubscribe();
};
}, []);
// 登录函数
const signIn = async (email: string, password: string) => {
setIsLoading(true);
try {
console.log('尝试登录:', { email });
// 尝试通过Supabase登录
const { data, error } = await supabase.auth.signInWithPassword({
email,
password,
});
if (error) {
console.error('登录出错:', error);
return { error };
}
setSession(data.session);
setUser(data.user);
router.push('/analytics');
return {};
} catch (error) {
console.error('登录过程出错:', error);
return { error };
} finally {
setIsLoading(false);
}
};
// Google登录函数
const signInWithGoogle = async () => {
setIsLoading(true);
try {
// 获取网站 URL如果环境变量不存在则使用当前来源
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
// 尝试通过Supabase登录Google
const { error } = await supabase.auth.signInWithOAuth({
provider: 'google',
options: {
redirectTo: `${siteUrl}/auth/callback`,
},
});
if (error) {
console.error('Google登录出错:', error);
return { error };
}
return {}; // Return empty object when successful
} catch (error) {
console.error('Google登录过程出错:', error);
return { error };
} finally {
setIsLoading(false);
}
};
// GitHub登录函数
const signInWithGitHub = async () => {
setIsLoading(true);
try {
// 获取网站 URL如果环境变量不存在则使用当前来源
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
// 尝试通过Supabase登录GitHub
const { error } = await supabase.auth.signInWithOAuth({
provider: 'github',
options: {
redirectTo: `${siteUrl}/auth/callback`,
},
});
if (error) {
console.error('GitHub login error:', error);
return { error };
}
return {}; // Return empty object when successful
} catch (error) {
console.error('GitHub login process error:', error);
return { error };
} finally {
setIsLoading(false);
}
};
// 注册函数
const signUp = async (email: string, password: string) => {
setIsLoading(true);
try {
// 获取网站 URL如果环境变量不存在则使用当前来源
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
// 尝试通过Supabase注册
const { error } = await supabase.auth.signUp({
email,
password,
options: {
emailRedirectTo: `${siteUrl}/auth/callback`,
}
});
if (error) {
console.error('注册出错:', error);
throw error;
}
// 注册成功后跳转到登录页面并显示确认消息
router.push('/login?message=Registration successful! Please check your email to verify your account before logging in.');
} catch (error) {
console.error('注册过程出错:', error);
throw error;
} finally {
setIsLoading(false);
}
};
// 登出函数
const signOut = async () => {
setIsLoading(true);
try {
// 尝试通过Supabase登出
const { error } = await supabase.auth.signOut();
if (error) {
console.error('登出出错:', error);
throw error;
}
setSession(null);
setUser(null);
router.push('/login');
} catch (error) {
console.error('登出过程出错:', error);
throw error;
} finally {
setIsLoading(false);
}
};
const contextValue: AuthContextType = {
user,
session,
isLoading,
signIn,
signInWithGoogle,
signInWithGitHub,
signUp,
signOut,
};
return (
<AuthContext.Provider value={contextValue}>
{children}
</AuthContext.Provider>
);
};
// 自定义钩子
export const useAuth = () => {
const context = useContext(AuthContext);
if (context === undefined) {
throw new Error('useAuth must be used within an AuthProvider');
}
return context;
};
// 受保护路由组件
export const ProtectedRoute: React.FC<{ children: React.ReactNode }> = ({ children }) => {
const { user, isLoading } = useAuth();
const router = useRouter();
useEffect(() => {
if (!isLoading && !user) {
router.push('/login');
}
}, [user, isLoading, router]);
if (isLoading) {
return (
<div className="flex items-center justify-center min-h-screen">
<div className="text-center">
<div className="animate-spin rounded-full h-16 w-16 border-t-2 border-b-2 border-blue-500 mx-auto"></div>
<p className="mt-4 text-lg text-gray-700 dark:text-gray-300">...</p>
</div>
</div>
);
}
if (!user) {
return null;
}
return <>{children}</>;
};
export default AuthContext;

View File

@@ -1,12 +1,12 @@
import { createClient } from '@clickhouse/client';
import type { EventsQueryParams } from './types';
import { EventsQueryParams } from './analytics';
// ClickHouse 客户端配置
const clickhouse = createClient({
url: process.env.CLICKHOUSE_URL || 'http://localhost:8123',
username: process.env.CLICKHOUSE_USER || 'admin',
password: process.env.CLICKHOUSE_PASSWORD || 'your_secure_password',
database: process.env.CLICKHOUSE_DB || 'shorturl_analytics'
url: process.env.CLICKHOUSE_URL,
username: process.env.CLICKHOUSE_USER ,
password: process.env.CLICKHOUSE_PASSWORD ,
database: process.env.CLICKHOUSE_DATABASE
});
// 构建日期过滤条件
@@ -26,84 +26,140 @@ function buildDateFilter(startTime?: string, endTime?: string): string {
// 构建通用过滤条件
export function buildFilter(params: Partial<EventsQueryParams>): string {
console.log('buildFilter received params:', JSON.stringify(params));
const filters = [];
// 时间范围过滤
// 添加日期过滤条件
if (params.startTime || params.endTime) {
const dateFilter = buildDateFilter(params.startTime, params.endTime).replace('WHERE ', '');
const dateFilter = buildDateFilter(params.startTime, params.endTime);
if (dateFilter) {
filters.push(dateFilter);
filters.push(dateFilter.replace('WHERE ', ''));
}
}
// 事件类型过滤
// 添加事件类型过滤条件
if (params.eventType) {
filters.push(`event_type = '${params.eventType}'`);
}
// 链接ID过滤
// 添加链接ID过滤条件
if (params.linkId) {
console.log('Adding link_id filter:', params.linkId);
filters.push(`link_id = '${params.linkId}'`);
}
// 链接短码过滤
// 添加链接Slug过滤条件
if (params.linkSlug) {
filters.push(`link_slug = '${params.linkSlug}'`);
}
// 用户ID过滤
// 添加用户ID过滤条件
if (params.userId) {
filters.push(`user_id = '${params.userId}'`);
}
// 团队ID过滤
// 添加子路径过滤条件 - 使用更精确的匹配方式
if (params.subpath && params.subpath.trim() !== '') {
console.log('====== SUBPATH DEBUG ======');
console.log('Raw subpath param:', params.subpath);
// 清理并准备subpath值
let cleanSubpath = params.subpath.trim();
// 移除开头的斜杠以便匹配
if (cleanSubpath.startsWith('/')) {
cleanSubpath = cleanSubpath.substring(1);
}
// 移除结尾的斜杠以便匹配
if (cleanSubpath.endsWith('/')) {
cleanSubpath = cleanSubpath.substring(0, cleanSubpath.length - 1);
}
console.log('Cleaned subpath:', cleanSubpath);
// 使用正则表达式匹配URL中的第二个路径部分
// 示例: 在 "https://abc.com/slug/subpath/" 中匹配 "subpath"
const condition = `match(JSONExtractString(event_attributes, 'full_url'), '/[^/]+/${cleanSubpath}(/|\\\\?|$)')`;
console.log('Final SQL condition:', condition);
console.log('==========================');
filters.push(condition);
}
// 添加团队ID过滤条件
if (params.teamId) {
filters.push(`team_id = '${params.teamId}'`);
}
// 项目ID过滤
// 处理多个团队ID
if (params.teamIds && params.teamIds.length > 0) {
filters.push(`team_id IN (${params.teamIds.map(id => `'${id}'`).join(', ')})`);
}
// 添加项目ID过滤条件
if (params.projectId) {
filters.push(`project_id = '${params.projectId}'`);
}
// 处理多个项目ID
if (params.projectIds && params.projectIds.length > 0) {
filters.push(`project_id IN (${params.projectIds.map(id => `'${id}'`).join(', ')})`);
}
// 处理标签过滤 - 使用LIKE来匹配标签字符串
if (params.tagIds && params.tagIds.length > 0) {
const tagConditions = params.tagIds.map(tag =>
`link_tags LIKE '%${tag}%'`
);
filters.push(`(${tagConditions.join(' OR ')})`);
}
return filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
}
// 构建分页
export function buildPagination(page?: number, pageSize?: number): string {
const limit = pageSize || 20;
const offset = ((page || 1) - 1) * limit;
return `LIMIT ${limit} OFFSET ${offset}`;
// 构建分页条件
export function buildPagination(page: number = 1, pageSize: number = 20): string {
const offset = (page - 1) * pageSize;
return `LIMIT ${pageSize} OFFSET ${offset}`;
}
// 构建排序
export function buildOrderBy(sortBy?: string, sortOrder?: 'asc' | 'desc'): string {
if (!sortBy) {
return 'ORDER BY event_time DESC';
}
return `ORDER BY ${sortBy} ${sortOrder || 'desc'}`;
// 构建排序条件
export function buildOrderBy(sortBy: string = 'event_time', sortOrder: string = 'desc'): string {
return `ORDER BY ${sortBy} ${sortOrder}`;
}
// 执行查询并处理错误
export async function executeQuery<T>(query: string): Promise<T[]> {
// 执行查询
export async function executeQuery(query: string) {
console.log('Executing query:', query); // 查询日志
try {
const resultSet = await clickhouse.query({
query,
format: 'JSONEachRow'
format: 'JSONEachRow',
});
const rows = await resultSet.json<T>();
return Array.isArray(rows) ? rows : [rows];
const rows = await resultSet.json();
return rows;
} catch (error) {
console.error('ClickHouse query error:', error);
console.error('查询执行错误:', error);
throw error;
}
}
// 执行查询并返回单结果
export async function executeQuerySingle<T>(query: string): Promise<T | null> {
const results = await executeQuery<T>(query);
return results.length > 0 ? results[0] : null;
// 执行返回单结果的查询
export async function executeQuerySingle(query: string) {
console.log('Executing single result query:', query); // 查询日志
try {
const resultSet = await clickhouse.query({
query,
format: 'JSONEachRow',
});
const rows = await resultSet.json();
return rows.length > 0 ? rows[0] : null;
} catch (error) {
console.error('单一结果查询执行错误:', error);
throw error;
}
}
export default clickhouse;

65
lib/supabase.ts Normal file
View File

@@ -0,0 +1,65 @@
import { createClient } from '@supabase/supabase-js';
// 从环境变量获取Supabase配置
const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL || '';
const supabaseAnonKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || process.env.SUPABASE_ANON_KEY || '';
console.log('Supabase Configuration Check:', {
urlDefined: !!supabaseUrl,
keyDefined: !!supabaseAnonKey,
url: supabaseUrl,
// 打印部分key以便调试
keyPrefix: supabaseAnonKey ? supabaseAnonKey.substring(0, 20) + '...' : 'undefined',
keyLength: supabaseAnonKey ? supabaseAnonKey.length : 0
});
if (!supabaseUrl || !supabaseAnonKey) {
console.error('Supabase URL and Anon Key are required');
}
// 尝试解码JWT token并打印解码内容
try {
if (supabaseAnonKey) {
const parts = supabaseAnonKey.split('.');
if (parts.length === 3) {
const payload = parts[1];
const decoded = atob(payload);
console.log('JWT Payload:', decoded);
} else {
console.error('Invalid JWT format, expected 3 parts but got:', parts.length);
}
}
} catch (error) {
console.error('JWT解码失败:', error);
}
// 创建Supabase客户端
export const supabase = createClient(supabaseUrl, supabaseAnonKey, {
auth: {
persistSession: true,
autoRefreshToken: true,
detectSessionInUrl: true,
}
});
// 测试Supabase连接
supabase.auth.onAuthStateChange((event, session) => {
console.log(`Supabase auth event: ${event}`, session ? 'Session exists' : 'No session');
if (session) {
console.log('Current user:', session.user.email);
}
});
// 尝试执行健康检查
async function checkSupabaseHealth() {
try {
const { data, error } = await supabase.from('_health').select('*').limit(1);
console.log('Supabase health check:', error ? `Error: ${error.message}` : 'Success', data);
} catch (error) {
console.error('Supabase health check error:', error);
}
}
checkSupabaseHealth();
export default supabase;

View File

@@ -24,6 +24,16 @@ export enum DeviceType {
OTHER = 'other'
}
// 标签类型
export interface Tag {
id: string;
name: string;
color?: string;
type?: string;
attributes?: Record<string, any>;
team_id?: string;
}
// API 响应基础接口
export interface ApiResponse<T> {
success: boolean;
@@ -45,7 +55,10 @@ export interface EventsQueryParams {
linkSlug?: string;
userId?: string;
teamId?: string;
teamIds?: string[]; // 团队ID数组支持多选
projectId?: string;
projectIds?: string[]; // 项目ID数组支持多选
tagIds?: string[]; // 标签ID数组支持多选
page?: number;
pageSize?: number;
sortBy?: string;

6
lib/utils.ts Normal file
View File

@@ -0,0 +1,6 @@
import { type ClassValue, clsx } from 'clsx';
import { twMerge } from 'tailwind-merge';
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
}

22
middleware.ts Normal file
View File

@@ -0,0 +1,22 @@
import { createMiddlewareClient } from '@supabase/auth-helpers-nextjs';
import { NextResponse } from 'next/server';
import type { NextRequest } from 'next/server';
export async function middleware(req: NextRequest) {
const res = NextResponse.next();
// Create a Supabase client configured to use cookies
const supabase = createMiddlewareClient({ req, res });
// Refresh session if expired - required for Server Components
await supabase.auth.getSession();
return res;
}
// Specify the paths where this middleware should run
export const config = {
matcher: [
'/((?!_next/static|_next/image|favicon.ico).*)',
],
};

View File

@@ -3,7 +3,6 @@ import type { NextConfig } from "next";
const nextConfig: NextConfig = {
/* config options here */
// 设置需要转译的包
transpilePackages: [],
// 配置实验性选项
experimental: {
@@ -14,8 +13,8 @@ const nextConfig: NextConfig = {
// 禁用严格模式,避免开发时重复渲染
reactStrictMode: false,
// 设置输出为独立应用
output: 'standalone',
// 暂时禁用standalone输出模式解决构建问题
// output: 'standalone',
// 忽略ESLint错误不会在构建时中断
eslint: {

454
package-lock.json generated
View File

@@ -9,9 +9,16 @@
"version": "0.1.0",
"dependencies": {
"@clickhouse/client": "^1.11.0",
"@types/chart.js": "^2.9.41",
"@types/recharts": "^1.8.29",
"@types/uuid": "^10.0.0",
"chart.js": "^4.4.8",
"date-fns": "^4.1.0",
"next": "15.2.3",
"react": "^19.0.0",
"react-dom": "^19.0.0"
"react-dom": "^19.0.0",
"recharts": "^2.15.1",
"uuid": "^10.0.0"
},
"devDependencies": {
"@eslint/eslintrc": "^3",
@@ -38,6 +45,18 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@babel/runtime": {
"version": "7.27.0",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.0.tgz",
"integrity": "sha512-VtPOkrdPHZsKc/clNqyi9WUA8TINkZ4cGk63UUE3u4pmB2k+ZMQRDuIOagv8UVd6j7k0T3+RRIb7beKTebNbcw==",
"license": "MIT",
"dependencies": {
"regenerator-runtime": "^0.14.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@clickhouse/client": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.11.0.tgz",
@@ -654,6 +673,12 @@
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@kurkle/color": {
"version": "0.3.4",
"resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz",
"integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==",
"license": "MIT"
},
"node_modules/@napi-rs/wasm-runtime": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.7.tgz",
@@ -1136,6 +1161,78 @@
"tslib": "^2.4.0"
}
},
"node_modules/@types/chart.js": {
"version": "2.9.41",
"resolved": "https://registry.npmjs.org/@types/chart.js/-/chart.js-2.9.41.tgz",
"integrity": "sha512-3dvkDvueckY83UyUXtJMalYoH6faOLkWQoaTlJgB4Djde3oORmNP0Jw85HtzTuXyliUHcdp704s0mZFQKio/KQ==",
"license": "MIT",
"dependencies": {
"moment": "^2.10.2"
}
},
"node_modules/@types/d3-array": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz",
"integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==",
"license": "MIT"
},
"node_modules/@types/d3-color": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz",
"integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==",
"license": "MIT"
},
"node_modules/@types/d3-ease": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz",
"integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==",
"license": "MIT"
},
"node_modules/@types/d3-interpolate": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz",
"integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==",
"license": "MIT",
"dependencies": {
"@types/d3-color": "*"
}
},
"node_modules/@types/d3-path": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-1.0.11.tgz",
"integrity": "sha512-4pQMp8ldf7UaB/gR8Fvvy69psNHkTpD/pVw3vmEi8iZAB9EPMBruB1JvHO4BIq9QkUUd2lV1F5YXpMNj7JPBpw==",
"license": "MIT"
},
"node_modules/@types/d3-scale": {
"version": "4.0.9",
"resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz",
"integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==",
"license": "MIT",
"dependencies": {
"@types/d3-time": "*"
}
},
"node_modules/@types/d3-shape": {
"version": "1.3.12",
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-1.3.12.tgz",
"integrity": "sha512-8oMzcd4+poSLGgV0R1Q1rOlx/xdmozS4Xab7np0eamFFUYq71AU9pOCJEFnkXW2aI/oXdVYJzw6pssbSut7Z9Q==",
"license": "MIT",
"dependencies": {
"@types/d3-path": "^1"
}
},
"node_modules/@types/d3-time": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz",
"integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==",
"license": "MIT"
},
"node_modules/@types/d3-timer": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz",
"integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==",
"license": "MIT"
},
"node_modules/@types/estree": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@@ -1171,7 +1268,6 @@
"version": "19.0.12",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.12.tgz",
"integrity": "sha512-V6Ar115dBDrjbtXSrS+/Oruobc+qVbbUxDFC1RSbRqLt5SYvxxyIDrSC85RWml54g+jfNeEMZhEj7wW07ONQhA==",
"dev": true,
"license": "MIT",
"dependencies": {
"csstype": "^3.0.2"
@@ -1187,6 +1283,22 @@
"@types/react": "^19.0.0"
}
},
"node_modules/@types/recharts": {
"version": "1.8.29",
"resolved": "https://registry.npmjs.org/@types/recharts/-/recharts-1.8.29.tgz",
"integrity": "sha512-ulKklaVsnFIIhTQsQw226TnOibrddW1qUQNFVhoQEyY1Z7FRQrNecFCGt7msRuJseudzE9czVawZb17dK/aPXw==",
"license": "MIT",
"dependencies": {
"@types/d3-shape": "^1",
"@types/react": "*"
}
},
"node_modules/@types/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
"license": "MIT"
},
"node_modules/@typescript-eslint/eslint-plugin": {
"version": "8.27.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.27.0.tgz",
@@ -2003,12 +2115,33 @@
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/chart.js": {
"version": "4.4.8",
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.8.tgz",
"integrity": "sha512-IkGZlVpXP+83QpMm4uxEiGqSI7jFizwVtF3+n5Pc3k7sMO+tkd0qxh2OzLhenM0K80xtmAONWGBn082EiBQSDA==",
"license": "MIT",
"dependencies": {
"@kurkle/color": "^0.3.0"
},
"engines": {
"pnpm": ">=8"
}
},
"node_modules/client-only": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
"integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==",
"license": "MIT"
},
"node_modules/clsx": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/color": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
@@ -2080,9 +2213,129 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
"dev": true,
"license": "MIT"
},
"node_modules/d3-array": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
"integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
"license": "ISC",
"dependencies": {
"internmap": "1 - 2"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-color": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
"integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-ease": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz",
"integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-format": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz",
"integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-interpolate": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
"integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
"license": "ISC",
"dependencies": {
"d3-color": "1 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-scale": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
"integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
"license": "ISC",
"dependencies": {
"d3-array": "2.10.0 - 3",
"d3-format": "1 - 3",
"d3-interpolate": "1.2.0 - 3",
"d3-time": "2.1.1 - 3",
"d3-time-format": "2 - 4"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-shape": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
"license": "ISC",
"dependencies": {
"d3-path": "^3.1.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-time": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz",
"integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==",
"license": "ISC",
"dependencies": {
"d3-array": "2 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-time-format": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz",
"integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==",
"license": "ISC",
"dependencies": {
"d3-time": "1 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-timer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz",
"integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/damerau-levenshtein": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",
@@ -2144,6 +2397,16 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/date-fns": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz",
"integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/kossnocorp"
}
},
"node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
@@ -2162,6 +2425,12 @@
}
}
},
"node_modules/decimal.js-light": {
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz",
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==",
"license": "MIT"
},
"node_modules/deep-is": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
@@ -2228,6 +2497,16 @@
"node": ">=0.10.0"
}
},
"node_modules/dom-helpers": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz",
"integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.8.7",
"csstype": "^3.0.2"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -2877,6 +3156,12 @@
"node": ">=0.10.0"
}
},
"node_modules/eventemitter3": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
"integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
"license": "MIT"
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -2884,6 +3169,15 @@
"dev": true,
"license": "MIT"
},
"node_modules/fast-equals": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.2.2.tgz",
"integrity": "sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==",
"license": "MIT",
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/fast-glob": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz",
@@ -3345,6 +3639,15 @@
"node": ">= 0.4"
}
},
"node_modules/internmap": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
"integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/is-array-buffer": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz",
@@ -3786,7 +4089,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true,
"license": "MIT"
},
"node_modules/js-yaml": {
@@ -4151,6 +4453,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
"license": "MIT"
},
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -4162,7 +4470,6 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
"integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"js-tokens": "^3.0.0 || ^4.0.0"
@@ -4228,6 +4535,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/moment": {
"version": "2.30.1",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
"integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
"license": "MIT",
"engines": {
"node": "*"
}
},
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -4346,7 +4662,6 @@
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
@@ -4645,7 +4960,6 @@
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
"dev": true,
"license": "MIT",
"dependencies": {
"loose-envify": "^1.4.0",
@@ -4709,7 +5023,75 @@
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
"dev": true,
"license": "MIT"
},
"node_modules/react-smooth": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz",
"integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==",
"license": "MIT",
"dependencies": {
"fast-equals": "^5.0.1",
"prop-types": "^15.8.1",
"react-transition-group": "^4.4.5"
},
"peerDependencies": {
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/react-transition-group": {
"version": "4.4.5",
"resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
"integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==",
"license": "BSD-3-Clause",
"dependencies": {
"@babel/runtime": "^7.5.5",
"dom-helpers": "^5.0.1",
"loose-envify": "^1.4.0",
"prop-types": "^15.6.2"
},
"peerDependencies": {
"react": ">=16.6.0",
"react-dom": ">=16.6.0"
}
},
"node_modules/recharts": {
"version": "2.15.1",
"resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.1.tgz",
"integrity": "sha512-v8PUTUlyiDe56qUj82w/EDVuzEFXwEHp9/xOowGAZwfLjB9uAy3GllQVIYMWF6nU+qibx85WF75zD7AjqoT54Q==",
"license": "MIT",
"dependencies": {
"clsx": "^2.0.0",
"eventemitter3": "^4.0.1",
"lodash": "^4.17.21",
"react-is": "^18.3.1",
"react-smooth": "^4.0.4",
"recharts-scale": "^0.4.4",
"tiny-invariant": "^1.3.1",
"victory-vendor": "^36.6.8"
},
"engines": {
"node": ">=14"
},
"peerDependencies": {
"react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/recharts-scale": {
"version": "0.4.5",
"resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz",
"integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==",
"license": "MIT",
"dependencies": {
"decimal.js-light": "^2.4.1"
}
},
"node_modules/recharts/node_modules/react-is": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
"integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
"license": "MIT"
},
"node_modules/reflect.getprototypeof": {
@@ -4735,6 +5117,12 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/regenerator-runtime": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
"integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==",
"license": "MIT"
},
"node_modules/regexp.prototype.flags": {
"version": "1.5.4",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz",
@@ -5353,6 +5741,12 @@
"node": ">=6"
}
},
"node_modules/tiny-invariant": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
"integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==",
"license": "MIT"
},
"node_modules/tinyglobby": {
"version": "0.2.12",
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.12.tgz",
@@ -5584,6 +5978,50 @@
"punycode": "^2.1.0"
}
},
"node_modules/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
"integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/victory-vendor": {
"version": "36.9.2",
"resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz",
"integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==",
"license": "MIT AND ISC",
"dependencies": {
"@types/d3-array": "^3.0.3",
"@types/d3-ease": "^3.0.0",
"@types/d3-interpolate": "^3.0.1",
"@types/d3-scale": "^4.0.2",
"@types/d3-shape": "^3.1.0",
"@types/d3-time": "^3.0.0",
"@types/d3-timer": "^3.0.0",
"d3-array": "^3.1.6",
"d3-ease": "^3.0.1",
"d3-interpolate": "^3.0.1",
"d3-scale": "^4.0.2",
"d3-shape": "^3.1.0",
"d3-time": "^3.0.0",
"d3-timer": "^3.0.1"
}
},
"node_modules/victory-vendor/node_modules/@types/d3-shape": {
"version": "3.1.7",
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.7.tgz",
"integrity": "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg==",
"license": "MIT",
"dependencies": {
"@types/d3-path": "*"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

View File

@@ -3,7 +3,7 @@
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev --turbo",
"dev": "next dev -p 3007",
"build": "next build",
"start": "next start",
"lint": "next lint",
@@ -24,20 +24,39 @@
},
"dependencies": {
"@clickhouse/client": "^1.11.0",
"@radix-ui/react-icons": "^1.3.2",
"@radix-ui/react-popover": "^1.1.6",
"@radix-ui/react-select": "^2.1.6",
"@radix-ui/themes": "^3.2.1",
"@supabase/auth-helpers-nextjs": "^0.10.0",
"@types/chart.js": "^2.9.41",
"@types/recharts": "^1.8.29",
"@types/uuid": "^10.0.0",
"chart.js": "^4.4.8",
"clsx": "^2.1.1",
"date-fns": "^4.1.0",
"lucide-react": "^0.486.0",
"next": "15.2.3",
"process": "^0.11.10",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"uuid": "^10.0.0"
"recharts": "^2.15.1",
"tailwind-merge": "^3.1.0",
"uuid": "^10.0.0",
"zustand": "^5.0.3"
},
"devDependencies": {
"@eslint/eslintrc": "^3",
"@supabase/supabase-js": "^2.49.4",
"@tailwindcss/postcss": "^4",
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",
"css-loader": "^7.1.2",
"dotenv": "^16.4.7",
"eslint": "^9",
"eslint-config-next": "15.2.3",
"style-loader": "^4.0.0",
"tailwindcss": "^4",
"typescript": "^5"
},

2799
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,225 +0,0 @@
获取所有表...
数据库 limq 中找到以下表:
- .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb
- .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc
- .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1
- .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0
- .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024
- .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea
- link_daily_stats
- link_events
- link_hourly_patterns
- links
- platform_distribution
- project_daily_stats
- projects
- qr_scans
- qrcode_daily_stats
- qrcodes
- sessions
- team_daily_stats
- team_members
- teams
所有ClickHouse表:
.inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb, .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc, .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1, .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0, .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024, .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea, link_daily_stats, link_events, link_hourly_patterns, links, platform_distribution, project_daily_stats, projects, qr_scans, qrcode_daily_stats, qrcodes, sessions, team_daily_stats, team_members, teams
获取表 .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb 的结构...
获取表 .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc 的结构...
获取表 .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1 的结构...
获取表 .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0 的结构...
获取表 .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024 的结构...
获取表 .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea 的结构...
获取表 link_daily_stats 的结构...
表 link_daily_stats 的列:
- date (Date, 无默认值)
- link_id (String, 无默认值)
- total_clicks (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
- unique_sessions (UInt64, 无默认值)
- total_time_spent (UInt64, 无默认值)
- avg_time_spent (Float64, 无默认值)
- bounce_count (UInt64, 无默认值)
- conversion_count (UInt64, 无默认值)
- unique_referrers (UInt64, 无默认值)
- mobile_count (UInt64, 无默认值)
- tablet_count (UInt64, 无默认值)
- desktop_count (UInt64, 无默认值)
- qr_scan_count (UInt64, 无默认值)
- total_conversion_value (Float64, 无默认值)
获取表 link_events 的结构...
表 link_events 的列:
- event_id (UUID, 默认值: generateUUIDv4())
- event_time (DateTime64(3), 默认值: now64())
- date (Date, 默认值: toDate(event_time))
- link_id (String, 无默认值)
- channel_id (String, 无默认值)
- visitor_id (String, 无默认值)
- session_id (String, 无默认值)
- event_type (Enum8('click' = 1, 'redirect' = 2, 'conversion' = 3, 'error' = 4), 无默认值)
- ip_address (String, 无默认值)
- country (String, 无默认值)
- city (String, 无默认值)
- referrer (String, 无默认值)
- utm_source (String, 无默认值)
- utm_medium (String, 无默认值)
- utm_campaign (String, 无默认值)
- user_agent (String, 无默认值)
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
- browser (String, 无默认值)
- os (String, 无默认值)
- time_spent_sec (UInt32, 默认值: 0)
- is_bounce (Bool, 默认值: true)
- is_qr_scan (Bool, 默认值: false)
- qr_code_id (String, 默认值: '')
- conversion_type (Enum8('visit' = 1, 'stay' = 2, 'interact' = 3, 'signup' = 4, 'subscription' = 5, 'purchase' = 6), 默认值: 'visit')
- conversion_value (Float64, 默认值: 0)
- custom_data (String, 默认值: '{}')
获取表 link_hourly_patterns 的结构...
表 link_hourly_patterns 的列:
- date (Date, 无默认值)
- hour (UInt8, 无默认值)
- link_id (String, 无默认值)
- visits (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
获取表 links 的结构...
表 links 的列:
- link_id (String, 无默认值)
- original_url (String, 无默认值)
- created_at (DateTime64(3), 无默认值)
- created_by (String, 无默认值)
- title (String, 无默认值)
- description (String, 无默认值)
- tags (Array(String), 无默认值)
- is_active (Bool, 默认值: true)
- expires_at (Nullable(DateTime), 无默认值)
- team_id (String, 默认值: '')
- project_id (String, 默认值: '')
获取表 platform_distribution 的结构...
表 platform_distribution 的列:
- date (Date, 无默认值)
- utm_source (String, 无默认值)
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
- visits (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
获取表 project_daily_stats 的结构...
表 project_daily_stats 的列:
- date (Date, 无默认值)
- project_id (String, 无默认值)
- total_clicks (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
- conversion_count (UInt64, 无默认值)
- links_used (UInt64, 无默认值)
- qr_scan_count (UInt64, 无默认值)
获取表 projects 的结构...
表 projects 的列:
- project_id (String, 无默认值)
- team_id (String, 无默认值)
- name (String, 无默认值)
- created_at (DateTime, 无默认值)
- created_by (String, 无默认值)
- description (String, 默认值: '')
- is_archived (Bool, 默认值: false)
- links_count (UInt32, 默认值: 0)
- total_clicks (UInt64, 默认值: 0)
- last_updated (DateTime, 默认值: now())
获取表 qr_scans 的结构...
表 qr_scans 的列:
- scan_id (UUID, 默认值: generateUUIDv4())
- qr_code_id (String, 无默认值)
- link_id (String, 无默认值)
- scan_time (DateTime64(3), 无默认值)
- visitor_id (String, 无默认值)
- location (String, 无默认值)
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
- led_to_conversion (Bool, 默认值: false)
获取表 qrcode_daily_stats 的结构...
表 qrcode_daily_stats 的列:
- date (Date, 无默认值)
- qr_code_id (String, 无默认值)
- total_scans (UInt64, 无默认值)
- unique_scanners (UInt64, 无默认值)
- conversions (UInt64, 无默认值)
- mobile_scans (UInt64, 无默认值)
- tablet_scans (UInt64, 无默认值)
- desktop_scans (UInt64, 无默认值)
- unique_locations (UInt64, 无默认值)
获取表 qrcodes 的结构...
表 qrcodes 的列:
- qr_code_id (String, 无默认值)
- link_id (String, 无默认值)
- team_id (String, 无默认值)
- project_id (String, 默认值: '')
- name (String, 无默认值)
- description (String, 默认值: '')
- created_at (DateTime, 无默认值)
- created_by (String, 无默认值)
- updated_at (DateTime, 默认值: now())
- qr_type (Enum8('standard' = 1, 'custom' = 2, 'dynamic' = 3), 默认值: 'standard')
- image_url (String, 默认值: '')
- design_config (String, 默认值: '{}')
- is_active (Bool, 默认值: true)
- total_scans (UInt64, 默认值: 0)
- unique_scanners (UInt32, 默认值: 0)
获取表 sessions 的结构...
表 sessions 的列:
- session_id (String, 无默认值)
- visitor_id (String, 无默认值)
- link_id (String, 无默认值)
- started_at (DateTime64(3), 无默认值)
- last_activity (DateTime64(3), 无默认值)
- ended_at (Nullable(DateTime64(3)), 无默认值)
- duration_sec (UInt32, 默认值: 0)
- session_pages (UInt8, 默认值: 1)
- is_completed (Bool, 默认值: false)
获取表 team_daily_stats 的结构...
表 team_daily_stats 的列:
- date (Date, 无默认值)
- team_id (String, 无默认值)
- total_clicks (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
- conversion_count (UInt64, 无默认值)
- links_used (UInt64, 无默认值)
- qr_scan_count (UInt64, 无默认值)
获取表 team_members 的结构...
表 team_members 的列:
- team_id (String, 无默认值)
- user_id (String, 无默认值)
- role (Enum8('owner' = 1, 'admin' = 2, 'editor' = 3, 'viewer' = 4), 无默认值)
- joined_at (DateTime, 默认值: now())
- invited_by (String, 无默认值)
- is_active (Bool, 默认值: true)
- last_active (DateTime, 默认值: now())
获取表 teams 的结构...
表 teams 的列:
- team_id (String, 无默认值)
- name (String, 无默认值)
- created_at (DateTime, 无默认值)
- created_by (String, 无默认值)
- description (String, 默认值: '')
- avatar_url (String, 默认值: '')
- is_active (Bool, 默认值: true)
- plan_type (Enum8('free' = 1, 'pro' = 2, 'enterprise' = 3), 无默认值)
- members_count (UInt32, 默认值: 1)
ClickHouse数据库结构检查完成

View File

@@ -0,0 +1,5 @@
-- 添加domain列到shorturl_analytics.shorturl表
ALTER TABLE
shorturl_analytics.shorturl
ADD
COLUMN IF NOT EXISTS domain Nullable(String) COMMENT '域名';

View File

@@ -0,0 +1,9 @@
-- add_req_full_path.sql
-- Add req_full_path column to the shorturl_analytics.events table
ALTER TABLE
shorturl_analytics.events
ADD
COLUMN IF NOT EXISTS req_full_path String COMMENT 'Full request path including query parameters';
-- Display the updated table structure
DESCRIBE TABLE shorturl_analytics.events;

View File

@@ -0,0 +1,41 @@
-- 添加缺失的UTM参数字段到shorturl_analytics.events表
-- 创建日期: 2024-07-02
-- 用途: 增强UTM参数追踪能力
-- 添加utm_term字段 (用于跟踪付费搜索关键词)
ALTER TABLE
shorturl_analytics.events
ADD
COLUMN utm_term String DEFAULT '' AFTER utm_campaign;
-- 添加utm_content字段 (用于区分相同广告的不同版本或A/B测试)
ALTER TABLE
shorturl_analytics.events
ADD
COLUMN utm_content String DEFAULT '' AFTER utm_term;
-- 验证字段添加成功
DESCRIBE TABLE shorturl_analytics.events;
-- 示例查询: 查看UTM参数分析数据
SELECT
utm_source,
utm_medium,
utm_campaign,
utm_term,
utm_content,
COUNT(*) as clicks
FROM
shorturl_analytics.events
WHERE
event_type = 'click'
AND utm_source != ''
GROUP BY
utm_source,
utm_medium,
utm_campaign,
utm_term,
utm_content
ORDER BY
clicks DESC
LIMIT
10;

View File

@@ -0,0 +1,46 @@
-- 使用shorturl_analytics数据库
USE shorturl_analytics;
-- 删除已存在的shorturl表
DROP TABLE IF EXISTS shorturl_analytics.shorturl;
-- 创建shorturl表
CREATE TABLE IF NOT EXISTS shorturl_analytics.shorturl (
-- 短链接基本信息来源于resources表
id String COMMENT '资源ID (resources.id)',
external_id String COMMENT '外部ID (resources.external_id)',
type String COMMENT '类型值为shorturl',
slug String COMMENT '短链接slug (存储在attributes中)',
original_url String COMMENT '原始URL (存储在attributes中)',
title String COMMENT '标题 (存储在attributes中)',
description String COMMENT '描述 (存储在attributes中)',
attributes String DEFAULT '{}' COMMENT '资源属性JSON',
schema_version Int32 COMMENT 'Schema版本',
-- 创建者信息
creator_id String COMMENT '创建者ID (resources.creator_id)',
creator_email String COMMENT '创建者邮箱 (来自users表)',
creator_name String COMMENT '创建者名称 (来自users表)',
-- 时间信息
created_at DateTime64(3) COMMENT '创建时间 (resources.created_at)',
updated_at DateTime64(3) COMMENT '更新时间 (resources.updated_at)',
deleted_at Nullable(DateTime64(3)) COMMENT '删除时间 (resources.deleted_at)',
-- 项目关联 (project_resources表)
projects String DEFAULT '[]' COMMENT '项目关联信息数组。结构: [{project_id: String, project_name: String, project_description: String, assigned_at: DateTime64}]',
-- 团队关联 (通过项目关联到团队)
teams String DEFAULT '[]' COMMENT '团队关联信息数组。结构: [{team_id: String, team_name: String, team_description: String, via_project_id: String}]',
-- 标签关联 (resource_tags表)
tags String DEFAULT '[]' COMMENT '标签关联信息数组。结构: [{tag_id: String, tag_name: String, tag_type: String, created_at: DateTime64}]',
-- QR码关联 (qr_code表)
qr_codes String DEFAULT '[]' COMMENT 'QR码信息数组。结构: [{qr_id: String, scan_count: Int32, url: String, template_name: String, created_at: DateTime64}]',
-- 渠道关联 (channel表)
channels String DEFAULT '[]' COMMENT '渠道信息数组。结构: [{channel_id: String, channel_name: String, channel_path: String, is_user_created: Boolean}]',
-- 收藏关联 (favorite表)
favorites String DEFAULT '[]' COMMENT '收藏信息数组。结构: [{favorite_id: String, user_id: String, user_name: String, created_at: DateTime64}]',
-- 自定义过期时间 (存储在attributes中)
expires_at Nullable(DateTime64(3)) COMMENT '过期时间',
-- 统计信息 (分析时聚合计算)
click_count UInt32 DEFAULT 0 COMMENT '点击次数',
unique_visitors UInt32 DEFAULT 0 COMMENT '唯一访问者数'
) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at)
ORDER BY
(id, created_at) SETTINGS index_granularity = 8192 COMMENT '用于存储所有shorturl类型资源的统一表集成了相关联的项目、团队、标签、QR码、渠道和收藏信息';

View File

@@ -0,0 +1 @@
./ch-query.sh -q "TRUNCATE TABLE shorturl_analytics.events"

View File

@@ -0,0 +1 @@
./ch-query.sh -q "TRUNCATE TABLE shorturl_analytics.shorturl"

View File

@@ -1,364 +0,0 @@
// Sync data from MongoDB trace table to ClickHouse events table
import { getVariable } from "npm:windmill-client@1";
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
interface MongoConfig {
host: string;
port: string;
db: string;
username: string;
password: string;
}
interface ClickHouseConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_database: string;
clickhouse_url: string;
}
interface TraceRecord {
_id: ObjectId;
slugId: ObjectId;
label: string | null;
ip: string;
type: number;
platform: string;
platformOS: string;
browser: string;
browserVersion: string;
url: string;
createTime: number;
}
export async function main(
batch_size = 1000,
max_records = 9999999,
timeout_minutes = 60,
skip_clickhouse_check = false,
force_insert = false
) {
const logWithTimestamp = (message: string) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
// Set timeout
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
return true;
}
return false;
};
// Get MongoDB and ClickHouse connection info
let mongoConfig: MongoConfig;
let clickhouseConfig: ClickHouseConfig;
try {
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
} catch (error) {
console.error("Failed to get config:", error);
throw error;
}
// Build MongoDB connection URL
let mongoUrl = "mongodb://";
if (mongoConfig.username && mongoConfig.password) {
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
}
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
// Connect to MongoDB
const client = new MongoClient();
try {
await client.connect(mongoUrl);
console.log("MongoDB connected successfully");
const db = client.database(mongoConfig.db);
const traceCollection = db.collection<TraceRecord>("trace");
// Build query conditions
const query: Record<string, unknown> = {
type: 1 // Only sync records with type 1
};
// Count total records
const totalRecords = await traceCollection.countDocuments(query);
console.log(`Found ${totalRecords} records to sync`);
const recordsToProcess = Math.min(totalRecords, max_records);
console.log(`Will process ${recordsToProcess} records`);
if (totalRecords === 0) {
console.log("No records to sync, task completed");
return {
success: true,
records_synced: 0,
message: "No records to sync"
};
}
// Check ClickHouse connection
const checkClickHouseConnection = async (): Promise<boolean> => {
if (skip_clickhouse_check) {
logWithTimestamp("Skipping ClickHouse connection check");
return true;
}
try {
logWithTimestamp("Testing ClickHouse connection...");
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
},
body: "SELECT 1",
signal: AbortSignal.timeout(5000)
});
if (response.ok) {
logWithTimestamp("ClickHouse connection test successful");
return true;
} else {
const errorText = await response.text();
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
return false;
}
} catch (err) {
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
return false;
}
};
// Check if records exist in ClickHouse
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
if (records.length === 0) return [];
if (skip_clickhouse_check || force_insert) {
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
return records;
}
try {
const recordIds = records.map(record => record._id.toString());
const query = `
SELECT event_id
FROM ${clickhouseConfig.clickhouse_database}.events
WHERE event_attributes LIKE '%"mongo_id":"%'
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
FORMAT JSON
`;
const response = await fetch(clickhouseConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: query,
signal: AbortSignal.timeout(10000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
}
const result = await response.json();
const existingIds = new Set(result.data.map((row: any) => {
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
return matches ? matches[1] : null;
}).filter(Boolean));
return records.filter(record => !existingIds.has(record._id.toString()));
} catch (err) {
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
return skip_clickhouse_check ? records : [];
}
};
// Process records function
const processRecords = async (records: TraceRecord[]) => {
if (records.length === 0) return 0;
const newRecords = await checkExistingRecords(records);
if (newRecords.length === 0) {
logWithTimestamp("All records already exist, skipping");
return 0;
}
// Prepare ClickHouse insert data
const clickhouseData = newRecords.map(record => {
const eventTime = new Date(record.createTime).toISOString();
return {
event_time: eventTime,
event_type: "click",
event_attributes: JSON.stringify({
mongo_id: record._id.toString(),
original_type: record.type
}),
// Link information
link_id: record.slugId.toString(),
link_slug: "",
link_label: record.label || "",
link_title: "",
link_original_url: record.url || "",
link_attributes: "{}",
link_created_at: eventTime,
link_expires_at: null,
link_tags: "[]",
// User information (empty as not available in trace)
user_id: "",
user_name: "",
user_email: "",
user_attributes: "{}",
// Team information (empty as not available in trace)
team_id: "",
team_name: "",
team_attributes: "{}",
// Project information (empty as not available in trace)
project_id: "",
project_name: "",
project_attributes: "{}",
// QR code information (empty as not available in trace)
qr_code_id: "",
qr_code_name: "",
qr_code_attributes: "{}",
// Visitor information
visitor_id: record._id.toString(),
session_id: `${record._id.toString()}-${record.createTime}`,
ip_address: record.ip || "",
country: "",
city: "",
device_type: record.platform || "unknown",
browser: record.browser || "",
os: record.platformOS || "",
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
// Source information
referrer: record.url || "",
utm_source: "",
utm_medium: "",
utm_campaign: "",
// Interaction information
time_spent_sec: 0,
is_bounce: true,
is_qr_scan: false,
conversion_type: "visit",
conversion_value: 0
};
});
// Generate ClickHouse insert SQL
const insertSQL = `
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
FORMAT JSONEachRow
${JSON.stringify(clickhouseData)}
`;
try {
const response = await fetch(clickhouseConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: insertSQL,
signal: AbortSignal.timeout(20000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
}
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
return newRecords.length;
} catch (err) {
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
throw err;
}
};
// Check ClickHouse connection before processing
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
throw new Error("ClickHouse connection failed, cannot continue sync");
}
// Process records in batches
let processedRecords = 0;
let totalBatchRecords = 0;
for (let page = 0; processedRecords < recordsToProcess; page++) {
if (checkTimeout()) {
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
break;
}
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
const records = await traceCollection.find(
query,
{
allowDiskUse: true,
sort: { createTime: 1 },
skip: page * batch_size,
limit: batch_size
}
).toArray();
if (records.length === 0) {
logWithTimestamp("No more records found, sync complete");
break;
}
const batchSize = await processRecords(records);
processedRecords += records.length;
totalBatchRecords += batchSize;
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
}
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
message: "Data sync completed"
};
} catch (err) {
console.error("Error during sync:", err);
return {
success: false,
error: err instanceof Error ? err.message : String(err),
stack: err instanceof Error ? err.stack : undefined
};
} finally {
await client.close();
console.log("MongoDB connection closed");
}
}

View File

@@ -6,7 +6,7 @@ const config: Config = {
"./components/**/*.{js,ts,jsx,tsx,mdx}",
"./app/**/*.{js,ts,jsx,tsx,mdx}",
],
darkMode: 'class',
darkMode: false,
theme: {
extend: {
colors: {

72
test-supabase-login.mjs Normal file
View File

@@ -0,0 +1,72 @@
// 测试Supabase登录功能
import { config } from 'dotenv';
import { createClient } from '@supabase/supabase-js';
// 加载环境变量
config({ path: '.env.local' });
async function testSupabaseLogin() {
// 获取Supabase配置
const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL;
const supabaseKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || process.env.SUPABASE_ANON_KEY;
console.log('Supabase Configuration:');
console.log('- URL defined:', !!supabaseUrl);
console.log('- Key defined:', !!supabaseKey);
console.log('- URL:', supabaseUrl);
if (!supabaseUrl || !supabaseKey) {
console.error('缺少Supabase配置信息请检查.env.local文件');
return;
}
// 创建Supabase客户端
const supabase = createClient(supabaseUrl, supabaseKey);
console.log('Supabase客户端创建成功');
try {
// 尝试获取会话状态
console.log('检查当前会话...');
const { data: sessionData, error: sessionError } = await supabase.auth.getSession();
if (sessionError) {
console.error('获取会话失败:', sessionError.message);
} else {
console.log('会话状态:', sessionData.session ? '已登录' : '未登录');
}
// 尝试使用测试账户登录
const testEmail = 'test@example.com';
const testPassword = 'password123';
console.log(`\n尝试使用测试账户登录: ${testEmail}`);
const { data, error } = await supabase.auth.signInWithPassword({
email: testEmail,
password: testPassword
});
if (error) {
console.error('登录失败:', error.message);
// 如果登录失败,尝试注册账户
console.log('\n尝试注册测试账户...');
const { data: signUpData, error: signUpError } = await supabase.auth.signUp({
email: testEmail,
password: testPassword
});
if (signUpError) {
console.error('注册失败:', signUpError.message);
} else {
console.log('注册成功:', signUpData);
}
} else {
console.log('登录成功!');
console.log('用户信息:', data.user);
}
} catch (error) {
console.error('发生错误:', error.message);
}
}
testSupabaseLogin();

1
types/react-simple-maps.d.ts vendored Normal file
View File

@@ -0,0 +1 @@

146
types/supabase.ts Normal file
View File

@@ -0,0 +1,146 @@
export type Json =
| string
| number
| boolean
| null
| { [key: string]: Json | undefined }
| Json[]
export interface Database {
public: {
Tables: {
teams: {
Row: {
id: string
name: string
description: string | null
attributes: Json | null
created_at: string | null
updated_at: string | null
deleted_at: string | null
schema_version: number | null
avatar_url: string | null
}
Insert: {
id?: string
name: string
description?: string | null
attributes?: Json | null
created_at?: string | null
updated_at?: string | null
deleted_at?: string | null
schema_version?: number | null
avatar_url?: string | null
}
Update: {
id?: string
name?: string
description?: string | null
attributes?: Json | null
created_at?: string | null
updated_at?: string | null
deleted_at?: string | null
schema_version?: number | null
avatar_url?: string | null
}
}
team_membership: {
Row: {
id: string
team_id: string
user_id: string
is_creator: boolean
role: string
}
Insert: {
id?: string
team_id: string
user_id: string
is_creator?: boolean
role: string
}
Update: {
id?: string
team_id?: string
user_id?: string
is_creator?: boolean
role?: string
}
}
}
Views: {
[_ in never]: never
}
Functions: {
[_ in never]: never
}
Enums: {
[_ in never]: never
}
}
limq: {
Tables: {
teams: {
Row: {
id: string
name: string
description: string | null
avatar_url: string | null
attributes: Json | null
created_at: string
updated_at: string
deleted_at: string | null
}
Insert: {
id?: string
name: string
description?: string | null
avatar_url?: string | null
attributes?: Json | null
created_at?: string
updated_at?: string
deleted_at?: string | null
}
Update: {
id?: string
name?: string
description?: string | null
avatar_url?: string | null
attributes?: Json | null
created_at?: string
updated_at?: string
deleted_at?: string | null
}
}
team_membership: {
Row: {
id: string
team_id: string
user_id: string
role: string
created_at: string
updated_at: string
deleted_at: string | null
}
Insert: {
id?: string
team_id: string
user_id: string
role: string
created_at?: string
updated_at?: string
deleted_at?: string | null
}
Update: {
id?: string
team_id?: string
user_id?: string
role?: string
created_at?: string
updated_at?: string
deleted_at?: string | null
}
}
}
}
}

2
windmill/scripts/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/node_modules
/package-lock.json

View File

@@ -0,0 +1,19 @@
{
"name": "scripts",
"version": "1.0.0",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": "",
"dependencies": {
"date-fns": "^4.1.0",
"dotenv": "^16.5.0",
"fs-extra": "^11.3.0",
"mongodb": "^6.16.0",
"node-fetch": "^2.7.0"
}
}

View File

@@ -0,0 +1,714 @@
// 从MongoDB的trace表同步数据到ClickHouse的events表
//
// 支持以下同步模式:
// 1. 增量同步:基于上次同步状态,只同步新数据(默认模式)
// 2. 自定义时间范围同步:通过指定开始时间和结束时间,同步特定时间范围内的数据
// - 可以通过时间戳参数(startTime/endTime)指定范围
// - 也可以通过日期字符串参数(startDate/endDate)指定范围支持ISO格式或yyyy-MM-dd格式
const { MongoClient, ObjectId } = require('mongodb');
const fs = require('fs');
const path = require('path');
const fetch = require('node-fetch');
// 同步状态键名和保存路径
const SYNC_STATE_FILE = path.join(__dirname, 'mongo_sync_state.json');
// 直接使用配置值
const mongoConfig = {
url: "mongodb://10.0.1.41:27017",
db: "main" // 注意:请替换为您的实际数据库名称
};
const clickhouseConfig = {
clickhouse_host: "10.0.1.60",
clickhouse_port: "8123",
clickhouse_user: "admin",
clickhouse_password: "your_secure_password",
clickhouse_database: "shorturl_analytics",
clickhouse_url: "http://10.0.1.60:8123"
};
// 封装本地读取变量函数
async function getVariable(key) {
try {
if (key === 'f/shorturl_analytics/mongodb') {
return mongoConfig;
} else if (key === 'f/shorturl_analytics/clickhouse') {
return clickhouseConfig;
} else if (key === 'f/shorturl_analytics/mongo_sync_state') {
if (fs.existsSync(SYNC_STATE_FILE)) {
return JSON.parse(fs.readFileSync(SYNC_STATE_FILE, 'utf8'));
}
}
return null;
} catch (error) {
console.error(`获取变量失败: ${error.message}`);
return null;
}
}
// 封装本地保存变量函数
async function setVariable(key, value) {
try {
if (key === 'f/shorturl_analytics/mongo_sync_state') {
fs.writeFileSync(SYNC_STATE_FILE, JSON.stringify(value, null, 2));
}
} catch (error) {
console.error(`保存变量失败: ${error.message}`);
throw error;
}
}
// 日期字符串转时间戳工具函数接受ISO字符串或yyyy-MM-dd格式
function dateToTimestamp(dateStr) {
try {
// 尝试直接解析完整的ISO日期字符串
const date = new Date(dateStr);
// 检查是否为有效日期
if (isNaN(date.getTime())) {
// 尝试解析yyyy-MM-dd格式默认设置为当天的00:00:00
const parts = dateStr.split('-');
if (parts.length === 3) {
const year = parseInt(parts[0], 10);
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
const day = parseInt(parts[2], 10);
const dateObj = new Date(year, month, day, 0, 0, 0);
return dateObj.getTime();
}
throw new Error(`无法解析日期字符串: ${dateStr}`);
}
return date.getTime();
} catch (err) {
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
}
}
// 从URL中提取UTM参数的函数增强版
function extractUtmParams(url, debug = false) {
const defaultUtmParams = {
utm_source: "",
utm_medium: "",
utm_campaign: "",
utm_term: "",
utm_content: ""
};
if (!url) return defaultUtmParams;
if (debug) {
console.log(`[UTM提取] 原始URL: ${url}`);
}
// 准备一个解析后的参数对象
const params = { ...defaultUtmParams };
// 尝试多种方法提取UTM参数
// 方法1: 使用URL对象解析
try {
// 先处理URL确保是完整的URL格式
let normalizedUrl = url;
if (!url.match(/^https?:\/\//i)) {
normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
}
const urlObj = new URL(normalizedUrl);
// 读取URL参数
if (urlObj.searchParams.has('utm_source'))
params.utm_source = urlObj.searchParams.get('utm_source') || "";
if (urlObj.searchParams.has('utm_medium'))
params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
if (urlObj.searchParams.has('utm_campaign'))
params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
if (urlObj.searchParams.has('utm_term'))
params.utm_term = urlObj.searchParams.get('utm_term') || "";
if (urlObj.searchParams.has('utm_content'))
params.utm_content = urlObj.searchParams.get('utm_content') || "";
if (debug) {
console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
}
// 如果至少找到一个UTM参数则返回
if (params.utm_source || params.utm_medium || params.utm_campaign ||
params.utm_term || params.utm_content) {
return params;
}
} catch (err) {
if (debug) {
console.log(`[UTM提取] URL对象解析失败尝试正则表达式`);
}
}
// 方法2: 使用正则表达式提取参数
// 使用正则表达式(最安全的方法,适用于任何格式)
const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
if (sourceMatch && sourceMatch[1]) {
try {
params.utm_source = decodeURIComponent(sourceMatch[1]);
} catch (err) {
params.utm_source = sourceMatch[1];
}
}
const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
if (mediumMatch && mediumMatch[1]) {
try {
params.utm_medium = decodeURIComponent(mediumMatch[1]);
} catch (err) {
params.utm_medium = mediumMatch[1];
}
}
const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
if (campaignMatch && campaignMatch[1]) {
try {
params.utm_campaign = decodeURIComponent(campaignMatch[1]);
} catch (err) {
params.utm_campaign = campaignMatch[1];
}
}
const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
if (termMatch && termMatch[1]) {
try {
params.utm_term = decodeURIComponent(termMatch[1]);
} catch (err) {
params.utm_term = termMatch[1];
}
}
const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
if (contentMatch && contentMatch[1]) {
try {
params.utm_content = decodeURIComponent(contentMatch[1]);
} catch (err) {
params.utm_content = contentMatch[1];
}
}
if (debug) {
console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
}
return params;
}
// 解析命令行参数
function parseCommandLineArgs() {
const args = {};
process.argv.slice(2).forEach(arg => {
if (arg.startsWith('--')) {
const [key, value] = arg.substring(2).split('=');
args[key] = value || true;
}
});
return args;
}
async function main() {
const args = parseCommandLineArgs();
// 参数设置
const batch_size = parseInt(args['batch-size'] || '1000');
const max_records = parseInt(args['max-records'] || '9999999');
const timeout_minutes = parseInt(args['timeout'] || '60');
const skip_clickhouse_check = args['skip-clickhouse-check'] === 'true';
const force_insert = args['force-insert'] !== 'false';
const database_override = args['database'] || 'shorturl_analytics';
const reset_sync_state = args['reset-sync-state'] === 'true';
const debug_utm = args['debug-utm'] === 'true';
const start_time = args['start-time'] ? parseInt(args['start-time']) : undefined;
const end_time = args['end-time'] ? parseInt(args['end-time']) : undefined;
const use_custom_time_range = args['use-custom-time-range'] === 'true';
const start_date = args['start-date'];
const end_date = args['end-date'];
const logWithTimestamp = (message) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
let customStartTime = start_time;
let customEndTime = end_time;
let useCustomTimeRange = use_custom_time_range;
// 处理日期字符串参数,转换为时间戳
if (start_date) {
try {
customStartTime = dateToTimestamp(start_date);
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${customStartTime}`);
useCustomTimeRange = true;
} catch (err) {
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
}
}
if (end_date) {
try {
customEndTime = dateToTimestamp(end_date);
// 如果是日期格式,设置为当天结束时间 (23:59:59.999)
if (end_date.split('-').length === 3 && end_date.length <= 10) {
customEndTime += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${customEndTime}`);
} else {
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${customEndTime}`);
}
useCustomTimeRange = true;
} catch (err) {
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
}
}
if (skip_clickhouse_check) {
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式不会检查记录是否已存在");
}
if (force_insert) {
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
}
if (reset_sync_state) {
logWithTimestamp("⚠️ 警告: 已启用重置同步状态,将从头开始同步数据");
}
if (debug_utm) {
logWithTimestamp("已启用UTM参数调试日志");
}
if (useCustomTimeRange) {
if (customStartTime) {
logWithTimestamp(`已启用自定义时间范围:开始时间 ${new Date(customStartTime).toISOString()}`);
}
if (customEndTime) {
logWithTimestamp(`已启用自定义时间范围:结束时间 ${new Date(customEndTime).toISOString()}`);
}
}
// 设置超时
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
// 检查是否超时
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
console.log(`运行时间超过${timeout_minutes}分钟,暂停执行`);
return true;
}
return false;
};
// 获取上次同步状态
let lastSyncState = null;
if (!reset_sync_state) {
try {
const rawSyncState = await getVariable("f/shorturl_analytics/mongo_sync_state");
if (rawSyncState) {
lastSyncState = rawSyncState;
}
} catch (error) {
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
}
}
if (lastSyncState) {
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
if (lastSyncState.last_sync_id) {
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
}
} else {
logWithTimestamp("没有找到上次同步状态,将从头开始同步");
}
// 连接MongoDB
const client = new MongoClient(mongoConfig.url);
try {
await client.connect();
console.log("MongoDB连接成功");
const db = client.db(mongoConfig.db);
const traceCollection = db.collection("trace");
const shortCollection = db.collection("short");
// 构建查询条件
const query = {
type: 1 // 只同步type为1的记录
};
// 根据时间范围参数构建查询条件
if (useCustomTimeRange) {
// 使用自定义时间范围
const timeQuery = {};
if (customStartTime) {
timeQuery.$gte = customStartTime;
logWithTimestamp(`将只同步createTime >= ${customStartTime} (${new Date(customStartTime).toISOString()}) 的记录`);
}
if (customEndTime) {
timeQuery.$lte = customEndTime;
logWithTimestamp(`将只同步createTime <= ${customEndTime} (${new Date(customEndTime).toISOString()}) 的记录`);
}
// 只有当至少指定了一个时间限制时才添加时间查询条件
if (Object.keys(timeQuery).length > 0) {
query.createTime = timeQuery;
}
}
// 如果不使用自定义时间范围,且有上次同步状态,则只获取更新的记录
else if (lastSyncState && lastSyncState.last_sync_time) {
// 使用上次同步时间作为过滤条件
query.createTime = { $gt: lastSyncState.last_sync_time };
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
}
// 计算总记录数
const totalRecords = await traceCollection.countDocuments(query);
console.log(`找到 ${totalRecords} 条新记录需要同步`);
// 限制此次处理的记录数量
const recordsToProcess = Math.min(totalRecords, max_records);
console.log(`本次将处理 ${recordsToProcess} 条记录`);
if (totalRecords === 0) {
console.log("没有新记录需要同步,任务完成");
return {
success: true,
records_synced: 0,
message: "没有新记录需要同步"
};
}
// 检查ClickHouse连接状态
const checkClickHouseConnection = async () => {
if (skip_clickhouse_check) {
logWithTimestamp("已启用跳过ClickHouse检查不测试连接");
return true;
}
try {
logWithTimestamp("测试ClickHouse连接...");
const clickhouseUrl = clickhouseConfig.clickhouse_url;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`,
},
body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
});
if (response.ok) {
logWithTimestamp("ClickHouse连接测试成功");
return true;
} else {
const errorText = await response.text();
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
return false;
}
} catch (err) {
logWithTimestamp(`ClickHouse连接测试失败: ${err.message}`);
return false;
}
};
// 在处理记录前先检查ClickHouse连接
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
logWithTimestamp("⚠️ ClickHouse连接测试失败请启用skip_clickhouse_check=true参数来跳过连接检查");
throw new Error("ClickHouse连接失败无法继续同步");
}
// 处理记录的函数
const processRecords = async (records) => {
if (records.length === 0) return 0;
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条记录...`);
// 强制使用所有记录,不检查重复
const newRecords = records;
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
// 获取链接信息
const slugIds = newRecords.map(record => new ObjectId(record.slugId));
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
const shortLinks = await shortCollection.find({
_id: { $in: slugIds }
}).toArray();
// 创建映射用于快速查找
const shortLinksMap = new Map(shortLinks.map((link) => [link._id.toString(), link]));
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息,${newRecords.length - shortLinks.length} 条数据将使用占位符`);
// 准备ClickHouse插入数据
const clickhouseData = newRecords.map(record => {
const eventTime = new Date(record.createTime);
// 获取对应的短链接信息
const shortLink = shortLinksMap.get(record.slugId.toString());
// 提取URL中的UTM参数
if (debug_utm && record.url) {
logWithTimestamp(`======== UTM参数调试 ========`);
logWithTimestamp(`记录ID: ${record._id.toString()}`);
logWithTimestamp(`原始URL: ${record.url}`);
}
const utmParams = extractUtmParams(record.url || "", debug_utm);
if (debug_utm) {
logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
logWithTimestamp(`===========================`);
}
// 保存提取的UTM参数和URL到event_attributes
const eventAttributes = {
mongo_id: record._id.toString(),
url: record.url || "",
...(record.url ? { raw_url: record.url } : {})
};
// 转换MongoDB记录为ClickHouse格式匹配ClickHouse表结构
return {
// UUID将由ClickHouse自动生成 (event_id)
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
event_type: record.type === 1 ? "visit" : "custom",
event_attributes: JSON.stringify(eventAttributes),
link_id: record.slugId.toString(),
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
link_label: record.label || "",
link_title: shortLink?.title || "unknown_title", // 使用占位符
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
link_created_at: shortLink?.createTime
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
link_expires_at: shortLink?.expiresAt
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
: null,
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
user_id: shortLink?.user || "unknown_user", // 使用占位符
user_name: "unknown_user", // 使用占位符
user_email: "",
user_attributes: "{}",
team_id: shortLink?.teamId || "unknown_team", // 使用占位符
team_name: "unknown_team", // 使用占位符
team_attributes: "{}",
project_id: shortLink?.projectId || "unknown_project", // 使用占位符
project_name: "unknown_project", // 使用占位符
project_attributes: "{}",
qr_code_id: "",
qr_code_name: "",
qr_code_attributes: "{}",
visitor_id: record._id.toString(),
session_id: record._id.toString() + "-" + record.createTime,
ip_address: record.ip || "0.0.0.0", // 使用占位符
country: "",
city: "",
device_type: record.platform || "unknown",
browser: record.browser || "unknown", // 使用占位符
os: record.platformOS || "unknown", // 使用占位符
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
referrer: record.url || "",
utm_source: utmParams.utm_source || "",
utm_medium: utmParams.utm_medium || "",
utm_campaign: utmParams.utm_campaign || "",
utm_term: utmParams.utm_term || "",
utm_content: utmParams.utm_content || "",
time_spent_sec: 0,
is_bounce: true,
is_qr_scan: false,
conversion_type: "visit",
conversion_value: 0,
req_full_path: record.url || ""
};
});
// 生成ClickHouse插入SQL
const insertSQL = `
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
(event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title,
link_original_url, link_attributes, link_created_at, link_expires_at, link_tags,
user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes,
project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes,
visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent,
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
VALUES ${clickhouseData.map(record => {
// 确保所有字符串值都是字符串类型,并安全处理替换
const safeReplace = (val) => {
// 确保值是字符串如果是null或undefined则使用空字符串
const str = val === null || val === undefined ? "" : String(val);
// 安全替换单引号
return str.replace(/'/g, "''");
};
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
'${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}',
'${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}',
${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}',
'${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}',
'${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}',
'${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}',
'${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}',
'${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}',
'${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}',
'${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}',
'${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}',
'${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}',
'${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan},
'${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
}).join(", ")}
`;
if (insertSQL.length === 0) {
console.log("没有新记录需要插入");
return 0;
}
// 发送请求到ClickHouse
const clickhouseUrl = clickhouseConfig.clickhouse_url;
try {
logWithTimestamp("发送插入请求到ClickHouse...");
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`
},
body: insertSQL,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
}
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
return newRecords.length;
} catch (err) {
logWithTimestamp(`向ClickHouse插入数据失败: ${err.message}`);
throw err;
}
};
// 批量处理记录
let processedRecords = 0;
let totalBatchRecords = 0;
let lastSyncTime = 0;
for (let page = 0; processedRecords < recordsToProcess; page++) {
// 检查超时
if (checkTimeout()) {
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
break;
}
// 每批次都输出进度
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
// 查询MongoDB数据
const records = await traceCollection.find(query)
.sort({ createTime: 1 })
.skip(page * batch_size)
.limit(batch_size)
.toArray();
if (records.length === 0) {
logWithTimestamp("没有找到更多数据,同步结束");
break;
}
// 找到数据,开始处理
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
// 输出当前批次的部分数据信息
if (records.length > 0) {
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
if (records.length > 1) {
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
}
// 如果开启了调试输出一些URL样本
if (debug_utm) {
const sampleSize = Math.min(5, records.length);
logWithTimestamp(`URL样本 (前${sampleSize}条):`);
for (let i = 0; i < sampleSize; i++) {
if (records[i].url) {
logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
}
}
}
}
const batchSize = await processRecords(records);
processedRecords += records.length;
totalBatchRecords += batchSize;
// 更新最后处理的记录时间和ID
if (records.length > 0) {
const lastRecord = records[records.length - 1];
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
}
logWithTimestamp(`${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
}
// 更新同步状态
if (processedRecords > 0 && lastSyncTime > 0) {
// 只在非自定义时间范围模式下更新同步状态
if (!useCustomTimeRange) {
// 创建新的同步状态,简化对象结构
const newSyncState = {
last_sync_time: lastSyncTime,
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords
};
try {
// 保存同步状态
await setVariable("f/shorturl_analytics/mongo_sync_state", newSyncState);
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
} catch (err) {
logWithTimestamp(`更新同步状态失败: ${err.message},将继续执行`);
}
} else {
logWithTimestamp("使用自定义时间范围模式,不更新全局同步状态");
}
}
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
message: useCustomTimeRange ? "自定义时间范围数据同步完成" : "数据同步完成",
custom_time_range_used: useCustomTimeRange
};
} catch (err) {
console.error("同步过程中发生错误:", err);
return {
success: false,
error: err.message,
stack: err.stack
};
} finally {
// 关闭MongoDB连接
await client.close();
console.log("MongoDB连接已关闭");
}
}
// 执行主函数
main().then(result => {
console.log("任务执行结果:", result);
process.exit(result.success ? 0 : 1);
}).catch(err => {
console.error("执行出错:", err);
process.exit(1);
});

View File

@@ -0,0 +1,522 @@
// 从MongoDB的main.short表同步数据到PostgreSQL的short_url.shorturl表
import { getVariable, setVariable, getResource } from "npm:windmill-client@1";
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
interface MongoConfig {
host: string;
port: string;
db: string;
username: string;
password: string;
}
interface PostgresConfig {
host: string;
port: number;
database: string;
user: string;
password: string;
schema: string;
}
// 扩展ShortRecord接口以包含更多可能的字段
interface ShortRecord {
_id: ObjectId;
origin: string;
slug: string;
domain: string | null;
createTime: number | { $numberLong: string } | string;
// 可选字段
expiredAt?: number | { $numberLong: string } | string | null;
expiredUrl?: string | null;
password?: string | null;
image?: string | null;
title?: string | null;
description?: string | null;
}
interface SyncState {
last_sync_time: number;
records_synced: number;
last_sync_id?: string;
}
// 同步状态键名
const SYNC_STATE_KEY = "f/limq/mongo_short_to_postgres_shorturl_shorturl_state";
export async function main(
batch_size = 1000,
max_records = 9999999,
timeout_minutes = 60,
skip_duplicate_check = false,
force_insert = false,
reset_sync_state = false,
postgres_schema = "short_url", // 添加schema参数允许运行时指定
postgres_database = "postgres", // 添加数据库名称参数默认为postgres
domain = "upj.to" // 添加domain参数允许用户指定域名
) {
const logWithTimestamp = (message: string) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("开始执行MongoDB到PostgreSQL的同步任务");
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
logWithTimestamp(`使用域名: ${domain}`);
if (skip_duplicate_check) {
logWithTimestamp("⚠️ 警告: 已启用跳过重复检查模式,不会检查记录是否已存在");
}
if (force_insert) {
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
}
if (reset_sync_state) {
logWithTimestamp("⚠️ 警告: 已启用重置同步状态,将从头开始同步数据");
}
// 设置超时
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
// 检查是否超时
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
logWithTimestamp(`运行时间超过${timeout_minutes}分钟,暂停执行`);
return true;
}
return false;
};
// 日期解析函数,处理不同格式的日期
const parseDate = (dateValue: any): Date | null => {
if (!dateValue) return null;
// 处理 MongoDB $numberLong 格式
if (dateValue.$numberLong) {
return new Date(Number(dateValue.$numberLong));
}
// 处理普通时间戳
if (typeof dateValue === 'number') {
return new Date(dateValue);
}
// 处理 ISO 字符串格式
if (typeof dateValue === 'string') {
const date = new Date(dateValue);
return isNaN(date.getTime()) ? null : date;
}
return null;
};
// 获取MongoDB和PostgreSQL的连接信息
let mongoConfig: MongoConfig;
let postgresConfig: PostgresConfig;
try {
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
if (typeof rawMongoConfig === "string") {
try {
mongoConfig = JSON.parse(rawMongoConfig);
} catch (e) {
console.error("MongoDB配置解析失败:", e);
throw e;
}
} else {
mongoConfig = rawMongoConfig as MongoConfig;
}
// 使用getResource获取PostgreSQL资源
try {
logWithTimestamp("正在获取PostgreSQL资源...");
const resourceConfig = await getResource("f/limq/production_supabase");
// 将resource转换为PostgresConfig
postgresConfig = {
host: resourceConfig.host || "",
port: Number(resourceConfig.port) || 5432,
user: resourceConfig.user || "",
password: resourceConfig.password || "",
database: resourceConfig.database || postgres_database, // 使用提供的数据库名称作为备选
schema: resourceConfig.schema || postgres_schema // 使用提供的schema作为备选
};
// 检查并记录配置信息
if (!postgresConfig.database || postgresConfig.database === "undefined") {
postgresConfig.database = postgres_database;
logWithTimestamp(`数据库名称未指定或为"undefined",使用提供的值: ${postgresConfig.database}`);
}
if (!postgresConfig.schema || postgresConfig.schema === "undefined") {
postgresConfig.schema = postgres_schema;
logWithTimestamp(`Schema未指定或为"undefined",使用提供的值: ${postgresConfig.schema}`);
}
logWithTimestamp(`PostgreSQL配置: 数据库=${postgresConfig.database}, Schema=${postgresConfig.schema}`);
} catch (e) {
console.error("获取PostgreSQL资源失败:", e);
throw e;
}
console.log("MongoDB配置:", JSON.stringify({
...mongoConfig,
password: "****" // 隐藏密码
}));
console.log("PostgreSQL配置:", JSON.stringify({
...postgresConfig,
password: "****" // 隐藏密码
}));
} catch (error) {
console.error("获取配置失败:", error);
throw error;
}
// 获取上次同步状态
let lastSyncState: SyncState | null = null;
if (!reset_sync_state) {
try {
const rawSyncState = await getVariable(SYNC_STATE_KEY);
if (rawSyncState) {
if (typeof rawSyncState === "string") {
try {
lastSyncState = JSON.parse(rawSyncState);
} catch (e) {
logWithTimestamp(`解析上次同步状态失败: ${e}, 将从头开始同步`);
}
} else {
lastSyncState = rawSyncState as SyncState;
}
}
} catch (error) {
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
}
}
if (lastSyncState) {
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
if (lastSyncState.last_sync_id) {
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
}
} else {
logWithTimestamp("没有找到上次同步状态,将从头开始同步");
}
// 构建MongoDB连接URL
let mongoUrl = "mongodb://";
if (mongoConfig.username && mongoConfig.password) {
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
}
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
// 构建PostgreSQL连接URL
const pgConnectionString = `postgres://${postgresConfig.user}:${postgresConfig.password}@${postgresConfig.host}:${postgresConfig.port}/${postgresConfig.database}`;
console.log(`PostgreSQL连接URL: ${pgConnectionString.replace(/:[^:]*@/, ":****@")}`);
// 连接MongoDB
const mongoClient = new MongoClient();
let pgClient: Client | null = null;
try {
await mongoClient.connect(mongoUrl);
logWithTimestamp("MongoDB连接成功");
// 连接PostgreSQL
pgClient = new Client(pgConnectionString);
await pgClient.connect();
logWithTimestamp("PostgreSQL连接成功");
// 确认PostgreSQL schema存在
try {
await pgClient.queryArray(`SELECT 1 FROM information_schema.schemata WHERE schema_name = '${postgresConfig.schema}'`);
logWithTimestamp(`PostgreSQL schema '${postgresConfig.schema}' 已确认存在`);
} catch (error) {
logWithTimestamp(`检查PostgreSQL schema失败: ${error}`);
throw new Error(`Schema '${postgresConfig.schema}' 可能不存在`);
}
const db = mongoClient.database(mongoConfig.db);
const shortCollection = db.collection<ShortRecord>("short");
// 构建查询条件,根据上次同步状态获取新记录
const query: Record<string, unknown> = {};
// 如果有上次同步状态,则只获取更新的记录
if (lastSyncState && lastSyncState.last_sync_time) {
// 使用上次同步时间作为过滤条件
query.createTime = { $gt: lastSyncState.last_sync_time };
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
}
// 计算总记录数
const totalRecords = await shortCollection.countDocuments(query);
logWithTimestamp(`找到 ${totalRecords} 条新记录需要同步`);
// 限制此次处理的记录数量
const recordsToProcess = Math.min(totalRecords, max_records);
logWithTimestamp(`本次将处理 ${recordsToProcess} 条记录`);
if (totalRecords === 0) {
logWithTimestamp("没有新记录需要同步,任务完成");
return {
success: true,
records_synced: 0,
message: "没有新记录需要同步"
};
}
// 检查记录是否已经存在于PostgreSQL中
const checkExistingRecords = async (records: ShortRecord[]): Promise<ShortRecord[]> => {
if (records.length === 0) return [];
// 如果跳过重复检查或强制插入,则直接返回所有记录
if (skip_duplicate_check || force_insert) {
logWithTimestamp(`已跳过重复检查,准备处理所有 ${records.length} 条记录`);
return records;
}
logWithTimestamp(`正在检查 ${records.length} 条记录是否已存在于PostgreSQL中...`);
try {
// 提取所有记录的slugs
const slugs = records.map(record => record.slug);
// 查询PostgreSQL中是否已存在这些slugs
const result = await pgClient!.queryArray(`
SELECT slug FROM ${postgresConfig.schema}.shorturl
WHERE slug = ANY($1::text[])
`, [slugs]);
// 将已存在的slugs加入到集合中
const existingSlugs = new Set<string>();
for (const row of result.rows) {
existingSlugs.add(row[0] as string);
}
logWithTimestamp(`检测到 ${existingSlugs.size} 条记录已存在于PostgreSQL中`);
// 过滤出不存在的记录
const newRecords = records.filter(record => !existingSlugs.has(record.slug));
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
return newRecords;
} catch (err) {
const error = err as Error;
logWithTimestamp(`PostgreSQL查询出错: ${error.message}`);
if (skip_duplicate_check) {
logWithTimestamp("已启用跳过重复检查,将继续处理所有记录");
return records;
} else {
throw error;
}
}
};
// 处理记录的函数
const processRecords = async (records: ShortRecord[]) => {
if (records.length === 0) return 0;
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条记录...`);
// 检查记录是否已存在
let newRecords;
try {
newRecords = await checkExistingRecords(records);
} catch (err) {
const error = err as Error;
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
if (!skip_duplicate_check && !force_insert) {
throw error;
}
// 如果跳过检查或强制插入,则使用所有记录
logWithTimestamp("将使用所有记录进行处理");
newRecords = records;
}
if (newRecords.length === 0) {
logWithTimestamp("所有记录都已存在,跳过处理");
return 0;
}
logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
// 批量插入PostgreSQL
try {
// 开始事务
await pgClient!.queryArray('BEGIN');
let insertedCount = 0;
// 由于参数可能很多,按小批次处理
const smallBatchSize = 100;
for (let i = 0; i < newRecords.length; i += smallBatchSize) {
const batchRecords = newRecords.slice(i, i + smallBatchSize);
// 构造批量插入语句
const placeholders = [];
const values = [];
let valueIndex = 1;
for (const record of batchRecords) {
// 参考提供的字段处理方式处理数据
const createdAt = parseDate(record.createTime);
const updatedAt = createdAt; // 设置更新时间等于创建时间
const fullShortUrl = `${domain}/${record.slug}`;
placeholders.push(`($${valueIndex}, $${valueIndex+1}, $${valueIndex+2}, $${valueIndex+3}, $${valueIndex+4}, $${valueIndex+5}, $${valueIndex+6}, $${valueIndex+7}, $${valueIndex+8}, $${valueIndex+9}, $${valueIndex+10}, $${valueIndex+11}, $${valueIndex+12})`);
values.push(
record._id.toString(), // id
record.slug, // slug
domain, // domain (使用提供的域名)
record.slug, // name (使用slug作为name)
record.slug, // title (使用slug作为title)
record.origin || '', // origin
createdAt, // created_at
updatedAt, // updated_at
fullShortUrl, // full_short_url
record.image || null, // image
record.description || null, // description
record.expiredUrl || null, // expired_url
parseDate(record.expiredAt) // expired_at
);
valueIndex += 13;
}
const query = `
INSERT INTO ${postgresConfig.schema}.shorturl
(id, slug, domain, name, title, origin, created_at, updated_at, full_short_url, image, description, expired_url, expired_at)
VALUES ${placeholders.join(', ')}
`;
await pgClient!.queryArray(query, values);
insertedCount += batchRecords.length;
logWithTimestamp(`已插入 ${insertedCount}/${newRecords.length} 条记录`);
}
// 提交事务
await pgClient!.queryArray('COMMIT');
logWithTimestamp(`成功插入 ${insertedCount} 条记录到PostgreSQL`);
return insertedCount;
} catch (err) {
const error = err as Error;
// 发生错误,回滚事务
await pgClient!.queryArray('ROLLBACK');
logWithTimestamp(`向PostgreSQL插入数据失败: ${error.message}`);
throw error;
}
};
// 批量处理记录
let processedRecords = 0;
let totalBatchRecords = 0;
let lastSyncTime = 0;
let lastSyncId = "";
for (let page = 0; processedRecords < recordsToProcess; page++) {
// 检查超时
if (checkTimeout()) {
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
break;
}
// 每批次都输出进度
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
const records = await shortCollection.find(
query,
{
sort: { createTime: 1 },
skip: page * batch_size,
limit: batch_size
}
).toArray();
if (records.length === 0) {
logWithTimestamp("没有找到更多数据,同步结束");
break;
}
// 找到数据,开始处理
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
// 输出当前批次的部分数据信息
if (records.length > 0) {
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, slug=${records[0].slug}, 时间=${new Date(typeof records[0].createTime === 'number' ? records[0].createTime : 0).toISOString()}`);
if (records.length > 1) {
const lastRec = records[records.length-1];
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${lastRec._id}, slug=${lastRec.slug}, 时间=${new Date(typeof lastRec.createTime === 'number' ? lastRec.createTime : 0).toISOString()}`);
}
}
const batchSize = await processRecords(records);
processedRecords += records.length;
totalBatchRecords += batchSize;
// 更新最后处理的记录时间和ID
if (records.length > 0) {
const lastRecord = records[records.length - 1];
// 提取数字时间戳
let lastCreateTime = 0;
if (typeof lastRecord.createTime === 'number') {
lastCreateTime = lastRecord.createTime;
} else if (lastRecord.createTime && lastRecord.createTime.$numberLong) {
lastCreateTime = Number(lastRecord.createTime.$numberLong);
}
lastSyncTime = Math.max(lastSyncTime, lastCreateTime);
lastSyncId = lastRecord._id.toString();
}
logWithTimestamp(`${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
}
// 更新同步状态
if (processedRecords > 0 && lastSyncTime > 0) {
// 创建新的同步状态
const newSyncState: SyncState = {
last_sync_time: lastSyncTime,
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + totalBatchRecords,
last_sync_id: lastSyncId
};
try {
// 保存同步状态
await setVariable(SYNC_STATE_KEY, newSyncState);
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
} catch (err) {
const error = err as Error;
logWithTimestamp(`更新同步状态失败: ${error.message}`);
}
}
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
message: "数据同步完成"
};
} catch (err) {
const error = err as Error;
console.error("同步过程中发生错误:", error);
return {
success: false,
error: error.message,
stack: error.stack
};
} finally {
// 关闭连接
if (pgClient) {
await pgClient.end();
logWithTimestamp("PostgreSQL连接已关闭");
}
await mongoClient.close();
logWithTimestamp("MongoDB连接已关闭");
}
}

View File

@@ -1,5 +1,13 @@
// Sync data from MongoDB trace table to ClickHouse events table
import { getVariable } from "npm:windmill-client@1";
// MongoDBtrace表同步数据到ClickHouseevents
//
// 支持以下同步模式:
// 1. 增量同步:基于上次同步状态,只同步新数据(默认模式)
// 2. 自定义时间范围同步:通过指定开始时间和结束时间,同步特定时间范围内的数据
// - 可以通过时间戳参数(start_time/end_time)指定范围
// - 也可以通过日期字符串参数(start_date/end_date)指定范围支持ISO格式或yyyy-MM-dd格式
//
// 使用自定义时间范围时,将不会更新同步状态,避免干扰增量同步进度
import { getVariable, setVariable } from "npm:windmill-client@1";
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
interface MongoConfig {
@@ -15,6 +23,7 @@ interface ClickHouseConfig {
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_database: string;
clickhouse_url: string;
}
@@ -32,6 +41,7 @@ interface TraceRecord {
createTime: number;
}
// 添加 ShortRecord 接口定义
interface ShortRecord {
_id: ObjectId;
slug: string; // 短链接的slug部分
@@ -48,9 +58,161 @@ interface ShortRecord {
projectId?: string; // 项目ID
}
interface ClickHouseRow {
event_id: string;
event_attributes: string;
interface SyncState {
last_sync_time: number;
records_synced: number;
last_sync_id?: string;
}
// 定义UTM参数接口
interface UtmParams {
utm_source: string;
utm_medium: string;
utm_campaign: string;
utm_term: string;
utm_content: string;
}
// 同步状态键名
const SYNC_STATE_KEY = "f/shorturl_analytics/mongo_sync_state";
// 日期字符串转时间戳工具函数接受ISO字符串或yyyy-MM-dd格式
function dateToTimestamp(dateStr: string): number {
try {
// 尝试直接解析完整的ISO日期字符串
const date = new Date(dateStr);
// 检查是否为有效日期
if (isNaN(date.getTime())) {
// 尝试解析yyyy-MM-dd格式默认设置为当天的00:00:00
const parts = dateStr.split('-');
if (parts.length === 3) {
const year = parseInt(parts[0], 10);
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
const day = parseInt(parts[2], 10);
const dateObj = new Date(year, month, day, 0, 0, 0);
return dateObj.getTime();
}
throw new Error(`无法解析日期字符串: ${dateStr}`);
}
return date.getTime();
} catch (err) {
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
}
}
// 从URL中提取UTM参数的函数增强版
function extractUtmParams(url: string, debug = false): UtmParams {
const defaultUtmParams: UtmParams = {
utm_source: "",
utm_medium: "",
utm_campaign: "",
utm_term: "",
utm_content: ""
};
if (!url) return defaultUtmParams;
if (debug) {
console.log(`[UTM提取] 原始URL: ${url}`);
}
// 准备一个解析后的参数对象
const params: UtmParams = { ...defaultUtmParams };
// 尝试多种方法提取UTM参数
// 方法1: 使用URL对象解析
try {
// 先处理URL确保是完整的URL格式
let normalizedUrl = url;
if (!url.match(/^https?:\/\//i)) {
normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
}
const urlObj = new URL(normalizedUrl);
// 读取URL参数
if (urlObj.searchParams.has('utm_source'))
params.utm_source = urlObj.searchParams.get('utm_source') || "";
if (urlObj.searchParams.has('utm_medium'))
params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
if (urlObj.searchParams.has('utm_campaign'))
params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
if (urlObj.searchParams.has('utm_term'))
params.utm_term = urlObj.searchParams.get('utm_term') || "";
if (urlObj.searchParams.has('utm_content'))
params.utm_content = urlObj.searchParams.get('utm_content') || "";
if (debug) {
console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
}
// 如果至少找到一个UTM参数则返回
if (params.utm_source || params.utm_medium || params.utm_campaign ||
params.utm_term || params.utm_content) {
return params;
}
} catch (_err) {
if (debug) {
console.log(`[UTM提取] URL对象解析失败尝试正则表达式`);
}
}
// 方法2: 使用正则表达式提取参数
// 使用正则表达式(最安全的方法,适用于任何格式)
const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
if (sourceMatch && sourceMatch[1]) {
try {
params.utm_source = decodeURIComponent(sourceMatch[1]);
} catch (_) {
params.utm_source = sourceMatch[1];
}
}
const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
if (mediumMatch && mediumMatch[1]) {
try {
params.utm_medium = decodeURIComponent(mediumMatch[1]);
} catch (_) {
params.utm_medium = mediumMatch[1];
}
}
const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
if (campaignMatch && campaignMatch[1]) {
try {
params.utm_campaign = decodeURIComponent(campaignMatch[1]);
} catch (_) {
params.utm_campaign = campaignMatch[1];
}
}
const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
if (termMatch && termMatch[1]) {
try {
params.utm_term = decodeURIComponent(termMatch[1]);
} catch (_) {
params.utm_term = termMatch[1];
}
}
const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
if (contentMatch && contentMatch[1]) {
try {
params.utm_content = decodeURIComponent(contentMatch[1]);
} catch (_) {
params.utm_content = contentMatch[1];
}
}
if (debug) {
console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
}
return params;
}
export async function main(
@@ -58,90 +220,246 @@ export async function main(
max_records = 9999999,
timeout_minutes = 60,
skip_clickhouse_check = false,
force_insert = false
force_insert = true,
database_override = "shorturl_analytics", // 添加数据库名称参数默认为shorturl_analytics
reset_sync_state = false, // 添加参数用于重置同步状态
debug_utm = false, // 添加参数控制UTM调试日志输出
start_time?: number, // 添加参数指定同步的开始时间戳,可选
end_time?: number, // 添加参数指定同步的结束时间戳,可选
use_custom_time_range = false, // 添加参数控制是否使用自定义时间范围
start_date?: string, // 添加开始日期字符串参数ISO格式或yyyy-MM-dd格式
end_date?: string // 添加结束日期字符串参数ISO格式或yyyy-MM-dd格式
) {
const logWithTimestamp = (message: string) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
logWithTimestamp("开始执行MongoDBClickHouse的同步任务");
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
// 处理日期字符串参数,转换为时间戳
if (start_date) {
try {
start_time = dateToTimestamp(start_date);
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${start_time}`);
use_custom_time_range = true;
} catch (err) {
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
}
}
if (end_date) {
try {
end_time = dateToTimestamp(end_date);
// 如果是日期格式,设置为当天结束时间 (23:59:59.999)
if (end_date.split('-').length === 3 && end_date.length <= 10) {
end_time += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${end_time}`);
} else {
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${end_time}`);
}
use_custom_time_range = true;
} catch (err) {
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
}
}
if (skip_clickhouse_check) {
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式不会检查记录是否已存在");
}
if (force_insert) {
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
}
if (reset_sync_state) {
logWithTimestamp("⚠️ 警告: 已启用重置同步状态,将从头开始同步数据");
}
if (debug_utm) {
logWithTimestamp("已启用UTM参数调试日志");
}
if (use_custom_time_range) {
if (start_time) {
logWithTimestamp(`已启用自定义时间范围:开始时间 ${new Date(start_time).toISOString()}`);
}
if (end_time) {
logWithTimestamp(`已启用自定义时间范围:结束时间 ${new Date(end_time).toISOString()}`);
}
}
// Set timeout
// 设置超时
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
// 检查是否超时
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
console.log(`运行时间超过${timeout_minutes}分钟,暂停执行`);
return true;
}
return false;
};
// Get MongoDB and ClickHouse connection info
// 获取MongoDBClickHouse的连接信息
let mongoConfig: MongoConfig;
let clickhouseConfig: ClickHouseConfig;
try {
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
console.log("原始MongoDB配置:", JSON.stringify(rawMongoConfig));
// 尝试解析配置,如果是字符串形式
if (typeof rawMongoConfig === "string") {
try {
mongoConfig = JSON.parse(rawMongoConfig);
} catch (e) {
console.error("MongoDB配置解析失败:", e);
throw e;
}
} else {
mongoConfig = rawMongoConfig as MongoConfig;
}
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
console.log("原始ClickHouse配置:", JSON.stringify(rawClickhouseConfig));
// 尝试解析配置,如果是字符串形式
if (typeof rawClickhouseConfig === "string") {
try {
clickhouseConfig = JSON.parse(rawClickhouseConfig);
} catch (e) {
console.error("ClickHouse配置解析失败:", e);
throw e;
}
} else {
clickhouseConfig = rawClickhouseConfig as ClickHouseConfig;
}
// 检查并修复数据库配置
if (!clickhouseConfig.clickhouse_database || clickhouseConfig.clickhouse_database === "undefined") {
logWithTimestamp(`⚠️ 警告: 数据库名称未定义或为'undefined',使用提供的默认值: ${database_override}`);
clickhouseConfig.clickhouse_database = database_override;
}
console.log("MongoDB配置解析为:", JSON.stringify(mongoConfig));
console.log("ClickHouse配置解析为:", JSON.stringify({
...clickhouseConfig,
clickhouse_password: "****" // 隐藏密码
}));
logWithTimestamp(`将使用ClickHouse数据库: ${clickhouseConfig.clickhouse_database}`);
} catch (error) {
console.error("Failed to get config:", error);
console.error("获取配置失败:", error);
throw error;
}
// Build MongoDB connection URL
// 获取上次同步状态
let lastSyncState: SyncState | null = null;
if (!reset_sync_state) {
try {
const rawSyncState = await getVariable(SYNC_STATE_KEY);
if (rawSyncState) {
if (typeof rawSyncState === "string") {
try {
lastSyncState = JSON.parse(rawSyncState);
} catch (e) {
logWithTimestamp(`解析上次同步状态失败: ${e}, 将从头开始同步`);
}
} else {
lastSyncState = rawSyncState as SyncState;
}
}
} catch (error) {
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
}
}
if (lastSyncState) {
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
if (lastSyncState.last_sync_id) {
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
}
} else {
logWithTimestamp("没有找到上次同步状态,将从头开始同步");
}
// 构建MongoDB连接URL
let mongoUrl = "mongodb://";
if (mongoConfig.username && mongoConfig.password) {
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
}
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
// Connect to MongoDB
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
// 连接MongoDB
const client = new MongoClient();
try {
await client.connect(mongoUrl);
console.log("MongoDB connected successfully");
console.log("MongoDB连接成功");
const db = client.database(mongoConfig.db);
const traceCollection = db.collection<TraceRecord>("trace");
// 添加对short集合的引用
const shortCollection = db.collection<ShortRecord>("short");
// Build query conditions
// 构建查询条件,根据上次同步状态获取新记录
const query: Record<string, unknown> = {
type: 1 // Only sync records with type 1
// 删除了 type: 1 的条件,将同步所有数据
};
// Count total records
const totalRecords = await traceCollection.countDocuments(query);
console.log(`Found ${totalRecords} records to sync`);
// 根据时间范围参数构建查询条件
if (use_custom_time_range) {
// 使用自定义时间范围
const timeQuery: Record<string, number> = {};
if (start_time) {
timeQuery.$gte = start_time;
logWithTimestamp(`将只同步createTime >= ${start_time} (${new Date(start_time).toISOString()}) 的记录`);
}
if (end_time) {
timeQuery.$lte = end_time;
logWithTimestamp(`将只同步createTime <= ${end_time} (${new Date(end_time).toISOString()}) 的记录`);
}
// 只有当至少指定了一个时间限制时才添加时间查询条件
if (Object.keys(timeQuery).length > 0) {
query.createTime = timeQuery;
}
}
// 如果不使用自定义时间范围,且有上次同步状态,则只获取更新的记录
else if (lastSyncState && lastSyncState.last_sync_time) {
// 使用上次同步时间作为过滤条件
query.createTime = { $gt: lastSyncState.last_sync_time };
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
}
// 计算总记录数
const totalRecords = await traceCollection.countDocuments(query);
console.log(`找到 ${totalRecords} 条新记录需要同步`);
// 限制此次处理的记录数量
const recordsToProcess = Math.min(totalRecords, max_records);
console.log(`Will process ${recordsToProcess} records`);
console.log(`本次将处理 ${recordsToProcess} 条记录`);
if (totalRecords === 0) {
console.log("No records to sync, task completed");
console.log("没有新记录需要同步,任务完成");
return {
success: true,
records_synced: 0,
message: "No records to sync"
message: "没有新记录需要同步"
};
}
// Check ClickHouse connection
// 检查ClickHouse连接状态
const checkClickHouseConnection = async (): Promise<boolean> => {
if (skip_clickhouse_check) {
logWithTimestamp("Skipping ClickHouse connection check");
logWithTimestamp("已启用跳过ClickHouse检查,不测试连接");
return true;
}
try {
logWithTimestamp("Testing ClickHouse connection...");
logWithTimestamp("测试ClickHouse连接...");
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
@@ -149,184 +467,197 @@ export async function main(
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
},
body: "SELECT 1",
body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
// 设置5秒超时
signal: AbortSignal.timeout(5000)
});
if (response.ok) {
logWithTimestamp("ClickHouse connection test successful");
logWithTimestamp("ClickHouse连接测试成功");
return true;
} else {
const errorText = await response.text();
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
return false;
}
} catch (err) {
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
const error = err as Error;
logWithTimestamp(`ClickHouse连接测试失败: ${error.message}`);
return false;
}
};
// Check if records exist in ClickHouse
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
if (records.length === 0) return [];
if (skip_clickhouse_check || force_insert) {
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
return records;
}
try {
const recordIds = records.map(record => record._id.toString());
const query = `
SELECT event_id
FROM shorturl_analytics.events
WHERE event_attributes LIKE '%"mongo_id":"%'
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
FORMAT JSON
`;
const response = await fetch(clickhouseConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: query,
signal: AbortSignal.timeout(10000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
}
const result = await response.json();
const existingIds = new Set(result.data.map((row: ClickHouseRow) => {
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
return matches ? matches[1] : null;
}).filter(Boolean));
return records.filter(record => !existingIds.has(record._id.toString()));
} catch (err) {
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
return skip_clickhouse_check ? records : [];
}
};
// 在处理记录前先检查ClickHouse连接
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
logWithTimestamp("⚠️ ClickHouse连接测试失败请启用skip_clickhouse_check=true参数来跳过连接检查");
throw new Error("ClickHouse连接失败,无法继续同步");
}
// Process records function
// 处理记录的函数
const processRecords = async (records: TraceRecord[]) => {
if (records.length === 0) return 0;
const newRecords = await checkExistingRecords(records);
if (newRecords.length === 0) {
logWithTimestamp("All records already exist, skipping");
return 0;
}
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条记录...`);
// Get link information for all records
// 强制使用所有记录,不检查重复
const newRecords = records;
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
// 获取链接信息 - 新增代码
const slugIds = newRecords.map(record => record.slugId);
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
const shortLinks = await shortCollection.find({
_id: { $in: slugIds }
}).toArray();
// Create a map for quick lookup
const shortLinksMap = new Map(shortLinks.map(link => [link._id.toString(), link]));
// Prepare ClickHouse insert data
// 创建映射用于快速查找 - 新增代码
const shortLinksMap = new Map(shortLinks.map((link: ShortRecord) => [link._id.toString(), link]));
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息,${newRecords.length - shortLinks.length} 条数据将使用占位符`);
// 准备ClickHouse插入数据
const clickhouseData = newRecords.map(record => {
const shortLink = shortLinksMap.get(record.slugId.toString());
const eventTime = new Date(record.createTime);
// 将毫秒时间戳转换为 DateTime64(3) 格式
const formatDateTime = (timestamp: number) => {
return new Date(timestamp).toISOString().replace('T', ' ').replace('Z', '');
// 获取对应的短链接信息 - 新增代码
const shortLink = shortLinksMap.get(record.slugId.toString()) as ShortRecord | undefined;
// 提取URL中的UTM参数 - 增加调试日志
if (debug_utm && record.url) {
logWithTimestamp(`======== UTM参数调试 ========`);
logWithTimestamp(`记录ID: ${record._id.toString()}`);
logWithTimestamp(`原始URL: ${record.url}`);
}
const utmParams = extractUtmParams(record.url || "", debug_utm);
if (debug_utm) {
logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
logWithTimestamp(`===========================`);
}
// 保存提取的UTM参数和URL到event_attributes
const eventAttributes = {
mongo_id: record._id.toString(),
url: record.url || "",
...(record.url ? { raw_url: record.url } : {})
};
// 转换MongoDB记录为ClickHouse格式匹配ClickHouse表结构
return {
// Event base information
event_id: record._id.toString(),
event_time: formatDateTime(record.createTime),
event_type: "click",
event_attributes: JSON.stringify({
original_type: record.type
}),
// Link information from short collection
// UUID将由ClickHouse自动生成 (event_id)
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
event_type: "click", // 将所有event_type都设置为click
event_attributes: JSON.stringify(eventAttributes),
link_id: record.slugId.toString(),
link_slug: shortLink?.slug || "",
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
link_label: record.label || "",
link_title: "",
link_original_url: shortLink?.origin || "",
link_attributes: JSON.stringify({
domain: shortLink?.domain || null
}),
link_created_at: shortLink?.createTime ? formatDateTime(shortLink.createTime) : formatDateTime(record.createTime),
link_expires_at: shortLink?.expiresAt ? formatDateTime(shortLink.expiresAt) : null,
link_tags: "[]", // Empty array as default
// User information
user_id: shortLink?.user || "",
user_name: "",
link_title: shortLink?.title || "unknown_title", // 使用占位符
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
link_created_at: shortLink?.createTime
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
link_expires_at: shortLink?.expiresAt
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
: null,
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
user_id: "3680f452-e404-4339-a3d2-2a8e1ff92102", // 使用占位符
user_name: "unknown_user", // 使用占位符
user_email: "",
user_attributes: "{}",
// Team information
team_id: shortLink?.teamId || "",
team_name: "",
team_id: "e02251eb-eb98-47c8-b5dd-4f6e4fdb1f49", // 使用占位符
team_name: "", // 使用占位符
team_attributes: "{}",
// Project information
project_id: shortLink?.projectId || "",
project_name: "",
project_id: "34cdb8b9-8b8e-4033-876a-0632002ef1f9", // 使用占位符
project_name: "", // 使用占位符
project_attributes: "{}",
// QR code information
qr_code_id: "",
qr_code_name: "",
qr_code_attributes: "{}",
// Visitor information
visitor_id: "", // Empty string as default
session_id: `${record.slugId.toString()}-${record.createTime}`,
ip_address: record.ip || "",
country: "",
visitor_id: record._id.toString(),
session_id: record._id.toString() + "-" + record.createTime,
ip_address: record.ip || "0.0.0.0", // 使用占位符
country: "",
city: "",
device_type: record.platform || "",
browser: record.browser || "",
os: record.platformOS || "",
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
// Source information
device_type: record.platform || "unknown",
browser: record.browser || "unknown", // 使用占位符
os: record.platformOS || "unknown", // 使用占位符
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
referrer: record.url || "",
utm_source: "",
utm_medium: "",
utm_campaign: "",
// Interaction information
utm_source: utmParams.utm_source || "",
utm_medium: utmParams.utm_medium || "",
utm_campaign: utmParams.utm_campaign || "",
utm_term: utmParams.utm_term || "",
utm_content: utmParams.utm_content || "",
time_spent_sec: 0,
is_bounce: true,
is_qr_scan: false,
conversion_type: "visit",
conversion_value: 0
conversion_value: 0,
req_full_path: record.url || ""
};
});
// Generate ClickHouse insert SQL
const rows = clickhouseData.map(row => {
// 只需要处理JSON字符串的转义
const formattedRow = {
...row,
event_attributes: row.event_attributes.replace(/\\/g, '\\\\'),
link_attributes: row.link_attributes.replace(/\\/g, '\\\\')
};
return JSON.stringify(formattedRow);
}).join('\n');
// 生成ClickHouse插入SQL
const insertSQL = `
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
(event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title,
link_original_url, link_attributes, link_created_at, link_expires_at, link_tags,
user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes,
project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes,
visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent,
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
VALUES ${clickhouseData.map(record => {
// 增强版安全替换函数,处理所有特殊字符
const safeReplace = (val: unknown): string => {
// 确保值是字符串如果是null或undefined则使用空字符串
const str = val === null || val === undefined ? "" : String(val);
// 转义所有可能导致SQL注入或格式错误的字符
// 1. 先替换所有反斜杠
// 2. 再替换单引号
// 3. 替换所有控制字符和特殊字符
return str
.replace(/\\/g, "\\\\") // 转义反斜杠
.replace(/'/g, "\\'") // 转义单引号
.replace(/\r/g, "\\r") // 转义回车
.replace(/\n/g, "\\n") // 转义换行
.replace(/\t/g, "\\t") // 转义制表符
.replace(/\0/g, "") // 移除空字符
.replace(/[\x00-\x1F\x7F-\x9F]/g, ""); // 移除所有控制字符
};
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
'${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}',
'${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}',
${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}',
'${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}',
'${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}',
'${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}',
'${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}',
'${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}',
'${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}',
'${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}',
'${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}',
'${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}',
'${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan},
'${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
}).join(", ")}
`;
const insertSQL = `INSERT INTO shorturl_analytics.events FORMAT JSONEachRow\n${rows}`;
if (insertSQL.length === 0) {
console.log("没有新记录需要插入");
return 0;
}
// 发送请求到ClickHouse添加20秒超时
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
try {
const response = await fetch(clickhouseConfig.clickhouse_url, {
logWithTimestamp("发送插入请求到ClickHouse...");
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
@@ -338,35 +669,34 @@ export async function main(
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
}
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
return newRecords.length;
} catch (err) {
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
throw err;
const error = err as Error;
logWithTimestamp(`向ClickHouse插入数据失败: ${error.message}`);
throw error;
}
};
// Check ClickHouse connection before processing
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
throw new Error("ClickHouse connection failed, cannot continue sync");
}
// Process records in batches
// 批量处理记录
let processedRecords = 0;
let totalBatchRecords = 0;
let lastSyncTime = 0;
for (let page = 0; processedRecords < recordsToProcess; page++) {
// 检查超时
if (checkTimeout()) {
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
break;
}
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
// 每批次都输出进度
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
const records = await traceCollection.find(
query,
{
@@ -378,32 +708,86 @@ export async function main(
).toArray();
if (records.length === 0) {
logWithTimestamp("No more records found, sync complete");
logWithTimestamp("没有找到更多数据,同步结束");
break;
}
// 找到数据,开始处理
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
// 输出当前批次的部分数据信息
if (records.length > 0) {
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
if (records.length > 1) {
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
}
// 如果开启了调试输出一些URL样本
if (debug_utm) {
const sampleSize = Math.min(5, records.length);
logWithTimestamp(`URL样本 (前${sampleSize}条):`);
for (let i = 0; i < sampleSize; i++) {
if (records[i].url) {
logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
}
}
}
}
const batchSize = await processRecords(records);
processedRecords += records.length;
totalBatchRecords += batchSize;
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
// 更新最后处理的记录时间和ID
if (records.length > 0) {
const lastRecord = records[records.length - 1];
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
}
logWithTimestamp(`${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
}
// 更新同步状态
if (processedRecords > 0 && lastSyncTime > 0) {
// 只在非自定义时间范围模式下更新同步状态
if (!use_custom_time_range) {
// 创建新的同步状态,简化对象结构
const newSyncState: SyncState = {
last_sync_time: lastSyncTime,
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords, // 使用处理的总记录数,而不是实际插入数
};
try {
// 保存同步状态
await setVariable(SYNC_STATE_KEY, newSyncState);
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
} catch (err) {
const error = err as Error;
logWithTimestamp(`更新同步状态失败: ${error.message},将继续执行`);
// 不抛出错误,继续执行
}
} else {
logWithTimestamp("使用自定义时间范围模式,不更新全局同步状态");
}
}
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
message: "Data sync completed"
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
message: use_custom_time_range ? "自定义时间范围数据同步完成" : "数据同步完成",
custom_time_range_used: use_custom_time_range
};
} catch (err) {
console.error("Error during sync:", err);
console.error("同步过程中发生错误:", err);
return {
success: false,
error: err instanceof Error ? err.message : String(err),
stack: err instanceof Error ? err.stack : undefined
};
} finally {
// 关闭MongoDB连接
await client.close();
console.log("MongoDB connection closed");
console.log("MongoDB连接已关闭");
}
}
}

View File

@@ -0,0 +1,527 @@
// 文件名: sync_shorturl_schema_to_clickhouse.ts
// 描述: 此脚本用于同步PostgreSQL中的short_url.shorturl表数据到ClickHouse
// 创建日期: 2023-11-21
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
// 同步状态接口
interface SyncState {
last_sync_time: string; // 上次同步的结束时间
records_synced: number; // 累计同步的记录数
last_run: string; // 上次运行的时间
}
// 同步状态键名
const SYNC_STATE_KEY = "f/shorturl_analytics/shorturl_to_clickhouse_state";
// PostgreSQL配置接口
interface PgConfig {
host: string;
port: number;
user: string;
password: string;
dbname?: string;
[key: string]: unknown;
}
// ClickHouse配置接口
interface ChConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_url?: string;
}
// Shorturl数据接口
interface ShortUrlData {
id: string;
slug: string;
origin: string; // 对应ClickHouse中的original_url
title?: string;
description?: string;
created_at?: string;
updated_at?: string;
deleted_at?: string;
expires_at?: string; // 注意这里已更正为expires_at
domain?: string; // 添加domain字段
}
/**
* 同步PostgreSQL short_url.shorturl表数据到ClickHouse
*/
export async function main(
/** 是否为测试模式(不执行实际更新) */
dry_run = false,
/** 是否显示详细日志 */
verbose = false,
/** 是否重置同步状态(从头开始同步) */
reset_sync_state = false,
/** 如果没有同步状态往前查询多少小时的数据默认1小时 */
default_hours_back = 1
) {
// 初始化日志函数
const log = (message: string, isVerbose = false) => {
if (!isVerbose || verbose) {
console.log(message);
}
};
// 获取同步状态
let syncState: SyncState | null = null;
if (!reset_sync_state) {
try {
log("获取同步状态...", true);
const rawState = await getVariable(SYNC_STATE_KEY);
if (rawState) {
if (typeof rawState === "string") {
syncState = JSON.parse(rawState);
} else {
syncState = rawState as SyncState;
}
log(`找到上次同步状态: 最后同步时间 ${syncState.last_sync_time}, 已同步记录数 ${syncState.records_synced}`, true);
}
} catch (error) {
log(`获取同步状态失败: ${error}, 将使用默认设置`, true);
}
} else {
log("重置同步状态,从头开始同步", true);
}
// 设置时间范围
const oneHourAgo = new Date(Date.now() - default_hours_back * 60 * 60 * 1000).toISOString();
// 如果有同步状态,使用上次同步时间作为开始时间;否则使用默认时间
const start_time = syncState ? syncState.last_sync_time : oneHourAgo;
const end_time = new Date().toISOString();
log(`开始同步shorturl表数据: ${start_time}${end_time}`);
let pgPool: Pool | null = null;
try {
// 1. 获取数据库配置
log("获取PostgreSQL数据库配置...", true);
const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
// 2. 创建PostgreSQL连接池
pgPool = new Pool({
hostname: pgConfig.host,
port: pgConfig.port,
user: pgConfig.user,
password: pgConfig.password,
database: pgConfig.dbname || 'postgres'
}, 3);
// 3. 获取需要更新的数据
const shorturlData = await getShortUrlData(pgPool, start_time, end_time, log);
log(`成功获取 ${shorturlData.length} 条shorturl数据`);
if (shorturlData.length === 0) {
// 更新同步状态,即使没有新数据
if (!dry_run) {
await updateSyncState(end_time, syncState ? syncState.records_synced : 0, log);
}
return { success: true, message: "没有找到需要更新的数据", updated: 0 };
}
// 4. 获取ClickHouse配置
const chConfig = await getClickHouseConfig();
// 5. 执行更新
if (!dry_run) {
const shorturlUpdated = await updateClickHouseShortUrl(shorturlData, chConfig, log);
// 更新同步状态
const totalSynced = (syncState ? syncState.records_synced : 0) + shorturlUpdated;
await updateSyncState(end_time, totalSynced, log);
return {
success: true,
message: "shorturl表数据同步完成",
shorturl_updated: shorturlUpdated,
total_synced: totalSynced,
sync_state: {
last_sync_time: end_time,
records_synced: totalSynced
}
};
} else {
log("测试模式: 不执行实际更新");
return {
success: true,
dry_run: true,
shorturl_count: shorturlData.length,
shorturl_sample: shorturlData.slice(0, 1)
};
}
} catch (error) {
const errorMessage = `同步过程中发生错误: ${(error as Error).message}`;
log(errorMessage);
if ((error as Error).stack) {
log(`错误堆栈: ${(error as Error).stack}`, true);
}
return { success: false, message: errorMessage };
} finally {
if (pgPool) {
await pgPool.end();
log("PostgreSQL连接池已关闭", true);
}
}
}
/**
* 更新同步状态
*/
async function updateSyncState(lastSyncTime: string, recordsSynced: number, log: (message: string, isVerbose?: boolean) => void): Promise<void> {
try {
const newState: SyncState = {
last_sync_time: lastSyncTime,
records_synced: recordsSynced,
last_run: new Date().toISOString()
};
await setVariable(SYNC_STATE_KEY, newState);
log(`同步状态已更新: 最后同步时间 ${lastSyncTime}, 累计同步记录数 ${recordsSynced}`, true);
} catch (error) {
log(`更新同步状态失败: ${error}`, true);
// 继续执行,不中断同步过程
}
}
/**
* 从PostgreSQL获取shorturl数据
*/
async function getShortUrlData(
pgPool: Pool,
startTime: string,
endTime: string,
log: (message: string, isVerbose?: boolean) => void
): Promise<ShortUrlData[]> {
const client = await pgPool.connect();
try {
log(`获取shorturl数据 (${startTime}${endTime})`, true);
const query = `
SELECT
id,
slug,
origin,
title,
description,
domain,
created_at,
updated_at,
deleted_at,
expired_at as expires_at
FROM
short_url.shorturl
WHERE
(created_at >= $1 AND created_at <= $2)
OR (updated_at >= $1 AND updated_at <= $2)
`;
const result = await client.queryObject(query, [startTime, endTime]);
return result.rows as ShortUrlData[];
} finally {
client.release();
}
}
/**
* 格式化日期时间为ClickHouse可接受的格式
*/
function formatDateTime(dateStr: string | null | undefined): string {
if (!dateStr) return 'NULL';
try {
// 将日期字符串转换为ISO格式
const date = new Date(dateStr);
if (isNaN(date.getTime())) {
return 'NULL';
}
// 返回ISO格式的日期字符串ClickHouse可以解析
return `parseDateTimeBestEffort('${date.toISOString()}')`;
} catch (error) {
console.error(`日期格式化错误: ${error}`);
return 'NULL';
}
}
/**
* 格式化进度显示
*/
function formatProgress(current: number, total: number): string {
const percent = Math.round((current / total) * 100);
const progressBar = '[' + '='.repeat(Math.floor(percent / 5)) + ' '.repeat(20 - Math.floor(percent / 5)) + ']';
return `${progressBar} ${percent}% (${current}/${total})`;
}
/**
* 更新ClickHouse中的shorturl表数据
*/
async function updateClickHouseShortUrl(
shorturls: ShortUrlData[],
chConfig: ChConfig,
log: (message: string, isVerbose?: boolean) => void
): Promise<number> {
if (shorturls.length === 0) {
log('没有找到shorturl数据跳过shorturl表更新');
return 0;
}
log(`准备更新 ${shorturls.length} 条shorturl数据`);
// 检查ClickHouse中是否存在shorturl表
const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.shorturl');
if (!tableExists) {
log('ClickHouse中未找到shorturl表请先创建表');
return 0;
}
let updatedCount = 0;
const startTime = Date.now();
// 使用批量插入更高效
const batchSize = 50; // 降低批次大小,使查询更稳定
for (let i = 0; i < shorturls.length; i += batchSize) {
const batch = shorturls.slice(i, i + batchSize);
let successCount = 0;
// 显示批处理进度信息
const batchNumber = Math.floor(i / batchSize) + 1;
const totalBatches = Math.ceil(shorturls.length / batchSize);
log(`处理批次 ${batchNumber}/${totalBatches}: ${formatProgress(i, shorturls.length)}`);
// 对每条记录使用单独的INSERT ... SELECT ... WHERE NOT EXISTS语句
for (let j = 0; j < batch.length; j++) {
const shorturl = batch[j];
// 显示记录处理细节进度
const overallProgress = i + j + 1;
if (overallProgress % 10 === 0 || overallProgress === shorturls.length) {
// 每10条记录或最后一条记录显示一次进度
const elapsedSeconds = (Date.now() - startTime) / 1000;
const recordsPerSecond = overallProgress / elapsedSeconds;
const remainingRecords = shorturls.length - overallProgress;
const estimatedSecondsRemaining = remainingRecords / recordsPerSecond;
log(`总进度: ${formatProgress(overallProgress, shorturls.length)} - 速率: ${recordsPerSecond.toFixed(1)}条/秒 - 预计剩余时间: ${formatTime(estimatedSecondsRemaining)}`);
}
try {
const insertQuery = `
INSERT INTO shorturl_analytics.shorturl
SELECT
'${escapeString(shorturl.id)}' AS id,
'${escapeString(shorturl.id)}' AS external_id,
'shorturl' AS type,
'${escapeString(shorturl.slug)}' AS slug,
'${escapeString(shorturl.origin)}' AS original_url,
${shorturl.title ? `'${escapeString(shorturl.title)}'` : 'NULL'} AS title,
${shorturl.description ? `'${escapeString(shorturl.description)}'` : 'NULL'} AS description,
'{}' AS attributes,
1 AS schema_version,
'' AS creator_id,
'' AS creator_email,
'' AS creator_name,
${formatDateTime(shorturl.created_at)} AS created_at,
${formatDateTime(shorturl.updated_at)} AS updated_at,
${formatDateTime(shorturl.deleted_at)} AS deleted_at,
'[]' AS projects,
'[]' AS teams,
'[]' AS tags,
'[]' AS qr_codes,
'[]' AS channels,
'[]' AS favorites,
${formatDateTime(shorturl.expires_at)} AS expires_at,
0 AS click_count,
0 AS unique_visitors,
${shorturl.domain ? `'${escapeString(shorturl.domain)}'` : 'NULL'} AS domain
WHERE NOT EXISTS (
SELECT 1 FROM shorturl_analytics.shorturl WHERE id = '${escapeString(shorturl.id)}'
)
`;
await executeClickHouseQuery(chConfig, insertQuery);
successCount++;
log(`成功处理shorturl: ${shorturl.id}`, true);
} catch (error) {
log(`处理shorturl ${shorturl.id} 失败: ${(error as Error).message}`);
// 尝试使用简单插入作为备选方案
try {
log(`尝试替代方法更新: ${shorturl.id}`, true);
// 先检查记录是否存在
const checkQuery = `SELECT count() FROM shorturl_analytics.shorturl WHERE id = '${escapeString(shorturl.id)}'`;
const existsResult = await executeClickHouseQuery(chConfig, checkQuery);
const exists = parseInt(existsResult.trim()) > 0;
if (!exists) {
const fallbackQuery = `
INSERT INTO shorturl_analytics.shorturl (
id, external_id, type, slug, original_url,
title, description, attributes, schema_version,
creator_id, creator_email, creator_name,
created_at, updated_at, deleted_at,
projects, teams, tags, qr_codes, channels, favorites,
expires_at, click_count, unique_visitors, domain
) VALUES (
'${escapeString(shorturl.id)}',
'${escapeString(shorturl.id)}',
'shorturl',
'${escapeString(shorturl.slug)}',
'${escapeString(shorturl.origin)}',
${shorturl.title ? `'${escapeString(shorturl.title)}'` : 'NULL'},
${shorturl.description ? `'${escapeString(shorturl.description)}'` : 'NULL'},
'{}',
1,
'',
'',
'',
${formatDateTime(shorturl.created_at)},
${formatDateTime(shorturl.updated_at)},
${formatDateTime(shorturl.deleted_at)},
'[]',
'[]',
'[]',
'[]',
'[]',
'[]',
${formatDateTime(shorturl.expires_at)},
0,
0,
${shorturl.domain ? `'${escapeString(shorturl.domain)}'` : 'NULL'}
)
`;
await executeClickHouseQuery(chConfig, fallbackQuery);
successCount++;
log(`备选方式插入成功: ${shorturl.id}`, true);
} else {
log(`记录已存在,跳过: ${shorturl.id}`, true);
}
} catch (fallbackError) {
log(`备选方式失败 ${shorturl.id}: ${(fallbackError as Error).message}`);
}
}
}
updatedCount += successCount;
log(`批次 ${batchNumber}/${totalBatches} 完成: ${successCount}/${batch.length} 条成功 (总计: ${updatedCount}/${shorturls.length})`);
}
const totalTime = (Date.now() - startTime) / 1000;
log(`同步完成! 总计处理: ${updatedCount}/${shorturls.length} 条记录, 耗时: ${formatTime(totalTime)}, 平均速率: ${(updatedCount / totalTime).toFixed(1)}条/秒`);
return updatedCount;
}
/**
* 获取ClickHouse配置
*/
async function getClickHouseConfig(): Promise<ChConfig> {
try {
const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
// 确保配置不为空
if (!chConfigJson) {
throw new Error("未找到ClickHouse配置");
}
// 解析JSON字符串为对象
let chConfig: ChConfig;
if (typeof chConfigJson === 'string') {
try {
chConfig = JSON.parse(chConfigJson);
} catch {
throw new Error("ClickHouse配置不是有效的JSON");
}
} else {
chConfig = chConfigJson as ChConfig;
}
// 验证并构建URL
if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
}
if (!chConfig.clickhouse_url) {
throw new Error("ClickHouse配置缺少URL");
}
return chConfig;
} catch (error) {
throw new Error(`获取ClickHouse配置失败: ${(error as Error).message}`);
}
}
/**
* 检查ClickHouse中是否存在指定表
*/
async function checkClickHouseTable(chConfig: ChConfig, tableName: string): Promise<boolean> {
try {
const query = `EXISTS TABLE ${tableName}`;
const result = await executeClickHouseQuery(chConfig, query);
return result.trim() === '1';
} catch (error) {
console.error(`检查表 ${tableName} 失败:`, error);
return false;
}
}
/**
* 执行ClickHouse查询
*/
async function executeClickHouseQuery(chConfig: ChConfig, query: string): Promise<string> {
// 确保URL有效
if (!chConfig.clickhouse_url) {
throw new Error("无效的ClickHouse URL: 未定义");
}
// 执行HTTP请求
try {
const response = await fetch(chConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
},
body: query,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse查询失败 (${response.status}): ${errorText}`);
}
return await response.text();
} catch (error) {
throw new Error(`执行ClickHouse查询失败: ${(error as Error).message}`);
}
}
/**
* 转义字符串避免SQL注入
*/
function escapeString(str: string): string {
if (!str) return '';
return str.replace(/'/g, "''");
}
/**
* 格式化时间(秒)为可读格式
*/
function formatTime(seconds: number): string {
const mins = Math.floor(seconds / 60);
const secs = Math.floor(seconds % 60);
if (mins === 0) {
return `${secs}`;
} else {
return `${mins}${secs}`;
}
}

View File

@@ -0,0 +1,709 @@
// Windmill script to sync shorturl data from PostgreSQL to ClickHouse
// 作者: AI Assistant
// 创建日期: 2023-10-30
// 描述: 此脚本从PostgreSQL数据库获取所有shorturl类型的资源及其关联数据并同步到ClickHouse
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
// 资源属性接口
interface ResourceAttributes {
slug?: string;
original_url?: string;
originalUrl?: string;
title?: string;
description?: string;
expires_at?: string;
expiresAt?: string;
[key: string]: unknown;
}
// ClickHouse配置接口
interface ChConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_url: string;
}
// PostgreSQL配置接口
interface PgConfig {
host: string;
port: number;
user: string;
password: string;
dbname?: string;
[key: string]: unknown;
}
// 上次同步状态接口
interface SyncState {
lastSyncTime: string;
lastRunTime: string;
}
// 状态变量名称
const STATE_VARIABLE_PATH = "f/shorturl_analytics/shorturl_sync_state";
// Windmill函数定义
export async function main(
/** PostgreSQL和ClickHouse同步脚本 */
params: {
/** 同步的资源数量限制默认500 */
limit?: number;
/** 是否包含已删除资源 */
includeDeleted?: boolean;
/** 是否执行实际写入操作 */
dryRun?: boolean;
/** 是否强制全量同步 */
forceFullSync?: boolean;
/** 手动指定开始时间ISO格式- 会覆盖自动增量设置 */
startTime?: string;
/** 手动指定结束时间ISO格式*/
endTime?: string;
}
) {
// 设置默认参数
const limit = params.limit || 500;
const includeDeleted = params.includeDeleted || false;
const dryRun = params.dryRun || false;
const forceFullSync = params.forceFullSync || false;
// 获取当前时间作为本次运行时间
const currentRunTime = new Date().toISOString();
// 初始化同步状态
let syncState: SyncState;
let startTime: Date | undefined;
const endTime: Date | undefined = params.endTime ? new Date(params.endTime) : new Date();
// 如果强制全量同步或手动指定了开始时间,则使用指定的开始时间
if (forceFullSync || params.startTime) {
startTime = params.startTime ? new Date(params.startTime) : undefined;
console.log(`使用${params.startTime ? '手动指定' : '全量同步'} - 开始时间: ${startTime ? startTime.toISOString() : '无限制'}`);
}
// 否则尝试获取上次同步时间作为增量同步的开始时间点
else {
try {
// 获取上次同步状态
const stateStr = await getVariable(STATE_VARIABLE_PATH);
if (stateStr) {
syncState = JSON.parse(stateStr);
console.log(`获取到上次同步状态: 同步时间=${syncState.lastSyncTime}, 运行时间=${syncState.lastRunTime}`);
// 使用上次运行时间作为本次的开始时间 (减去1分钟防止边界问题)
const lastRunTime = new Date(syncState.lastRunTime);
lastRunTime.setMinutes(lastRunTime.getMinutes() - 1);
startTime = lastRunTime;
} else {
console.log("未找到上次同步状态,将执行全量同步");
}
} catch (error: unknown) {
console.log(`获取同步状态出错: ${error instanceof Error ? error.message : String(error)},将执行全量同步`);
}
}
console.log(`开始同步PostgreSQL shorturl数据到ClickHouse`);
console.log(`参数: limit=${limit}, includeDeleted=${includeDeleted}, dryRun=${dryRun}`);
if (startTime) console.log(`开始时间: ${startTime.toISOString()}`);
if (endTime) console.log(`结束时间: ${endTime.toISOString()}`);
// 获取数据库配置
console.log("获取PostgreSQL数据库配置...");
const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
console.log(`数据库连接配置: host=${pgConfig.host}, port=${pgConfig.port}, database=${pgConfig.dbname || 'postgres'}, user=${pgConfig.user}`);
let pgPool: Pool | null = null;
try {
console.log("创建PostgreSQL连接池...");
pgPool = new Pool({
hostname: pgConfig.host,
port: pgConfig.port,
user: pgConfig.user,
password: pgConfig.password,
database: pgConfig.dbname || 'postgres'
}, 3);
console.log("PostgreSQL连接池创建完成尝试连接...");
// 测试连接
const client = await pgPool.connect();
try {
console.log("连接成功,执行测试查询...");
const testResult = await client.queryObject(`SELECT 1 AS test`);
console.log(`测试查询结果: ${JSON.stringify(testResult.rows)}`);
} finally {
client.release();
}
// 获取所有shorturl类型的资源
const shorturls = await fetchShorturlResources(pgPool, {
limit,
includeDeleted,
startTime,
endTime,
});
console.log(`获取到 ${shorturls.length} 个shorturl资源`);
if (shorturls.length === 0) {
// 即使没有数据也更新状态
await updateSyncState(currentRunTime);
return { synced: 0, message: "没有找到需要同步的shorturl资源" };
}
// 为每个资源获取关联数据
const enrichedShorturls = await enrichShorturlData(pgPool, shorturls);
console.log(`已丰富 ${enrichedShorturls.length} 个shorturl资源的关联数据`);
// 转换为ClickHouse格式
const clickhouseData = formatForClickhouse(enrichedShorturls);
if (!dryRun) {
// 写入ClickHouse
const inserted = await insertToClickhouse(clickhouseData);
console.log(`成功写入 ${inserted} 条记录到ClickHouse`);
// 更新同步状态
await updateSyncState(currentRunTime);
return { synced: inserted, message: "同步完成", lastSyncTime: currentRunTime };
} else {
console.log("Dry run模式 - 不执行实际写入");
console.log(`将写入 ${clickhouseData.length} 条记录到ClickHouse`);
// 输出示例数据
if (clickhouseData.length > 0) {
console.log("示例数据:");
console.log(JSON.stringify(clickhouseData[0], null, 2));
}
return { synced: 0, dryRun: true, sampleData: clickhouseData.slice(0, 1) };
}
} catch (error: unknown) {
console.error(`同步过程中发生错误: ${(error as Error).message}`);
console.error(`错误类型: ${(error as Error).name}`);
if ((error as Error).stack) {
console.error(`错误堆栈: ${(error as Error).stack}`);
}
throw error;
} finally {
if (pgPool) {
await pgPool.end();
console.log("PostgreSQL连接池已关闭");
}
}
}
// 更新同步状态
async function updateSyncState(currentRunTime: string): Promise<void> {
try {
const syncState: SyncState = {
lastSyncTime: new Date().toISOString(), // 记录数据同步完成的时间
lastRunTime: currentRunTime // 记录本次运行的时间点
};
console.log(`更新同步状态: ${JSON.stringify(syncState)}`);
await setVariable(STATE_VARIABLE_PATH, JSON.stringify(syncState));
} catch (error: unknown) {
console.error(`更新同步状态失败: ${error instanceof Error ? error.message : String(error)}`);
// 不中断主流程,即使状态更新失败
}
}
// 从PostgreSQL获取所有shorturl资源
async function fetchShorturlResources(
pgPool: Pool,
options: {
limit: number;
includeDeleted: boolean;
startTime?: Date;
endTime?: Date;
}
) {
let query = `
SELECT
r.id,
r.external_id,
r.type,
r.attributes,
r.schema_version,
r.creator_id,
r.created_at,
r.updated_at,
r.deleted_at,
u.email as creator_email,
u.first_name as creator_first_name,
u.last_name as creator_last_name
FROM
limq.resources r
LEFT JOIN
limq.users u ON r.creator_id = u.id
WHERE
r.type = 'shorturl'
`;
const params = [];
let paramCount = 1;
if (!options.includeDeleted) {
query += ` AND r.deleted_at IS NULL`;
}
// 修改为同时考虑created_at和updated_at确保捕获自上次同步以来创建或更新的记录
if (options.startTime) {
query += ` AND (r.created_at >= $${paramCount} OR r.updated_at >= $${paramCount})`;
params.push(options.startTime);
paramCount++;
}
if (options.endTime) {
query += ` AND r.created_at <= $${paramCount}`;
params.push(options.endTime);
paramCount++;
}
// 优先按更新时间排序,确保最近更新的记录先处理
query += ` ORDER BY r.updated_at DESC, r.created_at DESC LIMIT $${paramCount}`;
params.push(options.limit);
const client = await pgPool.connect();
try {
const result = await client.queryObject(query, params);
// 添加调试日志 - 显示获取的数据样本
if (result.rows.length > 0) {
console.log(`获取到 ${result.rows.length} 条shorturl记录`);
console.log(`第一条记录ID: ${result.rows[0].id}`);
console.log(`attributes类型: ${typeof result.rows[0].attributes}`);
console.log(`attributes内容示例: ${JSON.stringify(String(result.rows[0].attributes)).substring(0, 100)}...`);
}
return result.rows;
} finally {
client.release();
}
}
// 为每个shorturl资源获取关联数据
async function enrichShorturlData(pgPool: Pool, shorturls: Record<string, unknown>[]) {
const client = await pgPool.connect();
const enriched = [];
try {
for (const shorturl of shorturls) {
// 1. 获取项目关联
const projectsResult = await client.queryObject(`
SELECT
pr.resource_id, pr.project_id,
p.name as project_name, p.description as project_description,
pr.assigned_at
FROM
limq.project_resources pr
JOIN
limq.projects p ON pr.project_id = p.id
WHERE
pr.resource_id = $1
`, [shorturl.id]);
// 2. 获取团队关联(通过项目)
const teamIds = projectsResult.rows.map((p: Record<string, unknown>) => p.project_id);
const teamsResult = teamIds.length > 0 ? await client.queryObject(`
SELECT
tp.team_id, tp.project_id,
t.name as team_name, t.description as team_description
FROM
limq.team_projects tp
JOIN
limq.teams t ON tp.team_id = t.id
WHERE
tp.project_id = ANY($1::uuid[])
`, [teamIds]) : { rows: [] };
// 3. 获取标签关联
const tagsResult = await client.queryObject(`
SELECT
rt.resource_id, rt.tag_id, rt.created_at,
t.name as tag_name, t.type as tag_type
FROM
limq.resource_tags rt
JOIN
limq.tags t ON rt.tag_id = t.id
WHERE
rt.resource_id = $1
`, [shorturl.id]);
// 4. 获取QR码关联
const qrCodesResult = await client.queryObject(`
SELECT
id as qr_id, scan_count, url, template_name, created_at
FROM
limq.qr_code
WHERE
resource_id = $1
`, [shorturl.id]);
// 5. 获取渠道关联
const channelsResult = await client.queryObject(`
SELECT
id as channel_id, name as channel_name, path as channel_path,
"isUserCreated" as is_user_created
FROM
limq.channel
WHERE
"shortUrlId" = $1
`, [shorturl.id]);
// 6. 获取收藏关联
const favoritesResult = await client.queryObject(`
SELECT
f.id as favorite_id, f.user_id, f.created_at,
u.first_name, u.last_name
FROM
limq.favorite f
JOIN
limq.users u ON f.user_id = u.id
WHERE
f.favoritable_id = $1 AND f.favoritable_type = 'resource'
`, [shorturl.id]);
// 调试日志
console.log(`\n处理资源ID: ${shorturl.id}`);
console.log(`attributes类型: ${typeof shorturl.attributes}`);
// 改进的attributes解析逻辑
let attributes: ResourceAttributes = {};
try {
if (typeof shorturl.attributes === 'string') {
// 如果是字符串尝试解析为JSON
console.log(`尝试解析attributes字符串长度: ${shorturl.attributes.length}`);
attributes = JSON.parse(shorturl.attributes);
} else if (typeof shorturl.attributes === 'object' && shorturl.attributes !== null) {
// 如果已经是对象,直接使用
console.log('attributes已经是对象类型');
attributes = shorturl.attributes as ResourceAttributes;
} else {
console.log(`无效的attributes类型: ${typeof shorturl.attributes}`);
attributes = {};
}
} catch (err) {
const error = err as Error;
console.warn(`无法解析资源 ${shorturl.id} 的attributes JSON:`, error.message);
// 尝试进行更多原始数据分析
if (typeof shorturl.attributes === 'string') {
console.log(`原始字符串前100字符: ${shorturl.attributes.substring(0, 100)}`);
}
attributes = {};
}
// 尝试从QR码获取数据
let slugFromQr = "";
const urlFromQr = "";
if (qrCodesResult.rows.length > 0 && qrCodesResult.rows[0].url) {
const qrUrl = qrCodesResult.rows[0].url as string;
console.log(`找到QR码URL: ${qrUrl}`);
try {
const urlParts = qrUrl.split('/');
slugFromQr = urlParts[urlParts.length - 1];
console.log(`从QR码URL提取的slug: ${slugFromQr}`);
} catch (err) {
const error = err as Error;
console.log('无法从QR码URL提取slug:', error.message);
}
}
// 日志输出实际字段值
console.log(`提取字段 - name: ${attributes.name || 'N/A'}, slug: ${attributes.slug || 'N/A'}`);
console.log(`提取字段 - originalUrl: ${attributes.originalUrl || 'N/A'}, original_url: ${attributes.original_url || 'N/A'}`);
// 整合所有数据
const slug = attributes.slug || attributes.name || slugFromQr || "";
const originalUrl = attributes.originalUrl || attributes.original_url || urlFromQr || "";
console.log(`最终使用的slug: ${slug}`);
console.log(`最终使用的originalUrl: ${originalUrl}`);
enriched.push({
...shorturl,
attributes,
projects: projectsResult.rows,
teams: teamsResult.rows,
tags: tagsResult.rows,
qrCodes: qrCodesResult.rows,
channels: channelsResult.rows,
favorites: favoritesResult.rows,
// 从attributes中提取特定字段 - 使用改进的顺序和QR码备选
slug,
originalUrl,
title: attributes.title || "",
description: attributes.description || "",
expiresAt: attributes.expires_at || attributes.expiresAt || null
});
}
} finally {
client.release();
}
return enriched;
}
// 将PostgreSQL数据格式化为ClickHouse格式
function formatForClickhouse(shorturls: Record<string, unknown>[]) {
// 将日期格式化为ClickHouse兼容的DateTime64(3)格式
const formatDateTime = (date: Date | string | number | null | undefined): string | null => {
if (!date) return null;
// 转换为Date对象
const dateObj = date instanceof Date ? date : new Date(date);
// 返回格式化的字符串: YYYY-MM-DD HH:MM:SS.SSS
return dateObj.toISOString().replace('T', ' ').replace('Z', '');
};
console.log(`\n准备格式化 ${shorturls.length} 条记录为ClickHouse格式`);
return shorturls.map(shorturl => {
// 调试日志:输出关键字段
console.log(`处理资源: ${shorturl.id}`);
console.log(`slug: ${shorturl.slug || 'EMPTY'}`);
console.log(`originalUrl: ${shorturl.originalUrl || 'EMPTY'}`);
// 记录attributes状态
const attributesStr = JSON.stringify(shorturl.attributes || {});
const attributesPrev = attributesStr.length > 100 ?
attributesStr.substring(0, 100) + '...' :
attributesStr;
console.log(`attributes: ${attributesPrev}`);
const creatorName = [shorturl.creator_first_name, shorturl.creator_last_name]
.filter(Boolean)
.join(" ");
// 格式化项目数据为JSON数组
const projects = JSON.stringify((shorturl.projects as Record<string, unknown>[]).map((p) => ({
project_id: p.project_id,
project_name: p.project_name,
project_description: p.project_description,
assigned_at: p.assigned_at
})));
// 格式化团队数据为JSON数组
const teams = JSON.stringify((shorturl.teams as Record<string, unknown>[]).map((t) => ({
team_id: t.team_id,
team_name: t.team_name,
team_description: t.team_description,
via_project_id: t.project_id
})));
// 格式化标签数据为JSON数组
const tags = JSON.stringify((shorturl.tags as Record<string, unknown>[]).map((t) => ({
tag_id: t.tag_id,
tag_name: t.tag_name,
tag_type: t.tag_type,
created_at: t.created_at
})));
// 格式化QR码数据为JSON数组
const qrCodes = JSON.stringify((shorturl.qrCodes as Record<string, unknown>[]).map((q) => ({
qr_id: q.qr_id,
scan_count: q.scan_count,
url: q.url,
template_name: q.template_name,
created_at: q.created_at
})));
// 格式化渠道数据为JSON数组
const channels = JSON.stringify((shorturl.channels as Record<string, unknown>[]).map((c) => ({
channel_id: c.channel_id,
channel_name: c.channel_name,
channel_path: c.channel_path,
is_user_created: c.is_user_created
})));
// 格式化收藏数据为JSON数组
const favorites = JSON.stringify((shorturl.favorites as Record<string, unknown>[]).map((f) => ({
favorite_id: f.favorite_id,
user_id: f.user_id,
user_name: `${f.first_name || ""} ${f.last_name || ""}`.trim(),
created_at: f.created_at
})));
// 统计信息可通过events表聚合或在其他地方设置
const clickCount = (shorturl.attributes as ResourceAttributes).click_count as number || 0;
const uniqueVisitors = 0;
// 返回ClickHouse格式数据
return {
id: shorturl.id,
external_id: shorturl.external_id || "",
type: shorturl.type,
slug: shorturl.slug || "",
original_url: shorturl.originalUrl || "",
title: shorturl.title || "",
description: shorturl.description || "",
attributes: JSON.stringify(shorturl.attributes || {}),
schema_version: shorturl.schema_version || 1,
creator_id: shorturl.creator_id || "",
creator_email: shorturl.creator_email || "",
creator_name: creatorName,
created_at: formatDateTime(shorturl.created_at as Date),
updated_at: formatDateTime(shorturl.updated_at as Date),
deleted_at: formatDateTime(shorturl.deleted_at as Date | null),
projects,
teams,
tags,
qr_codes: qrCodes,
channels,
favorites,
expires_at: formatDateTime(shorturl.expiresAt as Date | null),
click_count: clickCount,
unique_visitors: uniqueVisitors
};
});
}
// 获取ClickHouse配置
async function getClickHouseConfig(): Promise<ChConfig> {
try {
// 使用getVariable而不是getResource获取ClickHouse配置
const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
console.log("原始ClickHouse配置:", typeof chConfigJson);
// 确保配置不为空
if (!chConfigJson) {
throw new Error("未找到ClickHouse配置");
}
// 解析JSON字符串为对象
let chConfig: ChConfig;
if (typeof chConfigJson === 'string') {
try {
chConfig = JSON.parse(chConfigJson);
} catch (parseError) {
console.error("解析JSON失败:", parseError);
throw new Error("ClickHouse配置不是有效的JSON");
}
} else {
chConfig = chConfigJson as ChConfig;
}
// 验证配置
if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
console.log(`已构建ClickHouse URL: ${chConfig.clickhouse_url}`);
}
if (!chConfig.clickhouse_url) {
throw new Error("ClickHouse配置缺少URL");
}
return chConfig;
} catch (error) {
console.error("获取ClickHouse配置失败:", error);
throw error;
}
}
// 写入数据到ClickHouse
async function insertToClickhouse(data: Record<string, unknown>[]) {
if (data.length === 0) return 0;
// 获取ClickHouse连接信息
const chConfig = await getClickHouseConfig();
// 确保URL有效
if (!chConfig.clickhouse_url) {
throw new Error("无效的ClickHouse URL: 未定义");
}
console.log(`准备写入数据到ClickHouse: ${chConfig.clickhouse_url}`);
// 构建INSERT查询
const columns = Object.keys(data[0]).join(", ");
// 收集所有记录的ID
const recordIds = data.map(record => record.id as string);
console.log(`需要处理的记录数: ${recordIds.length}`);
// 先删除可能存在的重复记录
try {
console.log(`删除可能存在的重复记录...`);
// 按批次处理删除,避免请求过大
const deleteBatchSize = 100;
for (let i = 0; i < recordIds.length; i += deleteBatchSize) {
const idBatch = recordIds.slice(i, i + deleteBatchSize);
const formattedIds = idBatch.map(id => `'${id}'`).join(', ');
const deleteQuery = `
ALTER TABLE shorturl_analytics.shorturl
DELETE WHERE id IN (${formattedIds})
`;
const response = await fetch(chConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
},
body: deleteQuery,
});
if (!response.ok) {
const errorText = await response.text();
console.warn(`删除记录时出错 (批次 ${i/deleteBatchSize + 1}): ${errorText}`);
// 继续执行,不中断流程
} else {
console.log(`成功删除批次 ${i/deleteBatchSize + 1}/${Math.ceil(recordIds.length/deleteBatchSize)}的潜在重复记录`);
}
}
} catch (error) {
console.warn(`删除重复记录时出错: ${(error as Error).message}`);
// 继续执行,不因为删除失败而中断整个过程
}
const query = `
INSERT INTO shorturl_analytics.shorturl (${columns})
FORMAT JSONEachRow
`;
// 批量插入
let inserted = 0;
const batchSize = 100;
for (let i = 0; i < data.length; i += batchSize) {
const batch = data.slice(i, i + batchSize);
// 使用JSONEachRow格式
const rows = batch.map(row => JSON.stringify(row)).join('\n');
// 使用HTTP接口执行查询
try {
console.log(`正在发送请求到: ${chConfig.clickhouse_url}`);
console.log(`认证信息: ${chConfig.clickhouse_user}:***`);
const response = await fetch(chConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
},
body: `${query}\n${rows}`,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse插入失败: ${errorText}`);
}
inserted += batch.length;
console.log(`已插入 ${inserted}/${data.length} 条记录`);
} catch (error) {
console.error(`请求ClickHouse时出错:`, error);
throw error;
}
}
return inserted;
}

View File

@@ -0,0 +1,660 @@
// 文件名: sync_resource_relations.ts
// 描述: 此脚本用于同步PostgreSQL中资源关联数据到ClickHouse
// 作者: AI Assistant
// 创建日期: 2023-10-31
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
// PostgreSQL配置接口
interface PgConfig {
host: string;
port: number;
user: string;
password: string;
dbname?: string;
[key: string]: unknown;
}
// ClickHouse配置接口
interface ChConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_url?: string;
}
// 资源相关接口定义
interface TeamData {
team_id: string;
team_name: string;
team_description?: string;
project_id?: string;
}
interface ProjectData {
project_id: string;
project_name: string;
project_description?: string;
assigned_at?: string;
resource_id?: string;
}
interface TagData {
tag_id: string;
tag_name: string;
tag_type?: string;
created_at?: string;
resource_id?: string;
}
interface FavoriteData {
favorite_id: string;
user_id: string;
first_name?: string;
last_name?: string;
email?: string;
created_at?: string;
}
// 资源关联数据接口
interface ResourceRelations {
resource_id: string;
teams?: TeamData[];
projects?: ProjectData[];
tags?: TagData[];
favorites?: FavoriteData[];
external_id?: string;
type?: string;
attributes?: Record<string, unknown>;
}
/**
* 同步PostgreSQL资源关联数据到ClickHouse
*/
export async function main(
params: {
/** 要同步的资源ID列表 */
resource_ids: string[];
/** 是否同步teams数据 */
sync_teams?: boolean;
/** 是否同步projects数据 */
sync_projects?: boolean;
/** 是否同步tags数据 */
sync_tags?: boolean;
/** 是否同步favorites数据 */
sync_favorites?: boolean;
/** 是否为测试模式(不执行实际更新) */
dry_run?: boolean;
/** 是否显示详细日志 */
verbose?: boolean;
}
) {
// 设置默认参数
const resource_ids = params.resource_ids || [];
const sync_teams = params.sync_teams !== false;
const sync_projects = params.sync_projects !== false;
const sync_tags = params.sync_tags !== false;
const sync_favorites = params.sync_favorites !== false;
const dry_run = params.dry_run || false;
const verbose = params.verbose || false;
if (resource_ids.length === 0) {
return { success: false, message: "至少需要提供一个资源ID" };
}
// 初始化日志函数
const log = (message: string, isVerbose = false) => {
if (!isVerbose || verbose) {
console.log(message);
}
};
log(`开始同步资源关联数据: ${resource_ids.join(", ")}`);
log(`同步选项: teams=${sync_teams}, projects=${sync_projects}, tags=${sync_tags}, favorites=${sync_favorites}`, true);
let pgPool: Pool | null = null;
try {
// 1. 获取数据库配置
log("获取PostgreSQL数据库配置...", true);
const pgConfig = await getResource('f/limq/postgresql') as PgConfig;
// 2. 创建PostgreSQL连接池
pgPool = new Pool({
hostname: pgConfig.host,
port: pgConfig.port,
user: pgConfig.user,
password: pgConfig.password,
database: pgConfig.dbname || 'postgres'
}, 3);
// 3. 获取需要更新的资源完整数据
const resourcesData = await getResourcesWithRelations(pgPool, resource_ids, {
sync_teams,
sync_projects,
sync_tags,
sync_favorites
}, log);
log(`成功获取 ${resourcesData.length} 个资源的关联数据`);
if (resourcesData.length === 0) {
return { success: true, message: "没有找到需要更新的资源数据", updated: 0 };
}
// 4. 获取ClickHouse配置
const chConfig = await getClickHouseConfig();
// 5. 对每个资源执行更新
if (!dry_run) {
// 5a. 更新shorturl表数据
const shorturlUpdated = await updateClickHouseShorturl(resourcesData, chConfig, log);
// 5b. 更新events表数据
const eventsUpdated = await updateClickHouseEvents(resourcesData, chConfig, log);
return {
success: true,
message: "资源关联数据同步完成",
shorturl_updated: shorturlUpdated,
events_updated: eventsUpdated,
total_updated: shorturlUpdated + eventsUpdated
};
} else {
log("测试模式: 不执行实际更新");
if (resourcesData.length > 0) {
log("示例数据:");
log(JSON.stringify(resourcesData[0], null, 2));
}
return { success: true, dry_run: true, resources: resourcesData };
}
} catch (error) {
const errorMessage = `同步过程中发生错误: ${(error as Error).message}`;
log(errorMessage);
if ((error as Error).stack) {
log(`错误堆栈: ${(error as Error).stack}`, true);
}
return { success: false, message: errorMessage };
} finally {
if (pgPool) {
await pgPool.end();
log("PostgreSQL连接池已关闭", true);
}
}
}
/**
* 从PostgreSQL获取资源及其关联数据
*/
async function getResourcesWithRelations(
pgPool: Pool,
resourceIds: string[],
options: {
sync_teams: boolean;
sync_projects: boolean;
sync_tags: boolean;
sync_favorites: boolean;
},
log: (message: string, isVerbose?: boolean) => void
): Promise<ResourceRelations[]> {
const client = await pgPool.connect();
try {
// 准备资源IDs参数
const resourceIdsParam = resourceIds.map(id => `'${id}'`).join(',');
// 1. 获取基本资源信息
log(`获取资源基本信息: ${resourceIdsParam}`, true);
const resourcesQuery = `
SELECT
r.id,
r.external_id,
r.type,
r.attributes,
r.schema_version,
r.created_at,
r.updated_at
FROM
limq.resources r
WHERE
r.id IN (${resourceIdsParam})
AND r.deleted_at IS NULL
`;
const resourcesResult = await client.queryObject(resourcesQuery);
if (resourcesResult.rows.length === 0) {
log(`未找到有效的资源数据`, true);
return [];
}
// 处理每个资源
const enrichedResources: ResourceRelations[] = [];
for (const resource of resourcesResult.rows) {
const resourceId = resource.id as string;
log(`处理资源ID: ${resourceId}`, true);
// 初始化关联数据对象
const relationData: ResourceRelations = {
resource_id: resourceId,
external_id: resource.external_id as string,
type: resource.type as string,
attributes: parseJsonField(resource.attributes)
};
// 2. 获取项目关联
if (options.sync_projects) {
const projectsQuery = `
SELECT
pr.resource_id, pr.project_id,
p.name as project_name, p.description as project_description,
pr.assigned_at
FROM
limq.project_resources pr
JOIN
limq.projects p ON pr.project_id = p.id
WHERE
pr.resource_id = $1
AND p.deleted_at IS NULL
`;
const projectsResult = await client.queryObject(projectsQuery, [resourceId]);
relationData.projects = projectsResult.rows as ProjectData[];
log(`找到 ${projectsResult.rows.length} 个关联项目`, true);
}
// 3. 获取标签关联
if (options.sync_tags) {
const tagsQuery = `
SELECT
rt.resource_id, rt.tag_id, rt.created_at,
t.name as tag_name, t.type as tag_type
FROM
limq.resource_tags rt
JOIN
limq.tags t ON rt.tag_id = t.id
WHERE
rt.resource_id = $1
AND t.deleted_at IS NULL
`;
const tagsResult = await client.queryObject(tagsQuery, [resourceId]);
relationData.tags = tagsResult.rows as TagData[];
log(`找到 ${tagsResult.rows.length} 个关联标签`, true);
}
// 4. 获取团队关联(通过项目)
if (options.sync_teams && relationData.projects && relationData.projects.length > 0) {
const projectIds = relationData.projects.map((p: ProjectData) => p.project_id);
if (projectIds.length > 0) {
const teamsQuery = `
SELECT
tp.team_id, tp.project_id,
t.name as team_name, t.description as team_description
FROM
limq.team_projects tp
JOIN
limq.teams t ON tp.team_id = t.id
WHERE
tp.project_id = ANY($1::uuid[])
AND t.deleted_at IS NULL
`;
const teamsResult = await client.queryObject(teamsQuery, [projectIds]);
relationData.teams = teamsResult.rows as TeamData[];
log(`找到 ${teamsResult.rows.length} 个关联团队`, true);
}
}
// 5. 获取收藏关联
if (options.sync_favorites) {
const favoritesQuery = `
SELECT
f.id as favorite_id, f.user_id, f.created_at,
u.first_name, u.last_name, u.email
FROM
limq.favorite f
JOIN
limq.users u ON f.user_id = u.id
WHERE
f.favoritable_id = $1
AND f.favoritable_type = 'resource'
AND f.deleted_at IS NULL
`;
const favoritesResult = await client.queryObject(favoritesQuery, [resourceId]);
relationData.favorites = favoritesResult.rows as FavoriteData[];
log(`找到 ${favoritesResult.rows.length} 个收藏记录`, true);
}
// 添加到结果集
enrichedResources.push(relationData);
}
return enrichedResources;
} finally {
client.release();
}
}
/**
* 更新ClickHouse中的shorturl表数据
*/
async function updateClickHouseShorturl(
resources: ResourceRelations[],
chConfig: ChConfig,
log: (message: string, isVerbose?: boolean) => void
): Promise<number> {
// 只处理类型为shorturl的资源
const shorturls = resources.filter(r => r.type === 'shorturl');
if (shorturls.length === 0) {
log('没有找到shorturl类型的资源跳过shorturl表更新');
return 0;
}
log(`准备更新 ${shorturls.length} 个shorturl资源`);
let updatedCount = 0;
// 检查ClickHouse中是否存在shorturl表
const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.shorturl');
if (!tableExists) {
log('ClickHouse中未找到shorturl表请先创建表');
return 0;
}
// 对每个资源执行更新
for (const shorturl of shorturls) {
try {
// 格式化团队数据
const teams = JSON.stringify(shorturl.teams || []);
// 格式化项目数据
const projects = JSON.stringify(shorturl.projects || []);
// 格式化标签数据
const tags = JSON.stringify((shorturl.tags || []).map((t: TagData) => ({
tag_id: t.tag_id,
tag_name: t.tag_name,
tag_type: t.tag_type,
created_at: t.created_at
})));
// 格式化收藏数据
const favorites = JSON.stringify((shorturl.favorites || []).map((f: FavoriteData) => ({
favorite_id: f.favorite_id,
user_id: f.user_id,
user_name: `${f.first_name || ""} ${f.last_name || ""}`.trim(),
created_at: f.created_at
})));
// 尝试更新ClickHouse数据
const updateQuery = `
ALTER TABLE shorturl_analytics.shorturl
UPDATE
teams = '${escapeString(teams)}',
projects = '${escapeString(projects)}',
tags = '${escapeString(tags)}',
favorites = '${escapeString(favorites)}'
WHERE id = '${shorturl.resource_id}'
`;
await executeClickHouseQuery(chConfig, updateQuery);
log(`更新shorturl完成: ${shorturl.resource_id}`, true);
updatedCount++;
} catch (error) {
log(`更新shorturl ${shorturl.resource_id} 失败: ${(error as Error).message}`);
}
}
return updatedCount;
}
/**
* 更新ClickHouse中的events表数据
*/
async function updateClickHouseEvents(
resources: ResourceRelations[],
chConfig: ChConfig,
log: (message: string, isVerbose?: boolean) => void
): Promise<number> {
// 过滤出有external_id的资源
const resourcesWithExternalId = resources.filter(r => r.external_id && r.external_id.trim() !== '');
if (resourcesWithExternalId.length === 0) {
log('没有找到具有external_id的资源跳过events表更新');
return 0;
}
log(`准备更新events表中与 ${resourcesWithExternalId.length} 个外部ID相关的记录`);
// 检查ClickHouse中是否存在events表
const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.events');
if (!tableExists) {
log('ClickHouse中未找到events表请先创建表');
return 0;
}
// 提取所有的external_id
const externalIds = resourcesWithExternalId.map(r => r.external_id).filter(Boolean) as string[];
// 构建资源数据映射使用external_id作为键
const resourceMapByExternalId = resourcesWithExternalId.reduce((map, resource) => {
if (resource.external_id) {
map[resource.external_id] = resource;
}
return map;
}, {} as Record<string, ResourceRelations>);
// 获取ClickHouse中相关资源的事件记录数量
let updatedCount = 0;
try {
// 格式化外部ID列表
const formattedExternalIds = externalIds.map(id => `'${id}'`).join(', ');
// 先查询是否有相关事件
const countQuery = `
SELECT COUNT(*) as count
FROM shorturl_analytics.events
WHERE event_id IN (${formattedExternalIds})
`;
const countResult = await executeClickHouseQuery(chConfig, countQuery);
const eventCount = parseInt(countResult.trim(), 10);
if (eventCount === 0) {
// 尝试另一种查询方式
const alternateCountQuery = `
SELECT COUNT(*) as count
FROM shorturl_analytics.events
WHERE link_id IN (${formattedExternalIds})
`;
const alternateCountResult = await executeClickHouseQuery(chConfig, alternateCountQuery);
const alternateEventCount = parseInt(alternateCountResult.trim(), 10);
if (alternateEventCount === 0) {
log('没有找到相关事件记录跳过events表更新');
log(`已尝试的匹配字段: event_idlink_id`, true);
return 0;
} else {
log(`找到 ${alternateEventCount} 条以link_id匹配的事件记录需要更新`);
}
} else {
log(`找到 ${eventCount} 条以event_id匹配的事件记录需要更新`);
}
// 批量更新每个资源相关的事件记录
for (const externalId of externalIds) {
const resource = resourceMapByExternalId[externalId];
if (!resource) continue;
// 获取关联数据
const tags = resource.tags ? JSON.stringify(resource.tags) : null;
if (tags) {
// 尝试通过event_id更新事件标签
const updateTagsQueryByEventId = `
ALTER TABLE shorturl_analytics.events
UPDATE link_tags = '${escapeString(tags)}'
WHERE event_id = '${externalId}'
`;
await executeClickHouseQuery(chConfig, updateTagsQueryByEventId);
log(`尝试通过event_id更新事件标签: ${externalId}`, true);
// 尝试通过link_id更新事件标签
const updateTagsQueryByLinkId = `
ALTER TABLE shorturl_analytics.events
UPDATE link_tags = '${escapeString(tags)}'
WHERE link_id = '${externalId}'
`;
await executeClickHouseQuery(chConfig, updateTagsQueryByLinkId);
log(`尝试通过link_id更新事件标签: ${externalId}`, true);
}
// 如果资源有resource_id也尝试使用它来更新
if (resource.resource_id) {
const updateByResourceId = `
ALTER TABLE shorturl_analytics.events
UPDATE link_tags = '${escapeString(tags || '[]')}'
WHERE link_id = '${resource.resource_id}'
`;
await executeClickHouseQuery(chConfig, updateByResourceId);
log(`尝试通过resource_id更新事件标签: ${resource.resource_id}`, true);
}
updatedCount++;
}
log(`已尝试更新 ${updatedCount} 个资源的事件记录`);
} catch (error) {
log(`更新events表失败: ${(error as Error).message}`);
}
return updatedCount;
}
/**
* 获取ClickHouse配置
*/
async function getClickHouseConfig(): Promise<ChConfig> {
try {
const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
// 确保配置不为空
if (!chConfigJson) {
throw new Error("未找到ClickHouse配置");
}
// 解析JSON字符串为对象
let chConfig: ChConfig;
if (typeof chConfigJson === 'string') {
try {
chConfig = JSON.parse(chConfigJson);
} catch (_) {
throw new Error("ClickHouse配置不是有效的JSON");
}
} else {
chConfig = chConfigJson as ChConfig;
}
// 验证并构建URL
if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
}
if (!chConfig.clickhouse_url) {
throw new Error("ClickHouse配置缺少URL");
}
return chConfig;
} catch (error) {
throw new Error(`获取ClickHouse配置失败: ${(error as Error).message}`);
}
}
/**
* 检查ClickHouse中是否存在指定表
*/
async function checkClickHouseTable(chConfig: ChConfig, tableName: string): Promise<boolean> {
try {
const query = `EXISTS TABLE ${tableName}`;
const result = await executeClickHouseQuery(chConfig, query);
return result.trim() === '1';
} catch (error) {
console.error(`检查表 ${tableName} 失败:`, error);
return false;
}
}
/**
* 执行ClickHouse查询
*/
async function executeClickHouseQuery(chConfig: ChConfig, query: string): Promise<string> {
// 确保URL有效
if (!chConfig.clickhouse_url) {
throw new Error("无效的ClickHouse URL: 未定义");
}
// 执行HTTP请求
try {
const response = await fetch(chConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
},
body: query,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse查询失败 (${response.status}): ${errorText}`);
}
return await response.text();
} catch (error) {
throw new Error(`执行ClickHouse查询失败: ${(error as Error).message}`);
}
}
/**
* 解析JSON字段
*/
function parseJsonField(field: unknown): Record<string, unknown> {
if (!field) return {};
try {
if (typeof field === 'string') {
return JSON.parse(field);
} else if (typeof field === 'object') {
return field as Record<string, unknown>;
}
} catch (error) {
console.warn(`无法解析JSON字段:`, error);
}
return {};
}
/**
* 转义字符串避免SQL注入
*/
function escapeString(str: string): string {
if (!str) return '';
return str.replace(/'/g, "''");
}