Compare commits
98 Commits
92d82b18a0
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
51e168ee3b | ||
|
|
cf0f35e274 | ||
| 3162836e91 | |||
| d80d5e976b | |||
| 5d5b501a66 | |||
| fe40aad835 | |||
| 92db5ad783 | |||
| b94a91914a | |||
| 8551f5c445 | |||
| dafa7f53ac | |||
| 0203cb4041 | |||
| ced29201da | |||
| a8c94c9621 | |||
| 4736ebe060 | |||
| 6858f2fda5 | |||
| 42f5be4dcb | |||
| 05af4aae70 | |||
| ed1d2e59f6 | |||
| 3cbb76db36 | |||
| ecef81b0ee | |||
| 9cb85a2910 | |||
| 3af015ca44 | |||
| f6f24d3450 | |||
| 4262f789da | |||
| 2e34cd5b4b | |||
| 2cb45781c7 | |||
| 53e1611670 | |||
| 6025641ab1 | |||
| b9c2828e54 | |||
| b1753449f5 | |||
| 85f29d8b49 | |||
| b8cd3716c4 | |||
| 48d5bdafa4 | |||
| ace231b93f | |||
| e101d19e00 | |||
| a8576121e9 | |||
| 8b407975e5 | |||
| ede83068af | |||
| d21026eafd | |||
| 6940d60510 | |||
| 4e7266240d | |||
| db70602e9f | |||
| d0e83f697b | |||
| ed327ad3f0 | |||
| f782dba0c9 | |||
| 0c4a67e769 | |||
| 694e005101 | |||
| 523e99a001 | |||
| 33dbf62665 | |||
| 1a9e28bd7e | |||
| d1d21948b6 | |||
| f32a45d24a | |||
| d61b8a62ff | |||
| 0b41f3ea42 | |||
| 63f434fd93 | |||
| 95f230b996 | |||
| 0f8419778c | |||
| a6f7172ec4 | |||
| 8054b0235d | |||
| b0dbd088e7 | |||
| bf7c62fdc9 | |||
| 9cb9f62686 | |||
| 4b7fb7a887 | |||
| bdae5c164c | |||
| 9fa61ccf8d | |||
| b187bdefdf | |||
| 87c3803236 | |||
| 75adb36111 | |||
| a4ef2c3147 | |||
| 57e16144a9 | |||
| 1be6a6dbf0 | |||
| 36f22059e9 | |||
| a8d364be1f | |||
| 326a6c6d63 | |||
| 0a881fd180 | |||
| 1b901bda90 | |||
| 53822f1087 | |||
| 1978e0224e | |||
| c0649ce10f | |||
| 696a434b95 | |||
| b8e6180212 | |||
| 6beb6c3666 | |||
| 17b588e249 | |||
| 26db8fe76d | |||
| 4ad505cda1 | |||
| 7a03396cdd | |||
| e9b9950ed3 | |||
| f5b14bf936 | |||
| ca8a7d56f1 | |||
| 913c9cd289 | |||
| e916eab92c | |||
| 63a578ef38 | |||
| b4aa765c17 | |||
| c0e5a9ccb2 | |||
| 1755b44a39 | |||
| e0ac87fb25 | |||
| ecf21a812f | |||
| efdfe8bf8e |
31
.env
Normal file
31
.env
Normal file
@@ -0,0 +1,31 @@
|
||||
PORT=3007
|
||||
|
||||
MONGO_URL="mongodb://10.0.1.41:27017"
|
||||
|
||||
# ClickHouse Configuration
|
||||
CLICKHOUSE_HOST=10.0.1.60
|
||||
CLICKHOUSE_PORT=8123
|
||||
CLICKHOUSE_USER=admin
|
||||
CLICKHOUSE_PASSWORD=your_secure_password
|
||||
CLICKHOUSE_DATABASE=shorturl_analytics
|
||||
CLICKHOUSE_URL=http://10.0.1.60:8123
|
||||
|
||||
REDIS_HOST="localhost"
|
||||
REDIS_PORT="6379"
|
||||
REDIS_PASSWORD=""
|
||||
|
||||
# Supabase Configuration
|
||||
SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
|
||||
SUPABASE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc0NDM0NTY0MywiZXhwIjoyMDU5OTIxNjQzfQ.ZenTsEAdGiDu1DCCOT7G8xxvgFXKLl4qhHB-AhSVf6w"
|
||||
SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
|
||||
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
|
||||
|
||||
|
||||
# Next.js Public Environment Variables (accessible in browser)
|
||||
NEXT_PUBLIC_SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
|
||||
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
|
||||
|
||||
NEXT_PUBLIC_LIMQ_API="https://app.upj.to"
|
||||
# Application URL for redirects (replace with your production URL)
|
||||
NEXT_PUBLIC_SITE_URL="https://main.upj.to"
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -31,7 +31,7 @@ yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
# .env
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
47
Date Format Handling for ClickHouse Events API.md
Normal file
47
Date Format Handling for ClickHouse Events API.md
Normal file
@@ -0,0 +1,47 @@
|
||||
|
||||
# Date Format Handling for ClickHouse Events API
|
||||
|
||||
## Problem Description
|
||||
|
||||
The event tracking API was experiencing issues with date format compatibility when inserting records into the ClickHouse database. ClickHouse has specific requirements for datetime formats, particularly for its `DateTime64` type fields, which weren't being properly addressed in the original implementation.
|
||||
|
||||
## Root Cause
|
||||
|
||||
- JavaScript's default date serialization (`toISOString()`) produces formats like `2023-08-24T12:34:56.789Z`, which include `T` as a separator and `Z` as the UTC timezone indicator
|
||||
- ClickHouse prefers datetime values in the format `YYYY-MM-DD HH:MM:SS.SSS` for seamless parsing
|
||||
- The mismatch between these formats was causing insertion errors in the database
|
||||
|
||||
## Solution Implemented
|
||||
|
||||
We created a `formatDateTime` utility function that properly formats JavaScript Date objects for ClickHouse compatibility:
|
||||
|
||||
```typescript
|
||||
const formatDateTime = (date: Date) => {
|
||||
return date.toISOString().replace('T', ' ').replace('Z', '');
|
||||
};
|
||||
```
|
||||
|
||||
This function:
|
||||
1. Takes a JavaScript Date object as input
|
||||
2. Converts it to ISO format string
|
||||
3. Replaces the 'T' separator with a space
|
||||
4. Removes the trailing 'Z' UTC indicator
|
||||
|
||||
The solution was applied to all date fields in the event payload:
|
||||
- `event_time`
|
||||
- `link_created_at`
|
||||
- `link_expires_at`
|
||||
|
||||
## Additional Improvements
|
||||
|
||||
- We standardized date handling by using a consistent `currentTime` variable
|
||||
- Added type checking for JSON fields to ensure proper serialization
|
||||
- Improved error handling for date parsing failures
|
||||
|
||||
## Best Practices for ClickHouse Date Handling
|
||||
|
||||
1. Always format dates as `YYYY-MM-DD HH:MM:SS.SSS` when inserting into ClickHouse
|
||||
2. Use consistent date handling utilities across your application
|
||||
3. Consider timezone handling explicitly when needed
|
||||
4. For query parameters, use ClickHouse's `parseDateTimeBestEffort` function when possible
|
||||
5. Test with various date formats and edge cases to ensure robustness
|
||||
42
README-auth-setup.md
Normal file
42
README-auth-setup.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# 身份验证重定向 URL 配置指南
|
||||
|
||||
## 问题:注册后确认邮件链接指向 localhost
|
||||
|
||||
如果您在生产环境中使用此应用,并且发现用户注册后收到的确认邮件中的链接指向 `localhost` 而非您的实际网站域名,请按照以下步骤解决:
|
||||
|
||||
## 解决方案
|
||||
|
||||
### 1. 设置环境变量
|
||||
|
||||
在项目根目录的 `.env.production` 文件中,确保 `NEXT_PUBLIC_SITE_URL` 变量设置为您的实际生产域名:
|
||||
|
||||
```
|
||||
NEXT_PUBLIC_SITE_URL="https://您的真实域名.com"
|
||||
```
|
||||
|
||||
### 2. 在 Supabase 控制台中配置
|
||||
|
||||
登录 [Supabase 控制台](https://app.supabase.com/),然后:
|
||||
|
||||
1. 选择您的项目
|
||||
2. 导航到 **Authentication** > **URL Configuration**
|
||||
3. 在 **Site URL** 字段中输入您的实际网站 URL
|
||||
4. 在 **Redirect URLs** 部分添加:
|
||||
- `https://您的真实域名.com/auth/callback`
|
||||
|
||||
### 3. 本地开发与生产环境
|
||||
|
||||
- **开发环境**:使用 `.env.development` 文件中的设置,通常为 `http://localhost:3007`
|
||||
- **生产环境**:使用 `.env.production` 文件中的设置,应为您的实际域名
|
||||
|
||||
### 4. 部署后验证
|
||||
|
||||
项目重新部署后:
|
||||
1. 尝试注册一个新账户
|
||||
2. 检查收到的确认邮件,确认链接现在指向您的实际域名而非 localhost
|
||||
|
||||
## 技术说明
|
||||
|
||||
身份验证流程中,应用使用环境变量 `NEXT_PUBLIC_SITE_URL` 构建重定向 URL。如果未设置此变量,它会回退到使用 `window.location.origin`,这在本地开发时会是 `localhost`。
|
||||
|
||||
通过正确设置此变量,您可以确保无论在何处运行应用,邮件中的链接都能正确指向应用的实际位置。
|
||||
50
README-google-auth.md
Normal file
50
README-google-auth.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# 配置 Google 登录功能
|
||||
|
||||
为了启用 Google 登录功能,您需要在 Supabase 和 Google Cloud Platform 进行配置。
|
||||
|
||||
## 步骤 1: 创建 Google OAuth 客户端
|
||||
|
||||
1. 访问 [Google Cloud Console](https://console.cloud.google.com/)
|
||||
2. 创建一个新项目或选择现有项目
|
||||
3. 在左侧菜单中导航到 "API 和服务" > "OAuth 同意屏幕"
|
||||
4. 选择用户类型(外部或内部),然后点击"创建"
|
||||
5. 填写必要的信息(应用名称、用户支持电子邮件等)并保存
|
||||
6. 导航到 "API 和服务" > "凭据"
|
||||
7. 点击"创建凭据" > "OAuth 客户端 ID"
|
||||
8. 应用类型选择 "Web 应用"
|
||||
9. 名称中输入您的应用名称
|
||||
10. 添加以下已获授权的重定向 URI:
|
||||
- `https://mwwvqwevplndzvmqmrxa.supabase.co/auth/v1/callback`
|
||||
11. 点击"创建"
|
||||
12. 复制生成的 "客户端 ID" 和 "客户端密钥"
|
||||
|
||||
## 步骤 2: 在 Supabase 中配置 Google 提供商
|
||||
|
||||
1. 登录 [Supabase 仪表板](https://app.supabase.com)
|
||||
2. 选择您的项目
|
||||
3. 导航到 "身份验证" > "提供商"
|
||||
4. 找到 Google 提供商并启用它
|
||||
5. 粘贴您刚才获取的 "客户端 ID" 和 "客户端密钥"
|
||||
6. 保存配置
|
||||
|
||||
## 步骤 3: 更新重定向 URL(如有需要)
|
||||
|
||||
如果您的应用需要在登录后重定向到特定页面,请确保在 Google Cloud Console 和 Supabase 中配置了正确的重定向 URL。
|
||||
|
||||
在 Supabase 中:
|
||||
1. 导航到 "身份验证" > "URL 配置"
|
||||
2. 添加您的前端 URL 到站点 URL 字段中
|
||||
3. 设置重定向 URL(通常是您的前端 URL)
|
||||
|
||||
## 测试
|
||||
|
||||
1. 在您的应用中,尝试使用 Google 登录
|
||||
2. 验证认证流程,确保可以成功登录并重定向到应用
|
||||
3. 检查 Supabase 中的用户数据,确认新用户已创建
|
||||
|
||||
## 故障排除
|
||||
|
||||
- 确保重定向 URI 完全匹配
|
||||
- 确保 OAuth 同意屏幕已正确配置
|
||||
- 查看 Supabase 和应用程序中的日志以获取详细的错误信息
|
||||
- 如果遇到 CORS 错误,检查您的站点 URL 配置
|
||||
152
api/events.ts
152
api/events.ts
@@ -1,152 +0,0 @@
|
||||
import { Router } from 'express';
|
||||
import type { Request, Response } from 'express';
|
||||
import type { ApiResponse, EventsQueryParams } from '../lib/types';
|
||||
import {
|
||||
getEvents,
|
||||
getEventsSummary,
|
||||
getTimeSeriesData,
|
||||
getGeoAnalytics,
|
||||
getDeviceAnalytics
|
||||
} from '../lib/analytics';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// 获取事件列表
|
||||
router.get('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const params: EventsQueryParams = {
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
eventType: req.query.eventType as string,
|
||||
linkId: req.query.linkId as string,
|
||||
linkSlug: req.query.linkSlug as string,
|
||||
userId: req.query.userId as string,
|
||||
teamId: req.query.teamId as string,
|
||||
projectId: req.query.projectId as string,
|
||||
page: req.query.page ? parseInt(req.query.page as string, 10) : 1,
|
||||
pageSize: req.query.pageSize ? parseInt(req.query.pageSize as string, 10) : 20,
|
||||
sortBy: req.query.sortBy as string,
|
||||
sortOrder: req.query.sortOrder as 'asc' | 'desc'
|
||||
};
|
||||
|
||||
const { events, total } = await getEvents(params);
|
||||
|
||||
const response: ApiResponse<typeof events> = {
|
||||
success: true,
|
||||
data: events,
|
||||
meta: {
|
||||
total,
|
||||
page: params.page,
|
||||
pageSize: params.pageSize
|
||||
}
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取事件概览
|
||||
router.get('/summary', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const summary = await getEventsSummary({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof summary> = {
|
||||
success: true,
|
||||
data: summary
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取时间序列数据
|
||||
router.get('/time-series', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const data = await getTimeSeriesData({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string,
|
||||
granularity: (req.query.granularity || 'day') as 'hour' | 'day' | 'week' | 'month'
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取地理位置分析
|
||||
router.get('/geo', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const data = await getGeoAnalytics({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string,
|
||||
groupBy: (req.query.groupBy || 'country') as 'country' | 'city'
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取设备分析
|
||||
router.get('/devices', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const data = await getDeviceAnalytics({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
28
app/(app)/layout.tsx
Normal file
28
app/(app)/layout.tsx
Normal file
@@ -0,0 +1,28 @@
|
||||
import '../globals.css';
|
||||
import type { Metadata } from 'next';
|
||||
import { Sidebar } from '@/app/components/Sidebar';
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: 'ShortURL Analytics',
|
||||
description: 'Analytics for your shortened URLs',
|
||||
};
|
||||
|
||||
export default function AppLayout({
|
||||
children,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex h-screen bg-gray-50">
|
||||
{/* 侧边栏 */}
|
||||
<Sidebar />
|
||||
|
||||
{/* 主内容区域 */}
|
||||
<div className="flex-1 flex flex-col overflow-auto">
|
||||
<main className="flex-1 overflow-y-auto">
|
||||
{children}
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
65
app/(app)/page.tsx
Normal file
65
app/(app)/page.tsx
Normal file
@@ -0,0 +1,65 @@
|
||||
export default function HomePage() {
|
||||
return (
|
||||
<div className="container mx-auto px-4 py-8">
|
||||
<div className="text-center">
|
||||
<h1 className="text-4xl font-bold text-gray-900 mb-8">
|
||||
Welcome to ShortURL Analytics
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-8">
|
||||
<a
|
||||
href="/dashboard"
|
||||
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
|
||||
>
|
||||
<h2 className="text-xl font-semibold text-gray-900 mb-2">
|
||||
Dashboard
|
||||
</h2>
|
||||
|
||||
<p className="text-gray-600">
|
||||
Get an overview of all your short URL analytics data.
|
||||
</p>
|
||||
</a>
|
||||
|
||||
<a
|
||||
href="/events"
|
||||
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
|
||||
>
|
||||
<h2 className="text-xl font-semibold text-gray-900 mb-2">
|
||||
Event Tracking
|
||||
</h2>
|
||||
|
||||
<p className="text-gray-600">
|
||||
View detailed events for all your short URLs.
|
||||
</p>
|
||||
</a>
|
||||
|
||||
<a
|
||||
href="/analytics"
|
||||
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
|
||||
>
|
||||
<h2 className="text-xl font-semibold text-gray-900 mb-2">
|
||||
URL Analysis
|
||||
</h2>
|
||||
|
||||
<p className="text-gray-600">
|
||||
Analyze performance of specific short URLs.
|
||||
</p>
|
||||
</a>
|
||||
|
||||
<a
|
||||
href="/account"
|
||||
className="block p-6 bg-white rounded-lg shadow hover:shadow-md transition-shadow"
|
||||
>
|
||||
<h2 className="text-xl font-semibold text-gray-900 mb-2">
|
||||
Account Settings
|
||||
</h2>
|
||||
|
||||
<p className="text-gray-600">
|
||||
Manage your account and team settings.
|
||||
</p>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
1123
app/analytics/page.tsx
Normal file
1123
app/analytics/page.tsx
Normal file
File diff suppressed because it is too large
Load Diff
126
app/api/activities/readme.md
Normal file
126
app/api/activities/readme.md
Normal file
@@ -0,0 +1,126 @@
|
||||
# Activities API Documentation
|
||||
|
||||
## Overview
|
||||
The Activities API provides event tracking data for short URLs. It allows retrieving visitor activity information based on various filters such as URL slug, domain, and date ranges.
|
||||
|
||||
## Endpoint
|
||||
```
|
||||
GET /api/activities
|
||||
```
|
||||
|
||||
## Request Parameters
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|------------|---------|----------|-------------|
|
||||
| `slug` | string | No* | The short URL slug to filter events by |
|
||||
| `domain` | string | No* | The domain to filter events by |
|
||||
| `startTime`| string | No* | Start time for date range filter (ISO format) |
|
||||
| `endTime` | string | No* | End time for date range filter (ISO format) |
|
||||
| `page` | integer | No | Page number for pagination (default: 1) |
|
||||
| `pageSize` | integer | No | Number of records per page (default: 50) |
|
||||
| `format` | string | No | Response format, set to 'csv' for CSV output (default: JSON) |
|
||||
|
||||
\* Either `slug`+`domain` combination OR at least one of `startTime`/`endTime` must be provided.
|
||||
|
||||
## Response Formats
|
||||
|
||||
### JSON Format (Default)
|
||||
JSON responses include the following structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": [
|
||||
{
|
||||
"id": "event-id",
|
||||
"type": "event-type",
|
||||
"time": "timestamp",
|
||||
"visitor": {
|
||||
"id": "visitor-id",
|
||||
"ipAddress": "ip-address",
|
||||
"userAgent": "user-agent-string",
|
||||
"referrer": "referrer-url"
|
||||
},
|
||||
"device": {
|
||||
"type": "device-type",
|
||||
"browser": "browser-name",
|
||||
"os": "operating-system"
|
||||
},
|
||||
"location": {
|
||||
"country": "country-code",
|
||||
"city": "city-name"
|
||||
},
|
||||
"link": {
|
||||
"id": "link-id",
|
||||
"slug": "link-slug",
|
||||
"originalUrl": "original-url",
|
||||
"label": "link-label",
|
||||
"tags": ["tag1", "tag2"]
|
||||
},
|
||||
"utm": {
|
||||
"source": "utm-source",
|
||||
"medium": "utm-medium",
|
||||
"campaign": "utm-campaign",
|
||||
"term": "utm-term",
|
||||
"content": "utm-content"
|
||||
}
|
||||
}
|
||||
],
|
||||
"meta": {
|
||||
"total": 100,
|
||||
"page": 1,
|
||||
"pageSize": 50
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
In case of an error:
|
||||
```json
|
||||
{
|
||||
"success": false,
|
||||
"data": null,
|
||||
"error": "Error message description"
|
||||
}
|
||||
```
|
||||
|
||||
### CSV Format
|
||||
When `format=csv` is specified, the response is returned as plain text in CSV format with the following columns:
|
||||
- `time`: Timestamp of the event
|
||||
- `activity`: Type of activity/event
|
||||
- `campaign`: UTM campaign value (defaults to "demo" if not found)
|
||||
- `clientId`: Visitor ID
|
||||
- `originPath`: Original request path or referrer URL
|
||||
|
||||
## Examples
|
||||
|
||||
### Get activities for a specific short URL
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link
|
||||
```
|
||||
|
||||
### Get activities within a date range
|
||||
```
|
||||
GET /api/activities?startTime=2023-06-01T00:00:00Z&endTime=2023-06-30T23:59:59Z
|
||||
```
|
||||
|
||||
### Get events as CSV
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link&format=csv
|
||||
```
|
||||
|
||||
### Pagination example
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link&page=2&pageSize=20
|
||||
```
|
||||
|
||||
## Error Codes
|
||||
|
||||
| Status Code | Description |
|
||||
|-------------|-------------|
|
||||
| 400 | Missing required parameters |
|
||||
| 500 | Server error while processing the request |
|
||||
|
||||
## Notes
|
||||
- For privacy and security reasons, some fields may be omitted or anonymized based on user settings.
|
||||
- The CSV format is optimized for easy import into spreadsheet applications.
|
||||
- When using the CSV format, the response is returned as plain text rather than a downloadable file.
|
||||
250
app/api/activities/route.ts
Normal file
250
app/api/activities/route.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getEvents } from '@/lib/analytics';
|
||||
import { ApiResponse } from '@/lib/types';
|
||||
|
||||
// Extended Event type with required fields
|
||||
interface EventWithFullPath {
|
||||
event_id?: string;
|
||||
event_time?: string;
|
||||
event_type?: string;
|
||||
visitor_id?: string;
|
||||
ip_address?: string;
|
||||
req_full_path?: string;
|
||||
referrer?: string;
|
||||
event_attributes?: string | Record<string, unknown>;
|
||||
link_tags?: string | string[];
|
||||
link_id?: string;
|
||||
link_slug?: string;
|
||||
link_original_url?: string;
|
||||
link_label?: string;
|
||||
device_type?: string;
|
||||
browser?: string;
|
||||
os?: string;
|
||||
country?: string;
|
||||
city?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
|
||||
// Get parameters
|
||||
const slug = searchParams.get('slug');
|
||||
const domain = searchParams.get('domain');
|
||||
const format = searchParams.get('format');
|
||||
|
||||
// Optional date range parameters
|
||||
const startTime = searchParams.get('startTime') || undefined;
|
||||
const endTime = searchParams.get('endTime') || undefined;
|
||||
|
||||
// Check if either slug or domain is provided without the other
|
||||
if ((slug && !domain) || (!slug && domain)) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Both slug and domain parameters must be provided together'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// Ensure either slug+domain or date range is provided
|
||||
if ((!slug && !domain) && (!startTime && !endTime)) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Missing filter parameters: provide either slug+domain or date range'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// Construct the shortUrl from domain and slug if both are provided
|
||||
let shortUrl = undefined;
|
||||
if (slug && domain) {
|
||||
shortUrl = `https://${domain}/${slug}`;
|
||||
|
||||
// Log the request for debugging
|
||||
console.log('Activities API received parameters:', {
|
||||
slug,
|
||||
domain,
|
||||
shortUrl,
|
||||
startTime,
|
||||
endTime
|
||||
});
|
||||
} else {
|
||||
console.log('Activities API using time range filter:', {
|
||||
startTime,
|
||||
endTime
|
||||
});
|
||||
}
|
||||
|
||||
// Set default page size and page
|
||||
const page = parseInt(searchParams.get('page') || '1');
|
||||
const pageSize = parseInt(searchParams.get('pageSize') || '50');
|
||||
|
||||
// Get events for the specified filters
|
||||
const { events, total } = await getEvents({
|
||||
linkSlug: slug || undefined,
|
||||
page,
|
||||
pageSize,
|
||||
startTime,
|
||||
endTime,
|
||||
sortBy: 'event_time',
|
||||
sortOrder: 'desc'
|
||||
});
|
||||
|
||||
// If format=csv, return CSV format data
|
||||
if (format === 'csv') {
|
||||
// CSV header line
|
||||
let csvContent = 'time,activity,campaign,clientId,originPath\n';
|
||||
|
||||
// Helper function to extract utm_campaign from URL
|
||||
const extractUtmCampaign = (url: string | null | undefined): string => {
|
||||
if (!url) return 'demo';
|
||||
|
||||
try {
|
||||
// Try to parse URL and extract utm_campaign parameter
|
||||
const urlObj = new URL(url.startsWith('http') ? url : `https://example.com${url}`);
|
||||
const campaign = urlObj.searchParams.get('utm_campaign');
|
||||
if (campaign) return campaign;
|
||||
|
||||
// If utm_campaign is not found or URL parsing fails, use regex as fallback
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
|
||||
} catch {
|
||||
// If URL parsing fails, try regex directly
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
|
||||
}
|
||||
|
||||
return 'demo'; // Default value
|
||||
};
|
||||
|
||||
// Process each event record
|
||||
events.forEach(event => {
|
||||
// 使用类型断言处理扩展字段
|
||||
const eventWithFullPath = event as unknown as EventWithFullPath;
|
||||
|
||||
// Get the full URL from appropriate field
|
||||
// Try different possible fields that might contain the URL
|
||||
const fullUrl = eventWithFullPath.req_full_path || eventWithFullPath.referrer || '';
|
||||
|
||||
// Extract campaign from URL
|
||||
const campaign = extractUtmCampaign(fullUrl);
|
||||
|
||||
// Format time
|
||||
const time = eventWithFullPath.event_time ?
|
||||
new Date(eventWithFullPath.event_time).toISOString().replace('T', ' ').slice(0, 19) :
|
||||
'';
|
||||
|
||||
// Determine activity (event_type)
|
||||
const activity = eventWithFullPath.event_type || '';
|
||||
|
||||
// 修改:使用link_label替代visitor_id作为clientId
|
||||
const clientId = eventWithFullPath.link_label || 'undefined';
|
||||
|
||||
// Original path - 修正:使用link_original_url作为原始URL来源
|
||||
const originPath = eventWithFullPath.link_original_url || 'undefined';
|
||||
|
||||
// Add to CSV content
|
||||
csvContent += `${time},${activity},${campaign},${clientId},${originPath}\n`;
|
||||
});
|
||||
|
||||
// No need to generate filename since we're not using Content-Disposition header
|
||||
|
||||
// Return CSV response without forcing download
|
||||
return new NextResponse(csvContent, {
|
||||
headers: {
|
||||
'Content-Type': 'text/plain'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Process the events to extract useful information
|
||||
const processedEvents = events.map(event => {
|
||||
// Parse JSON strings to objects safely
|
||||
let eventAttributes: Record<string, unknown> = {};
|
||||
|
||||
try {
|
||||
if (typeof event.event_attributes === 'string') {
|
||||
eventAttributes = JSON.parse(event.event_attributes);
|
||||
} else if (typeof event.event_attributes === 'object') {
|
||||
eventAttributes = event.event_attributes;
|
||||
}
|
||||
} catch {
|
||||
// Keep default empty object if parsing fails
|
||||
}
|
||||
|
||||
// Extract tags
|
||||
let tags: string[] = [];
|
||||
|
||||
try {
|
||||
if (typeof event.link_tags === 'string') {
|
||||
const parsedTags = JSON.parse(event.link_tags);
|
||||
if (Array.isArray(parsedTags)) {
|
||||
tags = parsedTags;
|
||||
}
|
||||
} else if (Array.isArray(event.link_tags)) {
|
||||
tags = event.link_tags;
|
||||
}
|
||||
} catch {
|
||||
// If parsing fails, keep tags as empty array
|
||||
}
|
||||
|
||||
// Return a simplified event object
|
||||
return {
|
||||
id: event.event_id,
|
||||
type: event.event_type,
|
||||
time: event.event_time,
|
||||
visitor: {
|
||||
id: event.visitor_id,
|
||||
ipAddress: event.ip_address,
|
||||
userAgent: eventAttributes.user_agent as string || null,
|
||||
referrer: eventAttributes.referrer as string || null
|
||||
},
|
||||
device: {
|
||||
type: event.device_type,
|
||||
browser: event.browser,
|
||||
os: event.os
|
||||
},
|
||||
location: {
|
||||
country: event.country,
|
||||
city: event.city
|
||||
},
|
||||
link: {
|
||||
id: event.link_id,
|
||||
slug: event.link_slug,
|
||||
originalUrl: event.link_original_url,
|
||||
label: event.link_label,
|
||||
tags
|
||||
},
|
||||
utm: {
|
||||
source: eventAttributes.utm_source as string || null,
|
||||
medium: eventAttributes.utm_medium as string || null,
|
||||
campaign: eventAttributes.utm_campaign as string || null,
|
||||
term: eventAttributes.utm_term as string || null,
|
||||
content: eventAttributes.utm_content as string || null
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Return processed events
|
||||
const response: ApiResponse<typeof processedEvents> = {
|
||||
success: true,
|
||||
data: processedEvents,
|
||||
meta: {
|
||||
total,
|
||||
page,
|
||||
pageSize
|
||||
}
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error retrieving activities:', error);
|
||||
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
data: null,
|
||||
error: error instanceof Error ? error.message : 'An error occurred while retrieving activities'
|
||||
};
|
||||
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -6,10 +6,21 @@ export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
// 获取团队、项目和标签筛选参数
|
||||
const teamIds = searchParams.getAll('teamId');
|
||||
const projectIds = searchParams.getAll('projectId');
|
||||
const tagIds = searchParams.getAll('tagId');
|
||||
|
||||
const data = await getDeviceAnalytics({
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
// 添加团队、项目和标签筛选
|
||||
teamIds: teamIds.length > 0 ? teamIds : undefined,
|
||||
projectIds: projectIds.length > 0 ? projectIds : undefined,
|
||||
tagIds: tagIds.length > 0 ? tagIds : undefined,
|
||||
// 添加子路径筛选
|
||||
subpath: searchParams.get('subpath') || undefined
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
|
||||
@@ -6,11 +6,25 @@ export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
// 获取团队、项目和标签筛选参数
|
||||
const teamIds = searchParams.getAll('teamId');
|
||||
const projectIds = searchParams.getAll('projectId');
|
||||
const tagIds = searchParams.getAll('tagId');
|
||||
|
||||
// Get the groupBy parameter
|
||||
const groupBy = searchParams.get('groupBy') as 'country' | 'city' | 'region' | 'continent' | null;
|
||||
|
||||
const data = await getGeoAnalytics({
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
groupBy: (searchParams.get('groupBy') || 'country') as 'country' | 'city'
|
||||
groupBy: groupBy || undefined,
|
||||
// 添加团队、项目和标签筛选
|
||||
teamIds: teamIds.length > 0 ? teamIds : undefined,
|
||||
projectIds: projectIds.length > 0 ? projectIds : undefined,
|
||||
tagIds: tagIds.length > 0 ? tagIds : undefined,
|
||||
// 添加子路径筛选
|
||||
subpath: searchParams.get('subpath') || undefined
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
|
||||
80
app/api/events/path-analytics/route.ts
Normal file
80
app/api/events/path-analytics/route.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
import { executeQuery } from '@/lib/clickhouse';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取查询参数
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const startTime = searchParams.get('startTime');
|
||||
const endTime = searchParams.get('endTime');
|
||||
const linkId = searchParams.get('linkId');
|
||||
|
||||
if (!startTime || !endTime || !linkId) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Missing required parameters'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// 查询链接的点击事件
|
||||
const query = `
|
||||
SELECT event_attributes
|
||||
FROM events
|
||||
WHERE link_id = '${linkId}'
|
||||
AND event_time >= parseDateTimeBestEffort('${startTime}')
|
||||
AND event_time <= parseDateTimeBestEffort('${endTime}')
|
||||
AND event_type = 'click'
|
||||
`;
|
||||
|
||||
const events = await executeQuery(query);
|
||||
|
||||
// 处理事件数据,按路径分组
|
||||
const pathMap = new Map<string, number>();
|
||||
let totalClicks = 0;
|
||||
|
||||
events.forEach((event: any) => {
|
||||
try {
|
||||
if (event.event_attributes) {
|
||||
const attrs = JSON.parse(event.event_attributes);
|
||||
if (attrs.full_url) {
|
||||
// 提取URL的路径和参数部分
|
||||
const url = new URL(attrs.full_url);
|
||||
const pathWithParams = url.pathname + (url.search || '');
|
||||
|
||||
// 更新路径计数
|
||||
const currentCount = pathMap.get(pathWithParams) || 0;
|
||||
pathMap.set(pathWithParams, currentCount + 1);
|
||||
totalClicks++;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// 忽略解析错误
|
||||
}
|
||||
});
|
||||
|
||||
// 转换为数组并按点击数排序
|
||||
const pathData = Array.from(pathMap.entries())
|
||||
.map(([path, count]) => ({
|
||||
path,
|
||||
count,
|
||||
percentage: totalClicks > 0 ? count / totalClicks : 0,
|
||||
}))
|
||||
.sort((a, b) => b.count - a.count);
|
||||
|
||||
const response: ApiResponse<typeof pathData> = {
|
||||
success: true,
|
||||
data: pathData,
|
||||
meta: { total: totalClicks }
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error fetching path analytics data:', error);
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Internal server error'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -1,50 +1,74 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse, EventsQueryParams, EventType } from '@/lib/types';
|
||||
import {
|
||||
getEvents,
|
||||
getEventsSummary,
|
||||
getTimeSeriesData,
|
||||
getGeoAnalytics,
|
||||
getDeviceAnalytics
|
||||
} from '@/lib/analytics';
|
||||
import { getEvents, EventsQueryParams } from '@/lib/analytics';
|
||||
import { ApiResponse } from '@/lib/types';
|
||||
|
||||
// 获取事件列表
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const { searchParams } = new URL(request.url);
|
||||
|
||||
// 获取查询参数
|
||||
const page = parseInt(searchParams.get('page') || '1');
|
||||
const pageSize = parseInt(searchParams.get('pageSize') || '20');
|
||||
const eventType = searchParams.get('eventType') || undefined;
|
||||
const linkId = searchParams.get('linkId') || undefined;
|
||||
const linkSlug = searchParams.get('linkSlug') || undefined;
|
||||
const userId = searchParams.get('userId') || undefined;
|
||||
const subpath = searchParams.get('subpath') || undefined;
|
||||
|
||||
// 获取可能存在的多个团队、项目和标签ID
|
||||
const teamIds = searchParams.getAll('teamId');
|
||||
const projectIds = searchParams.getAll('projectId');
|
||||
const tagIds = searchParams.getAll('tagId');
|
||||
|
||||
const startTime = searchParams.get('startTime') || undefined;
|
||||
const endTime = searchParams.get('endTime') || undefined;
|
||||
const sortBy = searchParams.get('sortBy') || undefined;
|
||||
const sortOrder = (searchParams.get('sortOrder') as 'asc' | 'desc') || undefined;
|
||||
|
||||
console.log("API接收到的tagIds:", tagIds); // 添加日志便于调试
|
||||
console.log("API接收到的subpath:", subpath); // 添加日志便于调试
|
||||
|
||||
// 获取事件列表
|
||||
const params: EventsQueryParams = {
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
eventType: searchParams.get('eventType') as EventType || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
linkSlug: searchParams.get('linkSlug') || undefined,
|
||||
userId: searchParams.get('userId') || undefined,
|
||||
teamId: searchParams.get('teamId') || undefined,
|
||||
projectId: searchParams.get('projectId') || undefined,
|
||||
page: searchParams.has('page') ? parseInt(searchParams.get('page')!, 10) : 1,
|
||||
pageSize: searchParams.has('pageSize') ? parseInt(searchParams.get('pageSize')!, 10) : 20,
|
||||
sortBy: searchParams.get('sortBy') || undefined,
|
||||
sortOrder: (searchParams.get('sortOrder') as 'asc' | 'desc') || undefined
|
||||
page,
|
||||
pageSize,
|
||||
eventType,
|
||||
linkId,
|
||||
linkSlug,
|
||||
userId,
|
||||
subpath,
|
||||
teamIds: teamIds.length > 0 ? teamIds : undefined,
|
||||
projectIds: projectIds.length > 0 ? projectIds : undefined,
|
||||
tagIds: tagIds.length > 0 ? tagIds : undefined,
|
||||
startTime,
|
||||
endTime,
|
||||
sortBy,
|
||||
sortOrder
|
||||
};
|
||||
|
||||
const { events, total } = await getEvents(params);
|
||||
|
||||
const response: ApiResponse<typeof events> = {
|
||||
|
||||
// 记录完整的参数用于调试
|
||||
console.log("完整请求参数:", JSON.stringify(params));
|
||||
|
||||
const result = await getEvents(params);
|
||||
|
||||
const response: ApiResponse<typeof result.events> = {
|
||||
success: true,
|
||||
data: events,
|
||||
data: result.events,
|
||||
meta: {
|
||||
total,
|
||||
page: params.page,
|
||||
pageSize: params.pageSize
|
||||
total: result.total,
|
||||
page,
|
||||
pageSize
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('获取事件列表失败:', error);
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
data: null,
|
||||
error: error instanceof Error ? error.message : '获取事件列表失败'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
|
||||
@@ -6,10 +6,27 @@ export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
// 获取可能存在的多个团队、项目和标签ID
|
||||
const teamIds = searchParams.getAll('teamId');
|
||||
const projectIds = searchParams.getAll('projectId');
|
||||
const tagIds = searchParams.getAll('tagId');
|
||||
|
||||
// Add debug log to check if linkId is being received
|
||||
const linkId = searchParams.get('linkId');
|
||||
const subpath = searchParams.get('subpath');
|
||||
console.log('Summary API received linkId:', linkId);
|
||||
console.log('Summary API received subpath:', subpath);
|
||||
console.log('Summary API full parameters:', Object.fromEntries(searchParams.entries()));
|
||||
console.log('Summary API URL:', request.url);
|
||||
|
||||
const summary = await getEventsSummary({
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
teamIds: teamIds.length > 0 ? teamIds : undefined,
|
||||
projectIds: projectIds.length > 0 ? projectIds : undefined,
|
||||
tagIds: tagIds.length > 0 ? tagIds : undefined,
|
||||
subpath: searchParams.get('subpath') || undefined
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof summary> = {
|
||||
|
||||
@@ -15,11 +15,22 @@ export async function GET(request: NextRequest) {
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// 获取团队、项目和标签筛选参数
|
||||
const teamIds = searchParams.getAll('teamId');
|
||||
const projectIds = searchParams.getAll('projectId');
|
||||
const tagIds = searchParams.getAll('tagId');
|
||||
|
||||
const data = await getTimeSeriesData({
|
||||
startTime,
|
||||
endTime,
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
granularity: (searchParams.get('granularity') || 'day') as 'hour' | 'day' | 'week' | 'month'
|
||||
granularity: (searchParams.get('granularity') || 'day') as 'hour' | 'day' | 'week' | 'month',
|
||||
// 添加团队、项目和标签筛选
|
||||
teamIds: teamIds.length > 0 ? teamIds : undefined,
|
||||
projectIds: projectIds.length > 0 ? projectIds : undefined,
|
||||
tagIds: tagIds.length > 0 ? tagIds : undefined,
|
||||
// 添加子路径筛选
|
||||
subpath: searchParams.get('subpath') || undefined
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
|
||||
208
app/api/events/track/readme.md
Normal file
208
app/api/events/track/readme.md
Normal file
@@ -0,0 +1,208 @@
|
||||
|
||||
# 事件跟踪接口说明
|
||||
|
||||
## 概述
|
||||
该接口用于跟踪用户交互事件并将数据存储到 ClickHouse 数据库中。支持记录各种类型的事件,并可包含与链接、用户、团队、项目等相关的详细信息。
|
||||
|
||||
## 接口信息
|
||||
- **URL**: `/api/events/track`
|
||||
- **方法**: `POST`
|
||||
- **Content-Type**: `application/json`
|
||||
|
||||
## 请求参数
|
||||
|
||||
### 必填字段
|
||||
| 参数 | 类型 | 描述 |
|
||||
|------|------|------|
|
||||
| `event_type` | string | 事件类型,如 'click', 'view', 'conversion' |
|
||||
|
||||
### 核心事件字段
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `event_id` | string | 否 | 事件唯一标识符,不提供时自动生成UUID |
|
||||
| `event_time` | string/Date | 否 | 事件发生时间,格式为ISO日期字符串,默认为当前时间 |
|
||||
| `event_attributes` | object/string | 否 | 事件相关的其他属性,可以是JSON对象或JSON字符串 |
|
||||
|
||||
### 链接信息
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `link_id` | string | 否 | 短链接的唯一ID |
|
||||
| `link_slug` | string | 否 | 短链接的slug部分 |
|
||||
| `link_label` | string | 否 | 短链接的显示名称 |
|
||||
| `link_title` | string | 否 | 短链接的标题 |
|
||||
| `link_original_url` | string | 否 | 原始目标URL |
|
||||
| `link_attributes` | object/string | 否 | 链接相关的额外属性 |
|
||||
| `link_created_at` | string/Date | 否 | 链接创建时间 |
|
||||
| `link_expires_at` | string/Date | 否 | 链接过期时间 |
|
||||
| `link_tags` | array/string | 否 | 链接标签,可以是数组或JSON字符串 |
|
||||
|
||||
### 用户信息
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `user_id` | string | 否 | 用户ID |
|
||||
| `user_name` | string | 否 | 用户名称 |
|
||||
| `user_email` | string | 否 | 用户邮箱 |
|
||||
| `user_attributes` | object/string | 否 | 用户相关的其他属性 |
|
||||
|
||||
### 团队和项目信息
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `team_id` | string | 否 | 团队ID |
|
||||
| `team_name` | string | 否 | 团队名称 |
|
||||
| `team_attributes` | object/string | 否 | 团队相关的其他属性 |
|
||||
| `project_id` | string | 否 | 项目ID |
|
||||
| `project_name` | string | 否 | 项目名称 |
|
||||
| `project_attributes` | object/string | 否 | 项目相关的其他属性 |
|
||||
|
||||
### 二维码信息
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `qr_code_id` | string | 否 | 二维码ID |
|
||||
| `qr_code_name` | string | 否 | 二维码名称 |
|
||||
| `qr_code_attributes` | object/string | 否 | 二维码相关的其他属性 |
|
||||
|
||||
### 访问者信息
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `visitor_id` | string | 否 | 访问者唯一标识符,不提供时自动生成 |
|
||||
| `session_id` | string | 否 | 会话ID,不提供时自动生成 |
|
||||
| `ip_address` | string | 否 | 访问者IP地址,默认从请求头获取 |
|
||||
| `country` | string | 否 | 访问者所在国家 |
|
||||
| `city` | string | 否 | 访问者所在城市 |
|
||||
| `device_type` | string | 否 | 设备类型 (如 desktop, mobile, tablet) |
|
||||
| `browser` | string | 否 | 浏览器名称 |
|
||||
| `os` | string | 否 | 操作系统 |
|
||||
| `user_agent` | string | 否 | 用户代理字符串,默认从请求头获取 |
|
||||
|
||||
### 引荐来源信息
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `referrer` | string | 否 | 引荐URL,默认从请求头获取 |
|
||||
| `utm_source` | string | 否 | UTM来源参数 |
|
||||
| `utm_medium` | string | 否 | UTM媒介参数 |
|
||||
| `utm_campaign` | string | 否 | UTM活动参数 |
|
||||
| `utm_term` | string | 否 | UTM术语参数 |
|
||||
| `utm_content` | string | 否 | UTM内容参数 |
|
||||
|
||||
### 交互信息
|
||||
| 参数 | 类型 | 必填 | 描述 |
|
||||
|------|------|------|------|
|
||||
| `time_spent_sec` | number | 否 | 用户在页面上停留的时间(秒),默认0 |
|
||||
| `is_bounce` | boolean | 否 | 是否是跳出(只访问一个页面),默认true |
|
||||
| `is_qr_scan` | boolean | 否 | 是否来自二维码扫描,默认false |
|
||||
| `conversion_type` | string | 否 | 转化类型 |
|
||||
| `conversion_value` | number | 否 | 转化价值,默认0 |
|
||||
|
||||
## 响应格式
|
||||
|
||||
### 成功响应 (201 Created)
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message": "Event tracked successfully",
|
||||
"event_id": "uuid-of-tracked-event"
|
||||
}
|
||||
```
|
||||
|
||||
### 错误响应
|
||||
|
||||
#### 缺少必填字段 (400 Bad Request)
|
||||
```json
|
||||
{
|
||||
"error": "Missing required field: event_type"
|
||||
}
|
||||
```
|
||||
|
||||
#### 服务器错误 (500 Internal Server Error)
|
||||
```json
|
||||
{
|
||||
"error": "Failed to track event",
|
||||
"details": "具体错误信息"
|
||||
}
|
||||
```
|
||||
|
||||
## 使用示例
|
||||
|
||||
### 基本事件跟踪请求
|
||||
```javascript
|
||||
fetch('/api/events/track', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
event_type: 'click',
|
||||
link_id: 'abc123',
|
||||
link_slug: 'promo-summer',
|
||||
link_original_url: 'https://example.com/summer-promotion'
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
### 详细事件跟踪请求
|
||||
```javascript
|
||||
fetch('/api/events/track', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
event_type: 'conversion',
|
||||
link_id: 'abc123',
|
||||
link_slug: 'promo-summer',
|
||||
link_original_url: 'https://example.com/summer-promotion',
|
||||
event_attributes: {
|
||||
page: '/checkout',
|
||||
product_id: 'xyz789'
|
||||
},
|
||||
user_id: 'user123',
|
||||
team_id: 'team456',
|
||||
project_id: 'proj789',
|
||||
visitor_id: 'vis987',
|
||||
is_bounce: false,
|
||||
time_spent_sec: 120,
|
||||
conversion_type: 'purchase',
|
||||
conversion_value: 99.99,
|
||||
utm_source: 'email',
|
||||
utm_campaign: 'summer_sale'
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
## 注意事项
|
||||
- 所有对象类型的字段(如 `event_attributes`)可以作为对象或预先格式化的JSON字符串传递
|
||||
- 如果不提供 `event_id`、`visitor_id` 或 `session_id`,系统将自动生成
|
||||
- 时间戳字段接受ISO格式的日期字符串,并会被转换为ClickHouse兼容的格式
|
||||
|
||||
|
||||
UTM 测试示例。1. 电子邮件营销链接
|
||||
https://short.domain.com/summer?utm_source=newsletter&utm_medium=email&utm_campaign=summer_promo&utm_term=discount&utm_content=header
|
||||
说明: 用于电子邮件营销活动,跟踪用户从邮件头部横幅点击的流量。
|
||||
|
||||
2. 社交媒体广告链接
|
||||
https://short.domain.com/product?utm_source=instagram&utm_medium=social&utm_campaign=fall_collection&utm_content=story
|
||||
说明: 用于 Instagram Story 广告,跟踪用户从社交媒体故事广告点击的情况。
|
||||
|
||||
3. 搜索引擎广告链接
|
||||
https://short.domain.com/service?utm_source=google&utm_medium=cpc&utm_campaign=brand_terms&utm_term=service+name
|
||||
说明: 用于 Google Ads 广告,跟踪用户从搜索引擎付费广告点击的流量,特别是针对特定搜索词。
|
||||
|
||||
4. QR 码链接
|
||||
https://short.domain.com/event?utm_source=flyer&utm_medium=print&utm_campaign=local_event&utm_content=qr_code&source=qr
|
||||
说明: 用于打印材料上的 QR 码,跟踪用户扫描实体宣传资料的情况。
|
||||
|
||||
5. 合作伙伴引荐链接
|
||||
https://short.domain.com/partner?utm_source=affiliate&utm_medium=referral&utm_campaign=partner_program&utm_content=banner
|
||||
说明: 用于合作伙伴网站上的推广横幅,跟踪来自联盟营销的转化率。
|
||||
|
||||
|
||||
https://upj.to/5seaii?utm_source=newsletter&utm_medium=email&utm_campaign=summer_promo&utm_term=discount&utm_content=header
|
||||
|
||||
https://upj.to/5seaii?utm_source=instagram&utm_medium=social&utm_campaign=fall_collection&utm_content=story
|
||||
|
||||
https://upj.to/5seaii?utm_source=google&utm_medium=cpc&utm_campaign=brand_terms&utm_term=service+name
|
||||
|
||||
|
||||
https://upj.to/5seaii?utm_source=flyer&utm_medium=print&utm_campaign=local_event&utm_content=qr_code&source=qr
|
||||
|
||||
https://upj.to/5seaii?utm_source=affiliate&utm_medium=referral&utm_campaign=partner_program&utm_content=banner
|
||||
139
app/api/events/track/route.ts
Normal file
139
app/api/events/track/route.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { Event } from '../../types';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
|
||||
// 将时间格式化为ClickHouse兼容的格式:YYYY-MM-DD HH:MM:SS.SSS
|
||||
const formatDateTime = (date: Date) => {
|
||||
return date.toISOString().replace('T', ' ').replace('Z', '');
|
||||
};
|
||||
|
||||
// Handler for POST request to track events
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
// Parse request body
|
||||
const eventData = await req.json();
|
||||
|
||||
// Validate required fields
|
||||
if (!eventData.event_type) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing required field: event_type' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 获取当前时间并格式化
|
||||
const currentTime = formatDateTime(new Date());
|
||||
|
||||
// Set default values for required fields if missing
|
||||
const event: Event = {
|
||||
// Core event fields
|
||||
event_id: eventData.event_id || uuid(),
|
||||
event_time: eventData.event_time ? formatDateTime(new Date(eventData.event_time)) : currentTime,
|
||||
event_type: eventData.event_type,
|
||||
event_attributes: eventData.event_attributes || '{}',
|
||||
|
||||
// Link information
|
||||
link_id: eventData.link_id || '',
|
||||
link_slug: eventData.link_slug || '',
|
||||
link_label: eventData.link_label || '',
|
||||
link_title: eventData.link_title || '',
|
||||
link_original_url: eventData.link_original_url || '',
|
||||
link_attributes: eventData.link_attributes || '{}',
|
||||
link_created_at: eventData.link_created_at ? formatDateTime(new Date(eventData.link_created_at)) : currentTime,
|
||||
link_expires_at: eventData.link_expires_at ? formatDateTime(new Date(eventData.link_expires_at)) : null,
|
||||
link_tags: eventData.link_tags || '[]',
|
||||
|
||||
// User information
|
||||
user_id: eventData.user_id || '',
|
||||
user_name: eventData.user_name || '',
|
||||
user_email: eventData.user_email || '',
|
||||
user_attributes: eventData.user_attributes || '{}',
|
||||
|
||||
// Team information
|
||||
team_id: eventData.team_id || '',
|
||||
team_name: eventData.team_name || '',
|
||||
team_attributes: eventData.team_attributes || '{}',
|
||||
|
||||
// Project information
|
||||
project_id: eventData.project_id || '',
|
||||
project_name: eventData.project_name || '',
|
||||
project_attributes: eventData.project_attributes || '{}',
|
||||
|
||||
// QR code information
|
||||
qr_code_id: eventData.qr_code_id || '',
|
||||
qr_code_name: eventData.qr_code_name || '',
|
||||
qr_code_attributes: eventData.qr_code_attributes || '{}',
|
||||
|
||||
// Visitor information
|
||||
visitor_id: eventData.visitor_id || uuid(),
|
||||
session_id: eventData.session_id || uuid(),
|
||||
ip_address: eventData.ip_address || req.headers.get('x-forwarded-for')?.toString() || '',
|
||||
country: eventData.country || '',
|
||||
city: eventData.city || '',
|
||||
device_type: eventData.device_type || '',
|
||||
browser: eventData.browser || '',
|
||||
os: eventData.os || '',
|
||||
user_agent: eventData.user_agent || req.headers.get('user-agent')?.toString() || '',
|
||||
|
||||
// Referrer information
|
||||
referrer: eventData.referrer || req.headers.get('referer')?.toString() || '',
|
||||
utm_source: eventData.utm_source || '',
|
||||
utm_medium: eventData.utm_medium || '',
|
||||
utm_campaign: eventData.utm_campaign || '',
|
||||
utm_term: eventData.utm_term || '',
|
||||
utm_content: eventData.utm_content || '',
|
||||
|
||||
// Interaction information
|
||||
time_spent_sec: eventData.time_spent_sec || 0,
|
||||
is_bounce: eventData.is_bounce !== undefined ? eventData.is_bounce : true,
|
||||
is_qr_scan: eventData.is_qr_scan !== undefined ? eventData.is_qr_scan : false,
|
||||
conversion_type: eventData.conversion_type || '',
|
||||
conversion_value: eventData.conversion_value || 0,
|
||||
};
|
||||
|
||||
// 确保JSON字符串字段的正确处理
|
||||
if (typeof event.event_attributes === 'object') {
|
||||
event.event_attributes = JSON.stringify(event.event_attributes);
|
||||
}
|
||||
if (typeof event.link_attributes === 'object') {
|
||||
event.link_attributes = JSON.stringify(event.link_attributes);
|
||||
}
|
||||
if (typeof event.user_attributes === 'object') {
|
||||
event.user_attributes = JSON.stringify(event.user_attributes);
|
||||
}
|
||||
if (typeof event.team_attributes === 'object') {
|
||||
event.team_attributes = JSON.stringify(event.team_attributes);
|
||||
}
|
||||
if (typeof event.project_attributes === 'object') {
|
||||
event.project_attributes = JSON.stringify(event.project_attributes);
|
||||
}
|
||||
if (typeof event.qr_code_attributes === 'object') {
|
||||
event.qr_code_attributes = JSON.stringify(event.qr_code_attributes);
|
||||
}
|
||||
if (typeof event.link_tags === 'object') {
|
||||
event.link_tags = JSON.stringify(event.link_tags);
|
||||
}
|
||||
|
||||
// Insert event into ClickHouse
|
||||
await clickhouse.insert({
|
||||
table: 'events',
|
||||
values: [event],
|
||||
format: 'JSONEachRow',
|
||||
});
|
||||
|
||||
// Return success response
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
message: 'Event tracked successfully',
|
||||
event_id: event.event_id
|
||||
}, { status: 201 });
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error tracking event:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to track event', details: (error as Error).message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
201
app/api/events/utm/route.ts
Normal file
201
app/api/events/utm/route.ts
Normal file
@@ -0,0 +1,201 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
|
||||
interface UtmData {
|
||||
utm_value: string;
|
||||
clicks: number;
|
||||
visitors: number;
|
||||
avg_time_spent: number;
|
||||
bounces: number;
|
||||
conversions: number;
|
||||
}
|
||||
|
||||
// 辅助函数,将日期格式化为标准格式
|
||||
function formatDateTime(dateString: string): string {
|
||||
const date = new Date(dateString);
|
||||
return date.toISOString().split('.')[0];
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
// 获取过滤参数
|
||||
const startTime = searchParams.get('startTime');
|
||||
const endTime = searchParams.get('endTime');
|
||||
const linkId = searchParams.get('linkId');
|
||||
const subpath = searchParams.get('subpath');
|
||||
|
||||
// 获取团队、项目和标签筛选参数
|
||||
const teamIds = searchParams.getAll('teamId');
|
||||
const projectIds = searchParams.getAll('projectId');
|
||||
const tagIds = searchParams.getAll('tagId');
|
||||
const tagNames = searchParams.getAll('tagName');
|
||||
|
||||
// 获取UTM类型参数
|
||||
const utmType = searchParams.get('utmType') || 'source';
|
||||
|
||||
// 添加调试日志
|
||||
console.log('UTM API received parameters:', {
|
||||
startTime,
|
||||
endTime,
|
||||
linkId,
|
||||
subpath,
|
||||
teamIds,
|
||||
projectIds,
|
||||
tagIds,
|
||||
tagNames,
|
||||
utmType,
|
||||
url: request.url
|
||||
});
|
||||
|
||||
// 构建WHERE子句
|
||||
let whereClause = '';
|
||||
const conditions = [];
|
||||
|
||||
if (startTime) {
|
||||
conditions.push(`event_time >= toDateTime('${formatDateTime(startTime)}')`);
|
||||
}
|
||||
|
||||
if (endTime) {
|
||||
conditions.push(`event_time <= toDateTime('${formatDateTime(endTime)}')`);
|
||||
}
|
||||
|
||||
if (linkId) {
|
||||
conditions.push(`link_id = '${linkId}'`);
|
||||
}
|
||||
|
||||
// 添加子路径筛选 - 使用更精确的匹配方式
|
||||
if (subpath && subpath.trim() !== '') {
|
||||
console.log('====== UTM API SUBPATH DEBUG ======');
|
||||
console.log('Raw subpath param:', subpath);
|
||||
|
||||
// 清理并准备subpath值
|
||||
let cleanSubpath = subpath.trim();
|
||||
// 移除开头的斜杠以便匹配
|
||||
if (cleanSubpath.startsWith('/')) {
|
||||
cleanSubpath = cleanSubpath.substring(1);
|
||||
}
|
||||
// 移除结尾的斜杠以便匹配
|
||||
if (cleanSubpath.endsWith('/')) {
|
||||
cleanSubpath = cleanSubpath.substring(0, cleanSubpath.length - 1);
|
||||
}
|
||||
|
||||
console.log('Cleaned subpath:', cleanSubpath);
|
||||
|
||||
// 使用正则表达式匹配URL中的第二个路径部分
|
||||
// 示例: 在 "https://abc.com/slug/subpath/" 中匹配 "subpath"
|
||||
const condition = `match(JSONExtractString(event_attributes, 'full_url'), '/[^/]+/${cleanSubpath}(/|\\\\?|$)')`;
|
||||
|
||||
console.log('Final SQL condition:', condition);
|
||||
console.log('==================================');
|
||||
|
||||
conditions.push(condition);
|
||||
}
|
||||
|
||||
// 添加团队筛选
|
||||
if (teamIds && teamIds.length > 0) {
|
||||
// 如果只有一个团队ID
|
||||
if (teamIds.length === 1) {
|
||||
conditions.push(`team_id = '${teamIds[0]}'`);
|
||||
} else {
|
||||
// 多个团队ID
|
||||
conditions.push(`team_id IN ('${teamIds.join("','")}')`);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加项目筛选
|
||||
if (projectIds && projectIds.length > 0) {
|
||||
// 如果只有一个项目ID
|
||||
if (projectIds.length === 1) {
|
||||
conditions.push(`project_id = '${projectIds[0]}'`);
|
||||
} else {
|
||||
// 多个项目ID
|
||||
conditions.push(`project_id IN ('${projectIds.join("','")}')`);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加标签筛选
|
||||
if ((tagIds && tagIds.length > 0) || (tagNames && tagNames.length > 0)) {
|
||||
// 优先使用tagNames,如果有的话
|
||||
const tagsToUse = tagNames.length > 0 ? tagNames : tagIds;
|
||||
|
||||
// 使用与buildFilter函数相同的处理方式
|
||||
const tagConditions = tagsToUse.map(tag =>
|
||||
`link_tags LIKE '%${tag}%'`
|
||||
);
|
||||
conditions.push(`(${tagConditions.join(' OR ')})`);
|
||||
}
|
||||
|
||||
if (conditions.length > 0) {
|
||||
whereClause = `WHERE ${conditions.join(' AND ')}`;
|
||||
}
|
||||
|
||||
// 确定要分组的UTM字段
|
||||
let utmField;
|
||||
switch (utmType) {
|
||||
case 'source':
|
||||
utmField = 'utm_source';
|
||||
break;
|
||||
case 'medium':
|
||||
utmField = 'utm_medium';
|
||||
break;
|
||||
case 'campaign':
|
||||
utmField = 'utm_campaign';
|
||||
break;
|
||||
case 'term':
|
||||
utmField = 'utm_term';
|
||||
break;
|
||||
case 'content':
|
||||
utmField = 'utm_content';
|
||||
break;
|
||||
default:
|
||||
utmField = 'utm_source';
|
||||
}
|
||||
|
||||
// 构建SQL查询
|
||||
const query = `
|
||||
SELECT
|
||||
${utmField} AS utm_value,
|
||||
COUNT(*) AS clicks,
|
||||
uniqExact(visitor_id) AS visitors,
|
||||
round(AVG(time_spent_sec), 2) AS avg_time_spent,
|
||||
countIf(is_bounce = 1) AS bounces,
|
||||
countIf(conversion_type IN ('visit', 'stay', 'interact', 'signup', 'subscription', 'purchase')) AS conversions
|
||||
FROM shorturl_analytics.events
|
||||
${whereClause}
|
||||
${whereClause ? 'AND' : 'WHERE'} ${utmField} != ''
|
||||
GROUP BY utm_value
|
||||
ORDER BY clicks DESC
|
||||
LIMIT 100
|
||||
`;
|
||||
|
||||
// 执行查询
|
||||
const result = await clickhouse.query({
|
||||
query,
|
||||
format: 'JSONEachRow',
|
||||
});
|
||||
|
||||
// 获取查询结果
|
||||
const rows = await result.json();
|
||||
const data = rows as UtmData[];
|
||||
|
||||
// 返回数据
|
||||
const response: ApiResponse<UtmData[]> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error fetching UTM data:', error);
|
||||
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
248
app/api/geo/batch/route.ts
Normal file
248
app/api/geo/batch/route.ts
Normal file
@@ -0,0 +1,248 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
|
||||
interface IpLocationData {
|
||||
ip: string;
|
||||
country_name: string;
|
||||
country_code: string;
|
||||
city: string;
|
||||
region: string;
|
||||
continent_code: string;
|
||||
continent_name: string;
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
// Simple in-memory cache on the server side to reduce API calls
|
||||
const serverCache: Record<string, IpLocationData> = {};
|
||||
|
||||
// Cache for IPs that have repeatedly failed to resolve
|
||||
const failedIPsCache: Record<string, { attempts: number, lastAttempt: number }> = {};
|
||||
|
||||
// Cache expiration time (30 days in milliseconds)
|
||||
const CACHE_EXPIRATION = 30 * 24 * 60 * 60 * 1000; // 30 days
|
||||
|
||||
// Max attempts to fetch an IP before considering it permanently failed
|
||||
const MAX_RETRY_ATTEMPTS = 3;
|
||||
|
||||
// Retry timeout - how long to wait before trying a failed IP again (24 hours)
|
||||
const RETRY_TIMEOUT = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Check if an IP has failed too many times and should be skipped
|
||||
*/
|
||||
function shouldSkipIP(ip: string): boolean {
|
||||
if (!failedIPsCache[ip]) return false;
|
||||
|
||||
const now = Date.now();
|
||||
|
||||
// Skip if max attempts reached
|
||||
if (failedIPsCache[ip].attempts >= MAX_RETRY_ATTEMPTS) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Skip if last attempt was recent
|
||||
if (now - failedIPsCache[ip].lastAttempt < RETRY_TIMEOUT) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark an IP as failed
|
||||
*/
|
||||
function markIPAsFailed(ip: string): void {
|
||||
const now = Date.now();
|
||||
|
||||
if (failedIPsCache[ip]) {
|
||||
failedIPsCache[ip] = {
|
||||
attempts: failedIPsCache[ip].attempts + 1,
|
||||
lastAttempt: now
|
||||
};
|
||||
} else {
|
||||
failedIPsCache[ip] = {
|
||||
attempts: 1,
|
||||
lastAttempt: now
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get location data for a single IP using ipapi.co
|
||||
*/
|
||||
async function fetchIpLocation(ip: string): Promise<IpLocationData | null> {
|
||||
try {
|
||||
// Skip this IP if it has failed too many times
|
||||
if (shouldSkipIP(ip)) {
|
||||
console.log(`[Server] Skipping blacklisted IP: ${ip}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check server cache first
|
||||
const now = Date.now();
|
||||
if (serverCache[ip] && (now - serverCache[ip].timestamp) < CACHE_EXPIRATION) {
|
||||
return serverCache[ip];
|
||||
}
|
||||
|
||||
// Add delay to avoid rate limiting (100 requests per minute max)
|
||||
await new Promise(resolve => setTimeout(resolve, 600)); // ~100 req/min = 1 req per 600ms
|
||||
|
||||
const response = await fetch(`https://ipapi.co/${ip}/json/`);
|
||||
|
||||
if (!response.ok) {
|
||||
console.error(`Error fetching location for IP ${ip}: ${response.statusText}`);
|
||||
markIPAsFailed(ip);
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.error) {
|
||||
console.error(`Error fetching location for IP ${ip}: ${data.reason}`);
|
||||
markIPAsFailed(ip);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Reset failed status if successful
|
||||
if (failedIPsCache[ip]) {
|
||||
delete failedIPsCache[ip];
|
||||
}
|
||||
|
||||
const locationData: IpLocationData = {
|
||||
ip: data.ip,
|
||||
country_name: data.country_name || 'Unknown',
|
||||
country_code: data.country_code || 'UN',
|
||||
city: data.city || 'Unknown',
|
||||
region: data.region || 'Unknown',
|
||||
continent_code: data.continent_code || 'UN',
|
||||
continent_name: getContinentName(data.continent_code) || 'Unknown',
|
||||
latitude: data.latitude || 0,
|
||||
longitude: data.longitude || 0,
|
||||
timestamp: Date.now()
|
||||
};
|
||||
|
||||
// Cache the result
|
||||
serverCache[ip] = locationData;
|
||||
|
||||
return locationData;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching location for IP ${ip}:`, error);
|
||||
markIPAsFailed(ip);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get continent name from continent code
|
||||
*/
|
||||
function getContinentName(code?: string): string {
|
||||
if (!code) return 'Unknown';
|
||||
|
||||
const continents: Record<string, string> = {
|
||||
'AF': 'Africa',
|
||||
'AN': 'Antarctica',
|
||||
'AS': 'Asia',
|
||||
'EU': 'Europe',
|
||||
'NA': 'North America',
|
||||
'OC': 'Oceania',
|
||||
'SA': 'South America'
|
||||
};
|
||||
|
||||
return continents[code] || 'Unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* API route handler for batch IP location lookups
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const { ips } = await request.json();
|
||||
|
||||
if (!ips || !Array.isArray(ips) || ips.length === 0) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Invalid or empty IP list'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// Limit batch size to 50 IPs to prevent abuse
|
||||
const ipList = ips.slice(0, 50);
|
||||
const results: Record<string, IpLocationData | null> = {};
|
||||
|
||||
// Filter out IPs that should be skipped
|
||||
const validIPs = ipList.filter(ip => {
|
||||
if (typeof ip !== 'string' || !ip.trim()) return false;
|
||||
if (isPrivateIP(ip)) {
|
||||
results[ip] = getPrivateIPData(ip);
|
||||
return false;
|
||||
}
|
||||
if (shouldSkipIP(ip)) {
|
||||
console.log(`[Server] Skipping blacklisted IP: ${ip}`);
|
||||
results[ip] = null;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
// Process remaining IPs sequentially to respect rate limits
|
||||
for (const ip of validIPs) {
|
||||
results[ip] = await fetchIpLocation(ip);
|
||||
}
|
||||
|
||||
const response: ApiResponse<Record<string, IpLocationData | null>> = {
|
||||
success: true,
|
||||
data: results
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Batch IP lookup error:', error);
|
||||
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if IP is a private/local address
|
||||
*/
|
||||
function isPrivateIP(ip: string): boolean {
|
||||
return (
|
||||
ip.startsWith('10.') ||
|
||||
ip.startsWith('192.168.') ||
|
||||
ip.startsWith('172.16.') ||
|
||||
ip.startsWith('172.17.') ||
|
||||
ip.startsWith('172.18.') ||
|
||||
ip.startsWith('172.19.') ||
|
||||
ip.startsWith('172.20.') ||
|
||||
ip.startsWith('172.21.') ||
|
||||
ip.startsWith('172.22.') ||
|
||||
ip.startsWith('127.') ||
|
||||
ip === 'localhost' ||
|
||||
ip === '::1'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate location data for private IP addresses
|
||||
*/
|
||||
function getPrivateIPData(ip: string): IpLocationData {
|
||||
return {
|
||||
ip,
|
||||
country_name: 'Local Network',
|
||||
country_code: 'LO',
|
||||
city: 'Local',
|
||||
region: 'Local',
|
||||
continent_code: 'LO',
|
||||
continent_name: 'Local',
|
||||
latitude: 0,
|
||||
longitude: 0,
|
||||
timestamp: Date.now()
|
||||
};
|
||||
}
|
||||
141
app/api/shortlinks/[id]/route.ts
Normal file
141
app/api/shortlinks/[id]/route.ts
Normal file
@@ -0,0 +1,141 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { executeQuery } from '@/lib/clickhouse';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
{ params }: { params: { id: string } }
|
||||
) {
|
||||
try {
|
||||
// Get the id from the URL parameters
|
||||
const { id } = params;
|
||||
|
||||
if (!id) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'ID parameter is required'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
console.log('Fetching shortlink by ID:', id);
|
||||
|
||||
// Query to fetch a single shortlink by id
|
||||
const query = `
|
||||
SELECT
|
||||
id,
|
||||
external_id,
|
||||
type,
|
||||
slug,
|
||||
original_url,
|
||||
title,
|
||||
description,
|
||||
attributes,
|
||||
schema_version,
|
||||
creator_id,
|
||||
creator_email,
|
||||
creator_name,
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
projects,
|
||||
teams,
|
||||
tags,
|
||||
qr_codes AS qr_codes,
|
||||
channels,
|
||||
favorites,
|
||||
expires_at,
|
||||
click_count,
|
||||
unique_visitors,
|
||||
domain
|
||||
FROM shorturl_analytics.shorturl
|
||||
WHERE id = '${id}' AND deleted_at IS NULL
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
console.log('Executing query:', query);
|
||||
|
||||
// Execute the query
|
||||
const result = await executeQuery(query);
|
||||
|
||||
// If no shortlink found with the specified ID
|
||||
if (!Array.isArray(result) || result.length === 0) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Shortlink not found'
|
||||
}, { status: 404 });
|
||||
}
|
||||
|
||||
// Process the shortlink data
|
||||
const shortlink = result[0] as any;
|
||||
|
||||
// Extract shortUrl from attributes
|
||||
let shortUrl = '';
|
||||
try {
|
||||
if (shortlink.attributes && typeof shortlink.attributes === 'string') {
|
||||
const attributes = JSON.parse(shortlink.attributes) as { shortUrl?: string };
|
||||
shortUrl = attributes.shortUrl || '';
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing shortlink attributes:', e);
|
||||
}
|
||||
|
||||
// Process teams
|
||||
let teams: any[] = [];
|
||||
try {
|
||||
if (shortlink.teams && typeof shortlink.teams === 'string') {
|
||||
teams = JSON.parse(shortlink.teams);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing teams:', e);
|
||||
}
|
||||
|
||||
// Process tags
|
||||
let tags: any[] = [];
|
||||
try {
|
||||
if (shortlink.tags && typeof shortlink.tags === 'string') {
|
||||
tags = JSON.parse(shortlink.tags);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing tags:', e);
|
||||
}
|
||||
|
||||
// Process projects
|
||||
let projects: any[] = [];
|
||||
try {
|
||||
if (shortlink.projects && typeof shortlink.projects === 'string') {
|
||||
projects = JSON.parse(shortlink.projects);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing projects:', e);
|
||||
}
|
||||
|
||||
// Format the data to match what our store expects
|
||||
const formattedShortlink = {
|
||||
id: shortlink.id || '',
|
||||
externalId: shortlink.external_id || '',
|
||||
slug: shortlink.slug || '',
|
||||
originalUrl: shortlink.original_url || '',
|
||||
title: shortlink.title || '',
|
||||
shortUrl: shortUrl,
|
||||
teams: teams,
|
||||
projects: projects,
|
||||
tags: tags.map((tag: any) => tag.tag_name || ''),
|
||||
createdAt: shortlink.created_at,
|
||||
domain: shortlink.domain || (shortUrl ? new URL(shortUrl).hostname : '')
|
||||
};
|
||||
|
||||
const response: ApiResponse<typeof formattedShortlink> = {
|
||||
success: true,
|
||||
data: formattedShortlink
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error fetching shortlink by ID:', error);
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
143
app/api/shortlinks/byUrl/route.ts
Normal file
143
app/api/shortlinks/byUrl/route.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { executeQuery } from '@/lib/clickhouse';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// Get the url from query parameters
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const url = searchParams.get('url');
|
||||
|
||||
if (!url) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'URL parameter is required'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
console.log('Fetching shortlink by URL:', url);
|
||||
|
||||
// Query to fetch a single shortlink by shortUrl in attributes
|
||||
const query = `
|
||||
SELECT
|
||||
id,
|
||||
external_id,
|
||||
type,
|
||||
slug,
|
||||
original_url,
|
||||
title,
|
||||
description,
|
||||
attributes,
|
||||
schema_version,
|
||||
creator_id,
|
||||
creator_email,
|
||||
creator_name,
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
projects,
|
||||
teams,
|
||||
tags,
|
||||
qr_codes AS qr_codes,
|
||||
channels,
|
||||
favorites,
|
||||
expires_at,
|
||||
click_count,
|
||||
unique_visitors,
|
||||
domain
|
||||
FROM shorturl_analytics.shorturl
|
||||
WHERE JSONHas(attributes, 'shortUrl')
|
||||
AND JSONExtractString(attributes, 'shortUrl') = '${url}'
|
||||
AND deleted_at IS NULL
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
console.log('Executing query:', query);
|
||||
|
||||
// Execute the query
|
||||
const result = await executeQuery(query);
|
||||
|
||||
// If no shortlink found with the specified URL
|
||||
if (!Array.isArray(result) || result.length === 0) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Shortlink not found'
|
||||
}, { status: 404 });
|
||||
}
|
||||
|
||||
// Process the shortlink data
|
||||
const shortlink = result[0];
|
||||
|
||||
// Extract shortUrl from attributes
|
||||
let shortUrl = '';
|
||||
try {
|
||||
if (shortlink.attributes && typeof shortlink.attributes === 'string') {
|
||||
const attributes = JSON.parse(shortlink.attributes);
|
||||
shortUrl = attributes.shortUrl || '';
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing shortlink attributes:', e);
|
||||
}
|
||||
|
||||
// Process teams
|
||||
let teams = [];
|
||||
try {
|
||||
if (shortlink.teams && typeof shortlink.teams === 'string') {
|
||||
teams = JSON.parse(shortlink.teams);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing teams:', e);
|
||||
}
|
||||
|
||||
// Process tags
|
||||
let tags = [];
|
||||
try {
|
||||
if (shortlink.tags && typeof shortlink.tags === 'string') {
|
||||
tags = JSON.parse(shortlink.tags);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing tags:', e);
|
||||
}
|
||||
|
||||
// Process projects
|
||||
let projects = [];
|
||||
try {
|
||||
if (shortlink.projects && typeof shortlink.projects === 'string') {
|
||||
projects = JSON.parse(shortlink.projects);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing projects:', e);
|
||||
}
|
||||
|
||||
// Format the data to match what our store expects
|
||||
const formattedShortlink = {
|
||||
id: shortlink.id || '',
|
||||
externalId: shortlink.external_id || '',
|
||||
slug: shortlink.slug || '',
|
||||
originalUrl: shortlink.original_url || '',
|
||||
title: shortlink.title || '',
|
||||
shortUrl: shortUrl,
|
||||
teams: teams,
|
||||
projects: projects,
|
||||
tags: tags.map((tag) => tag.tag_name || ''),
|
||||
createdAt: shortlink.created_at,
|
||||
domain: shortlink.domain || (shortUrl ? new URL(shortUrl).hostname : '')
|
||||
};
|
||||
|
||||
console.log('Shortlink data formatted with externalId:', shortlink.external_id, 'Final object:', formattedShortlink);
|
||||
|
||||
const response: ApiResponse<typeof formattedShortlink> = {
|
||||
success: true,
|
||||
data: formattedShortlink
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error fetching shortlink by URL:', error);
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
143
app/api/shortlinks/exact/route.ts
Normal file
143
app/api/shortlinks/exact/route.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { executeQuery } from '@/lib/clickhouse';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// Get the url from query parameters
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const shortUrl = searchParams.get('shortUrl');
|
||||
|
||||
if (!shortUrl) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'shortUrl parameter is required'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
console.log('Fetching shortlink by exact shortUrl:', shortUrl);
|
||||
|
||||
// Query to fetch a single shortlink by shortUrl in attributes
|
||||
const query = `
|
||||
SELECT
|
||||
id,
|
||||
external_id,
|
||||
type,
|
||||
slug,
|
||||
original_url,
|
||||
title,
|
||||
description,
|
||||
attributes,
|
||||
schema_version,
|
||||
creator_id,
|
||||
creator_email,
|
||||
creator_name,
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
projects,
|
||||
teams,
|
||||
tags,
|
||||
qr_codes AS qr_codes,
|
||||
channels,
|
||||
favorites,
|
||||
expires_at,
|
||||
click_count,
|
||||
unique_visitors,
|
||||
domain
|
||||
FROM shorturl_analytics.shorturl
|
||||
WHERE JSONHas(attributes, 'shortUrl')
|
||||
AND JSONExtractString(attributes, 'shortUrl') = '${shortUrl}'
|
||||
AND deleted_at IS NULL
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
console.log('Executing query:', query);
|
||||
|
||||
// Execute the query
|
||||
const result = await executeQuery(query);
|
||||
|
||||
// If no shortlink found with the specified URL
|
||||
if (!Array.isArray(result) || result.length === 0) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Shortlink not found'
|
||||
}, { status: 404 });
|
||||
}
|
||||
|
||||
// Process the shortlink data
|
||||
const shortlink = result[0] as Record<string, any>;
|
||||
|
||||
// Extract shortUrl from attributes
|
||||
let shortUrlValue = '';
|
||||
try {
|
||||
if (shortlink.attributes && typeof shortlink.attributes === 'string') {
|
||||
const attributes = JSON.parse(shortlink.attributes) as { shortUrl?: string };
|
||||
shortUrlValue = attributes.shortUrl || '';
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing shortlink attributes:', e);
|
||||
}
|
||||
|
||||
// Process teams
|
||||
let teams: any[] = [];
|
||||
try {
|
||||
if (shortlink.teams && typeof shortlink.teams === 'string') {
|
||||
teams = JSON.parse(shortlink.teams);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing teams:', e);
|
||||
}
|
||||
|
||||
// Process tags
|
||||
let tags: any[] = [];
|
||||
try {
|
||||
if (shortlink.tags && typeof shortlink.tags === 'string') {
|
||||
tags = JSON.parse(shortlink.tags);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing tags:', e);
|
||||
}
|
||||
|
||||
// Process projects
|
||||
let projects: any[] = [];
|
||||
try {
|
||||
if (shortlink.projects && typeof shortlink.projects === 'string') {
|
||||
projects = JSON.parse(shortlink.projects);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error parsing projects:', e);
|
||||
}
|
||||
|
||||
// Format the data to match what our store expects
|
||||
const formattedShortlink = {
|
||||
id: shortlink.id || '',
|
||||
externalId: shortlink.external_id || '',
|
||||
slug: shortlink.slug || '',
|
||||
originalUrl: shortlink.original_url || '',
|
||||
title: shortlink.title || '',
|
||||
shortUrl: shortUrlValue,
|
||||
teams: teams,
|
||||
projects: projects,
|
||||
tags: tags.map((tag: any) => tag.tag_name || ''),
|
||||
createdAt: shortlink.created_at,
|
||||
domain: shortlink.domain || (shortUrlValue ? new URL(shortUrlValue).hostname : '')
|
||||
};
|
||||
|
||||
console.log('Formatted shortlink with externalId:', shortlink.external_id);
|
||||
|
||||
const response: ApiResponse<typeof formattedShortlink> = {
|
||||
success: true,
|
||||
data: formattedShortlink
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error fetching shortlink by exact URL:', error);
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
104
app/api/shortlinks/route.ts
Normal file
104
app/api/shortlinks/route.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { executeQuery } from '@/lib/clickhouse';
|
||||
import { NextRequest } from 'next/server';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// Get pagination and filter parameters from the URL
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const page = parseInt(searchParams.get('page') || '1', 10);
|
||||
const pageSize = parseInt(searchParams.get('page_size') || '10', 10);
|
||||
const search = searchParams.get('search');
|
||||
const team = searchParams.get('team');
|
||||
|
||||
// Calculate OFFSET
|
||||
const offset = (page - 1) * pageSize;
|
||||
|
||||
// Build WHERE conditions
|
||||
const whereConditions = ['deleted_at IS NULL'];
|
||||
|
||||
if (search) {
|
||||
// Expand search to include more fields: slug, shortUrl in attributes, team name, tag name, original_url
|
||||
whereConditions.push(`(
|
||||
slug ILIKE '%${search}%' OR
|
||||
original_url ILIKE '%${search}%' OR
|
||||
title ILIKE '%${search}%' OR
|
||||
JSONHas(attributes, 'shortUrl') AND JSONExtractString(attributes, 'shortUrl') ILIKE '%${search}%' OR
|
||||
arrayExists(x -> JSONExtractString(x, 'team_name') ILIKE '%${search}%', JSONExtractArrayRaw(teams)) OR
|
||||
arrayExists(x -> JSONExtractString(x, 'tag_name') ILIKE '%${search}%', JSONExtractArrayRaw(tags))
|
||||
)`);
|
||||
}
|
||||
|
||||
if (team) {
|
||||
whereConditions.push(`arrayExists(x -> JSONExtractString(x, 'team_id') = '${team}', JSONExtractArrayRaw(teams))`);
|
||||
}
|
||||
|
||||
const whereClause = whereConditions.join(' AND ');
|
||||
|
||||
// First query to get total count
|
||||
const countQuery = `
|
||||
SELECT count(*) as total
|
||||
FROM shorturl_analytics.shorturl
|
||||
WHERE ${whereClause}
|
||||
`;
|
||||
|
||||
const countResult = await executeQuery(countQuery);
|
||||
// Handle the result safely by using an explicit type check
|
||||
const total = Array.isArray(countResult) && countResult.length > 0 && typeof countResult[0] === 'object' && countResult[0] !== null && 'total' in countResult[0]
|
||||
? Number(countResult[0].total)
|
||||
: 0;
|
||||
const totalPages = Math.ceil(total / pageSize);
|
||||
|
||||
// Main query with pagination
|
||||
const query = `
|
||||
SELECT
|
||||
id,
|
||||
external_id,
|
||||
type,
|
||||
slug,
|
||||
original_url,
|
||||
title,
|
||||
description,
|
||||
attributes,
|
||||
schema_version,
|
||||
creator_id,
|
||||
creator_email,
|
||||
creator_name,
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
projects,
|
||||
teams,
|
||||
tags,
|
||||
qr_codes AS qr_codes,
|
||||
channels,
|
||||
favorites,
|
||||
expires_at,
|
||||
click_count,
|
||||
unique_visitors,
|
||||
domain
|
||||
FROM shorturl_analytics.shorturl
|
||||
WHERE ${whereClause}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ${pageSize} OFFSET ${offset}
|
||||
`;
|
||||
|
||||
// Execute the query using the shared client
|
||||
const rows = await executeQuery(query);
|
||||
|
||||
// Return the data with pagination metadata
|
||||
return NextResponse.json({
|
||||
links: rows,
|
||||
total: total,
|
||||
total_pages: totalPages,
|
||||
page: page,
|
||||
page_size: pageSize
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching shortlinks from ClickHouse:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch shortlinks' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
41
app/api/teams/list/route.ts
Normal file
41
app/api/teams/list/route.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
|
||||
import { cookies } from 'next/headers';
|
||||
import { NextResponse } from 'next/server';
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const supabase = createRouteHandlerClient({ cookies });
|
||||
|
||||
// 获取当前用户
|
||||
const { data: { user }, error: userError } = await supabase.auth.getUser();
|
||||
if (userError || !user) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
}
|
||||
|
||||
// 获取用户所属的所有团队
|
||||
const { data: teams, error: teamsError } = await supabase
|
||||
.from('teams')
|
||||
.select(`
|
||||
id,
|
||||
name,
|
||||
description,
|
||||
avatar_url
|
||||
`)
|
||||
.innerJoin('team_membership', 'teams.id = team_membership.team_id')
|
||||
.eq('team_membership.user_id', user.id)
|
||||
.is('teams.deleted_at', null);
|
||||
|
||||
if (teamsError) {
|
||||
console.error('Error fetching teams:', teamsError);
|
||||
return NextResponse.json({ error: 'Failed to fetch teams' }, { status: 500 });
|
||||
}
|
||||
|
||||
return NextResponse.json(teams);
|
||||
} catch (error) {
|
||||
console.error('Error in /api/teams/list:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Internal server error' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
168
app/api/types.ts
Normal file
168
app/api/types.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
// Event Types
|
||||
export interface Event {
|
||||
// 核心事件信息
|
||||
event_id: string;
|
||||
event_time: string;
|
||||
event_type: string;
|
||||
event_attributes: string;
|
||||
|
||||
// 链接信息
|
||||
link_id: string;
|
||||
link_slug: string;
|
||||
link_label: string;
|
||||
link_title: string;
|
||||
link_original_url: string;
|
||||
link_attributes: string;
|
||||
link_created_at: string;
|
||||
link_expires_at: string | null;
|
||||
link_tags: string;
|
||||
|
||||
// 用户信息
|
||||
user_id: string;
|
||||
user_name: string;
|
||||
user_email: string;
|
||||
user_attributes: string;
|
||||
|
||||
// 团队信息
|
||||
team_id: string;
|
||||
team_name: string;
|
||||
team_attributes: string;
|
||||
|
||||
// 项目信息
|
||||
project_id: string;
|
||||
project_name: string;
|
||||
project_attributes: string;
|
||||
|
||||
// 二维码信息
|
||||
qr_code_id: string;
|
||||
qr_code_name: string;
|
||||
qr_code_attributes: string;
|
||||
|
||||
// 访问者信息
|
||||
visitor_id: string;
|
||||
session_id: string;
|
||||
ip_address: string;
|
||||
country: string;
|
||||
city: string;
|
||||
device_type: string;
|
||||
browser: string;
|
||||
os: string;
|
||||
user_agent: string;
|
||||
|
||||
// 来源信息
|
||||
referrer: string;
|
||||
utm_source: string;
|
||||
utm_medium: string;
|
||||
utm_campaign: string;
|
||||
utm_term: string;
|
||||
utm_content: string;
|
||||
|
||||
// 交互信息
|
||||
time_spent_sec: number;
|
||||
is_bounce: boolean;
|
||||
is_qr_scan: boolean;
|
||||
conversion_type: string;
|
||||
conversion_value: number;
|
||||
|
||||
// 旧接口兼容字段
|
||||
id?: string;
|
||||
time?: string;
|
||||
type?: string;
|
||||
linkInfo?: {
|
||||
id: string;
|
||||
shortUrl: string;
|
||||
originalUrl: string;
|
||||
};
|
||||
visitor?: {
|
||||
id: string;
|
||||
browser: string;
|
||||
os: string;
|
||||
device: string;
|
||||
};
|
||||
location?: {
|
||||
country: string;
|
||||
region: string;
|
||||
city: string;
|
||||
};
|
||||
}
|
||||
|
||||
// Analytics Types
|
||||
export interface TimeSeriesData {
|
||||
timestamp: string;
|
||||
events: number;
|
||||
visitors: number;
|
||||
conversions: number;
|
||||
}
|
||||
|
||||
export interface GeoData {
|
||||
location: string;
|
||||
area: string;
|
||||
visits: number;
|
||||
visitors: number;
|
||||
percentage: number;
|
||||
}
|
||||
|
||||
export type DeviceType = 'mobile' | 'desktop' | 'tablet' | 'other';
|
||||
|
||||
export interface DeviceAnalytics {
|
||||
deviceTypes: {
|
||||
type: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}[];
|
||||
browsers: {
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}[];
|
||||
operatingSystems: {
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface EventsSummary {
|
||||
totalEvents: number;
|
||||
uniqueVisitors: number;
|
||||
totalConversions: number;
|
||||
averageTimeSpent: number;
|
||||
deviceTypes: {
|
||||
mobile: number;
|
||||
desktop: number;
|
||||
tablet: number;
|
||||
other: number;
|
||||
};
|
||||
browsers: {
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}[];
|
||||
operatingSystems: {
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface ConversionStats {
|
||||
totalConversions: number;
|
||||
conversionRate: number;
|
||||
averageValue: number;
|
||||
byType: {
|
||||
type: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
value: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface EventFilters {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
eventType?: string;
|
||||
linkId?: string;
|
||||
linkSlug?: string;
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
}
|
||||
18
app/auth/callback/route.ts
Normal file
18
app/auth/callback/route.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
|
||||
import { cookies } from 'next/headers';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const requestUrl = new URL(request.url);
|
||||
const code = requestUrl.searchParams.get('code');
|
||||
|
||||
if (code) {
|
||||
const cookieStore = cookies();
|
||||
const supabase = createRouteHandlerClient({ cookies: () => cookieStore });
|
||||
await supabase.auth.exchangeCodeForSession(code);
|
||||
}
|
||||
|
||||
// URL to redirect to after sign in process completes
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || 'https://main.upj.to';
|
||||
return NextResponse.redirect(new URL('/analytics', siteUrl));
|
||||
}
|
||||
45
app/components/ClientRouteGuard.tsx
Normal file
45
app/components/ClientRouteGuard.tsx
Normal file
@@ -0,0 +1,45 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
|
||||
// 这个组件会检查 localStorage 中是否有认证令牌,如果没有则重定向到登录页面
|
||||
export default function ClientRouteGuard({ children }: { children: React.ReactNode }) {
|
||||
const router = useRouter();
|
||||
const [isAuthenticated, setIsAuthenticated] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
// 检查 localStorage 中是否有认证令牌
|
||||
const checkAuth = () => {
|
||||
// 查找 Supabase 认证令牌
|
||||
const hasAuthToken = !!localStorage.getItem('sb-mwwvqwevplndzvmqmrxa-auth-token') ||
|
||||
!!localStorage.getItem('sb-auth-token');
|
||||
|
||||
if (!hasAuthToken) {
|
||||
// 如果没有令牌,重定向到登录页面
|
||||
router.push('/login');
|
||||
} else {
|
||||
setIsAuthenticated(true);
|
||||
}
|
||||
setIsLoading(false);
|
||||
};
|
||||
|
||||
checkAuth();
|
||||
}, [router]);
|
||||
|
||||
// 显示加载状态
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen">
|
||||
<div className="text-center">
|
||||
<div className="animate-spin rounded-full h-16 w-16 border-t-2 border-b-2 border-blue-500 mx-auto"></div>
|
||||
<p className="mt-4 text-lg text-gray-700 dark:text-gray-300">加载中...</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// 只有当用户已认证时才显示子组件
|
||||
return isAuthenticated ? <>{children}</> : null;
|
||||
}
|
||||
111
app/components/Sidebar.tsx
Normal file
111
app/components/Sidebar.tsx
Normal file
@@ -0,0 +1,111 @@
|
||||
'use client';
|
||||
|
||||
import { useState } from 'react';
|
||||
import Link from 'next/link';
|
||||
import { usePathname } from 'next/navigation';
|
||||
import {
|
||||
BarChartIcon,
|
||||
HomeIcon,
|
||||
PersonIcon,
|
||||
ChevronLeftIcon,
|
||||
ChevronRightIcon
|
||||
} from '@radix-ui/react-icons';
|
||||
|
||||
interface NavItemProps {
|
||||
href: string;
|
||||
label: string;
|
||||
icon: React.ReactNode;
|
||||
isCollapsed: boolean;
|
||||
isActive?: boolean;
|
||||
}
|
||||
|
||||
const NavItem = ({ href, label, icon, isCollapsed, isActive }: NavItemProps) => {
|
||||
return (
|
||||
<Link
|
||||
href={href}
|
||||
className={`flex items-center p-2 rounded-lg ${
|
||||
isActive
|
||||
? 'bg-blue-100 text-blue-700'
|
||||
: 'text-gray-700 hover:bg-gray-100'
|
||||
} transition-all duration-200 group`}
|
||||
>
|
||||
<div className="w-6 h-6 flex items-center justify-center">{icon}</div>
|
||||
{!isCollapsed && (
|
||||
<span className={`ml-3 whitespace-nowrap transition-opacity duration-200 ${
|
||||
isCollapsed ? 'opacity-0 w-0' : 'opacity-100'
|
||||
}`}>
|
||||
{label}
|
||||
</span>
|
||||
)}
|
||||
{isCollapsed && (
|
||||
<span className="sr-only">{label}</span>
|
||||
)}
|
||||
</Link>
|
||||
);
|
||||
};
|
||||
|
||||
export function Sidebar() {
|
||||
const [isCollapsed, setIsCollapsed] = useState(false);
|
||||
const pathname = usePathname();
|
||||
|
||||
const toggleSidebar = () => {
|
||||
setIsCollapsed(!isCollapsed);
|
||||
};
|
||||
|
||||
const navigation = [
|
||||
{ name: 'Dashboard', href: '/dashboard', icon: <HomeIcon className="w-5 h-5" /> },
|
||||
{ name: 'Analytics', href: '/analytics', icon: <BarChartIcon className="w-5 h-5" /> },
|
||||
{ name: 'Account', href: '/account', icon: <PersonIcon className="w-5 h-5" /> },
|
||||
];
|
||||
|
||||
return (
|
||||
<div className={`flex flex-col h-full transition-all duration-300 ${
|
||||
isCollapsed ? 'w-16' : 'w-64'
|
||||
} bg-white border-r border-gray-200 relative`}>
|
||||
{/* 顶部Logo和标题 */}
|
||||
<div className="flex items-center p-4 border-b border-gray-200">
|
||||
<div className="flex-shrink-0 flex items-center justify-center w-8 h-8 bg-blue-500 text-white rounded">
|
||||
<span className="font-bold">S</span>
|
||||
</div>
|
||||
{!isCollapsed && (
|
||||
<span className="ml-3 font-medium text-gray-900 transition-opacity duration-200">
|
||||
ShortURL Analytics
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 导航菜单 */}
|
||||
<div className="flex-grow p-4 overflow-y-auto">
|
||||
<ul className="space-y-2">
|
||||
{navigation.map((item) => (
|
||||
<li key={item.name}>
|
||||
<NavItem
|
||||
href={item.href}
|
||||
label={item.name}
|
||||
icon={item.icon}
|
||||
isCollapsed={isCollapsed}
|
||||
isActive={pathname?.startsWith(item.href)}
|
||||
/>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
{/* 底部折叠按钮 */}
|
||||
<div className="border-t border-gray-200 p-4">
|
||||
<button
|
||||
onClick={toggleSidebar}
|
||||
className="w-full flex items-center justify-center p-2 rounded-lg text-gray-500 hover:bg-gray-100"
|
||||
aria-label={isCollapsed ? "Expand sidebar" : "Collapse sidebar"}
|
||||
>
|
||||
{isCollapsed ? (
|
||||
<ChevronRightIcon className="w-5 h-5" />
|
||||
) : (
|
||||
<ChevronLeftIcon className="w-5 h-5" />
|
||||
)}
|
||||
{!isCollapsed && <span className="ml-2">Collapse</span>}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
52
app/components/analytics/DeviceAnalytics.tsx
Normal file
52
app/components/analytics/DeviceAnalytics.tsx
Normal file
@@ -0,0 +1,52 @@
|
||||
"use client";
|
||||
|
||||
import { DeviceAnalytics as DeviceAnalyticsType } from '@/app/api/types';
|
||||
|
||||
interface CategoryItem {
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}
|
||||
|
||||
interface DeviceAnalyticsProps {
|
||||
data: DeviceAnalyticsType;
|
||||
}
|
||||
|
||||
export default function DeviceAnalytics({ data }: DeviceAnalyticsProps) {
|
||||
const renderCategory = (items: CategoryItem[], title: string) => (
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">{title}</h3>
|
||||
<div className="space-y-4">
|
||||
{items.map((item, index) => (
|
||||
<div key={index}>
|
||||
<div className="flex justify-between text-sm text-gray-600 mb-1">
|
||||
<span>{item.name}</span>
|
||||
<span>{item.percentage.toFixed(1)}% ({item.count})</span>
|
||||
</div>
|
||||
<div className="w-full bg-gray-200 rounded-full h-2">
|
||||
<div
|
||||
className="bg-blue-600 h-2 rounded-full"
|
||||
style={{ width: `${item.percentage}%` }}
|
||||
></div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
// Prepare device types data
|
||||
const deviceItems = data.deviceTypes.map(item => ({
|
||||
name: item.type || 'Unknown',
|
||||
count: item.count,
|
||||
percentage: item.percentage
|
||||
}));
|
||||
|
||||
return (
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-6">
|
||||
{renderCategory(deviceItems, 'Device Types')}
|
||||
{renderCategory(data.browsers, 'Browsers')}
|
||||
{renderCategory(data.operatingSystems, 'Operating Systems')}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
274
app/components/analytics/GeoAnalytics.tsx
Normal file
274
app/components/analytics/GeoAnalytics.tsx
Normal file
@@ -0,0 +1,274 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { GeoData } from '@/app/api/types';
|
||||
import { getLocationsFromIPs } from '@/app/utils/ipLocation';
|
||||
|
||||
interface GeoAnalyticsProps {
|
||||
data: GeoData[];
|
||||
}
|
||||
|
||||
// Interface for IP location data in our cache
|
||||
interface IpLocationDetail {
|
||||
country: string;
|
||||
city: string;
|
||||
region: string;
|
||||
continent: string;
|
||||
}
|
||||
|
||||
// Cache for IP location data
|
||||
interface LocationCache {
|
||||
[key: string]: IpLocationDetail;
|
||||
}
|
||||
|
||||
export default function GeoAnalytics({ data }: GeoAnalyticsProps) {
|
||||
const [viewMode, setViewMode] = useState<'country' | 'city' | 'region' | 'continent'>('country');
|
||||
const [locationCache, setLocationCache] = useState<LocationCache>({});
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
// Track IPs that failed to resolve
|
||||
const [failedIPs, setFailedIPs] = useState<Set<string>>(new Set());
|
||||
|
||||
// 安全地格式化数字
|
||||
const formatNumber = (value: number | undefined | null): string => {
|
||||
if (value === undefined || value === null) return '0';
|
||||
return value.toLocaleString();
|
||||
};
|
||||
|
||||
// 安全地格式化百分比
|
||||
const formatPercent = (value: number | undefined | null): string => {
|
||||
if (value === undefined || value === null) return '0';
|
||||
return value.toFixed(1);
|
||||
};
|
||||
|
||||
const sortedData = [...data].sort((a, b) => (b.visits || 0) - (a.visits || 0));
|
||||
|
||||
// Handle tab selection - only change local view mode
|
||||
const handleViewModeChange = (mode: 'country' | 'city' | 'region' | 'continent') => {
|
||||
setViewMode(mode);
|
||||
};
|
||||
|
||||
// Load location data for all IPs when the data changes
|
||||
useEffect(() => {
|
||||
const fetchLocations = async () => {
|
||||
if (sortedData.length === 0) return;
|
||||
|
||||
setIsLoading(true);
|
||||
const tempCache: LocationCache = {...locationCache};
|
||||
const tempFailedIPs = new Set(failedIPs);
|
||||
|
||||
// Get all unique IPs that aren't already in the cache and haven't failed
|
||||
const uniqueIPs = [...new Set(sortedData.map(item => item.location))].filter(ip =>
|
||||
ip &&
|
||||
ip !== 'Unknown' &&
|
||||
!tempCache[ip] &&
|
||||
!tempFailedIPs.has(ip)
|
||||
);
|
||||
|
||||
if (uniqueIPs.length === 0) {
|
||||
setIsLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Use batch lookup for better performance
|
||||
const batchResults = await getLocationsFromIPs(uniqueIPs);
|
||||
|
||||
// Convert results to our cache format
|
||||
for (const [ip, data] of Object.entries(batchResults)) {
|
||||
if (data) {
|
||||
tempCache[ip] = {
|
||||
country: data.country_name,
|
||||
city: data.city,
|
||||
region: data.region,
|
||||
continent: data.continent_name
|
||||
};
|
||||
} else {
|
||||
// Mark as failed
|
||||
tempFailedIPs.add(ip);
|
||||
}
|
||||
}
|
||||
|
||||
setLocationCache(tempCache);
|
||||
setFailedIPs(tempFailedIPs);
|
||||
} catch (error) {
|
||||
console.error('Error fetching location data:', error);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchLocations();
|
||||
}, [data]);
|
||||
|
||||
// Get the appropriate location value based on the current view mode
|
||||
const getLocationValue = (item: GeoData): string => {
|
||||
const ip = item.location || '';
|
||||
|
||||
// If there's no IP or it's "Unknown", return that value
|
||||
if (!ip || ip === 'Unknown') return 'Unknown';
|
||||
|
||||
// If IP failed to resolve, return Unknown
|
||||
if (failedIPs.has(ip)) {
|
||||
return 'Unknown';
|
||||
}
|
||||
|
||||
// Return from cache if available
|
||||
if (locationCache[ip]) {
|
||||
switch (viewMode) {
|
||||
case 'country':
|
||||
return locationCache[ip].country || 'Unknown';
|
||||
case 'city':
|
||||
return locationCache[ip].city || 'Unknown';
|
||||
case 'region':
|
||||
return locationCache[ip].region || 'Unknown';
|
||||
case 'continent':
|
||||
return locationCache[ip].continent || 'Unknown';
|
||||
default:
|
||||
return ip;
|
||||
}
|
||||
}
|
||||
|
||||
// Return placeholder if not in cache yet
|
||||
return `Loading...`;
|
||||
};
|
||||
|
||||
// Get the appropriate area value based on the current view mode
|
||||
const getAreaValue = (item: GeoData): string => {
|
||||
const ip = item.location || '';
|
||||
|
||||
// If there's no IP or it's "Unknown", return empty string
|
||||
if (!ip || ip === 'Unknown' || failedIPs.has(ip)) return '';
|
||||
|
||||
// Return from cache if available
|
||||
if (locationCache[ip]) {
|
||||
switch (viewMode) {
|
||||
case 'country':
|
||||
// For country view, show the continent as area
|
||||
return locationCache[ip].continent || '';
|
||||
case 'city':
|
||||
// For city view, show the country and region
|
||||
return `${locationCache[ip].country}, ${locationCache[ip].region}`;
|
||||
case 'region':
|
||||
// For region view, show the country
|
||||
return locationCache[ip].country || '';
|
||||
case 'continent':
|
||||
// For continent view, no additional area needed
|
||||
return '';
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Return empty if not in cache yet
|
||||
return '';
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
{/* Tabs for geographic levels */}
|
||||
<div className="flex border-b mb-6">
|
||||
<button
|
||||
onClick={() => handleViewModeChange('country')}
|
||||
className={`px-4 py-2 ${viewMode === 'country' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Countries
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleViewModeChange('city')}
|
||||
className={`px-4 py-2 ${viewMode === 'city' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Cities
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleViewModeChange('region')}
|
||||
className={`px-4 py-2 ${viewMode === 'region' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Regions
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleViewModeChange('continent')}
|
||||
className={`px-4 py-2 ${viewMode === 'continent' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Continents
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Loading indicator */}
|
||||
{isLoading && (
|
||||
<div className="flex justify-center items-center py-2 mb-4">
|
||||
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-blue-500 mr-2"></div>
|
||||
<span className="text-sm text-gray-500">Loading location data...</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Table with added area column */}
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
{viewMode === 'country' ? 'Country' :
|
||||
viewMode === 'city' ? 'City' :
|
||||
viewMode === 'region' ? 'Region' : 'Continent'}
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
{viewMode === 'country' ? 'Continent' :
|
||||
viewMode === 'city' ? 'Location' :
|
||||
viewMode === 'region' ? 'Country' : 'Area'}
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Visits
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Unique Visitors
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
% of Total
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="bg-white divide-y divide-gray-200">
|
||||
{sortedData.length > 0 ? (
|
||||
sortedData.map((item, index) => (
|
||||
<tr key={index} className={index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{getLocationValue(item)}
|
||||
{item.location && (
|
||||
<div className="text-xs text-gray-500 mt-1">{item.location}</div>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
|
||||
{getAreaValue(item)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{formatNumber(item.visits)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{formatNumber(item.visitors)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
<div className="flex items-center">
|
||||
<div className="w-24 bg-gray-200 rounded-full h-2">
|
||||
<div
|
||||
className="bg-blue-600 h-2 rounded-full"
|
||||
style={{ width: `${item.percentage || 0}%` }}
|
||||
/>
|
||||
</div>
|
||||
<span className="ml-2">{formatPercent(item.percentage)}%</span>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))
|
||||
) : (
|
||||
<tr>
|
||||
<td colSpan={5} className="px-6 py-4 text-center text-sm text-gray-500">
|
||||
No location data available
|
||||
</td>
|
||||
</tr>
|
||||
)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
162
app/components/analytics/PathAnalytics.tsx
Normal file
162
app/components/analytics/PathAnalytics.tsx
Normal file
@@ -0,0 +1,162 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
|
||||
interface PathAnalyticsProps {
|
||||
startTime: string;
|
||||
endTime: string;
|
||||
linkId?: string;
|
||||
onPathClick?: (path: string) => void;
|
||||
}
|
||||
|
||||
interface PathData {
|
||||
path: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}
|
||||
|
||||
const PathAnalytics: React.FC<PathAnalyticsProps> = ({ startTime, endTime, linkId, onPathClick }) => {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [pathData, setPathData] = useState<PathData[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!linkId) {
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const fetchPathData = async () => {
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
startTime,
|
||||
endTime,
|
||||
linkId
|
||||
});
|
||||
|
||||
const response = await fetch(`/api/events/path-analytics?${params.toString()}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch path analytics data');
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.success && result.data) {
|
||||
// 自定义处理路径数据,根据是否有子路径来分组
|
||||
const rawData = result.data;
|
||||
const pathMap = new Map<string, number>();
|
||||
let totalClicks = 0;
|
||||
|
||||
rawData.forEach((item: PathData) => {
|
||||
const urlPath = item.path.split('?')[0];
|
||||
totalClicks += item.count;
|
||||
|
||||
// 解析路径,检查是否有子路径
|
||||
const pathParts = urlPath.split('/').filter(Boolean);
|
||||
|
||||
// 基础路径(例如/5seaii)或者带有查询参数但没有子路径的路径视为同一个路径
|
||||
// 子路径(例如/5seaii/bbbbb)单独统计
|
||||
const groupKey = pathParts.length > 1 ? urlPath : `/${pathParts[0]}`;
|
||||
|
||||
const currentCount = pathMap.get(groupKey) || 0;
|
||||
pathMap.set(groupKey, currentCount + item.count);
|
||||
});
|
||||
|
||||
// 转换回数组并排序
|
||||
const groupedPathData = Array.from(pathMap.entries())
|
||||
.map(([path, count]) => ({
|
||||
path,
|
||||
count,
|
||||
percentage: totalClicks > 0 ? count / totalClicks : 0,
|
||||
}))
|
||||
.sort((a, b) => b.count - a.count);
|
||||
|
||||
setPathData(groupedPathData);
|
||||
} else {
|
||||
setError(result.error || 'Failed to load path analytics');
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'An error occurred');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchPathData();
|
||||
}, [startTime, endTime, linkId]);
|
||||
|
||||
const handlePathClick = (path: string, e: React.MouseEvent) => {
|
||||
e.preventDefault();
|
||||
console.log('====== PATH CLICK DEBUG ======');
|
||||
console.log('Path value:', path);
|
||||
console.log('Path type:', typeof path);
|
||||
console.log('Path length:', path.length);
|
||||
console.log('Path chars:', Array.from(path).map(c => c.charCodeAt(0)));
|
||||
console.log('==============================');
|
||||
if (onPathClick) {
|
||||
onPathClick(path);
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return <div className="py-8 flex justify-center">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-t-2 border-b-2 border-blue-500" />
|
||||
</div>;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return <div className="py-4 text-red-500">{error}</div>;
|
||||
}
|
||||
|
||||
if (!linkId) {
|
||||
return <div className="py-4 text-gray-500">Select a specific link to view path analytics.</div>;
|
||||
}
|
||||
|
||||
if (pathData.length === 0) {
|
||||
return <div className="py-4 text-gray-500">No path data available for this link.</div>;
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="text-sm text-gray-500 mb-4">
|
||||
Note: Paths are grouped by subpath. URLs with different query parameters but the same base path (without subpath) are grouped together.
|
||||
</div>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead>
|
||||
<tr>
|
||||
<th className="px-6 py-3 bg-gray-50 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Path</th>
|
||||
<th className="px-6 py-3 bg-gray-50 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">Clicks</th>
|
||||
<th className="px-6 py-3 bg-gray-50 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">Percentage</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="bg-white divide-y divide-gray-200">
|
||||
{pathData.map((item, index) => (
|
||||
<tr key={index} className={index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm font-medium text-gray-900">
|
||||
<a
|
||||
href="#"
|
||||
className="hover:text-blue-600 hover:underline cursor-pointer"
|
||||
onClick={(e) => handlePathClick(item.path, e)}
|
||||
>
|
||||
{item.path}
|
||||
</a>
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500 text-right">{item.count}</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-right">
|
||||
<div className="flex items-center justify-end">
|
||||
<span className="text-sm text-gray-500 mr-2">{(item.percentage * 100).toFixed(1)}%</span>
|
||||
<div className="w-32 bg-gray-200 rounded-full h-2.5">
|
||||
<div className="bg-blue-600 h-2.5 rounded-full" style={{ width: `${item.percentage * 100}%` }}></div>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default PathAnalytics;
|
||||
205
app/components/analytics/UtmAnalytics.tsx
Normal file
205
app/components/analytics/UtmAnalytics.tsx
Normal file
@@ -0,0 +1,205 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
interface UtmData {
|
||||
utm_value: string;
|
||||
clicks: number;
|
||||
visitors: number;
|
||||
avg_time_spent: number;
|
||||
bounces: number;
|
||||
conversions: number;
|
||||
}
|
||||
|
||||
interface UtmAnalyticsProps {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
linkId?: string;
|
||||
teamIds?: string[];
|
||||
projectIds?: string[];
|
||||
tagIds?: string[];
|
||||
subpath?: string;
|
||||
}
|
||||
|
||||
export default function UtmAnalytics({ startTime, endTime, linkId, teamIds, projectIds, tagIds, subpath }: UtmAnalyticsProps) {
|
||||
const [activeTab, setActiveTab] = useState<string>('source');
|
||||
const [utmData, setUtmData] = useState<UtmData[]>([]);
|
||||
const [isLoading, setIsLoading] = useState<boolean>(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// 加载UTM数据
|
||||
useEffect(() => {
|
||||
const fetchUtmData = async () => {
|
||||
setIsLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// 构建URL参数
|
||||
const params = new URLSearchParams();
|
||||
if (startTime) params.append('startTime', startTime);
|
||||
if (endTime) params.append('endTime', endTime);
|
||||
if (linkId) params.append('linkId', linkId);
|
||||
if (subpath) params.append('subpath', subpath);
|
||||
params.append('utmType', activeTab);
|
||||
|
||||
// 添加团队ID参数
|
||||
if (teamIds && teamIds.length > 0) {
|
||||
teamIds.forEach(id => params.append('teamId', id));
|
||||
}
|
||||
|
||||
// 添加项目ID参数
|
||||
if (projectIds && projectIds.length > 0) {
|
||||
projectIds.forEach(id => params.append('projectId', id));
|
||||
}
|
||||
|
||||
// 添加标签名称参数
|
||||
if (tagIds && tagIds.length > 0) {
|
||||
tagIds.forEach(tagName => params.append('tagName', tagName));
|
||||
}
|
||||
|
||||
// 发送请求
|
||||
const response = await fetch(`/api/events/utm?${params}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch UTM data');
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.success) {
|
||||
setUtmData(result.data || []);
|
||||
} else {
|
||||
throw new Error(result.error || 'Failed to fetch UTM data');
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error occurred');
|
||||
console.error('Error fetching UTM data:', err);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchUtmData();
|
||||
}, [activeTab, startTime, endTime, linkId, teamIds, projectIds, tagIds, subpath]);
|
||||
|
||||
// 安全地格式化数字
|
||||
const formatNumber = (value: number | undefined | null): string => {
|
||||
if (value === undefined || value === null) return '0';
|
||||
return value.toLocaleString();
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="bg-white rounded-lg shadow p-6 mb-8">
|
||||
<h2 className="text-lg font-semibold text-gray-900 mb-4">UTM Parameters</h2>
|
||||
|
||||
<div className="mb-4 border-b">
|
||||
<div className="flex">
|
||||
<button
|
||||
onClick={() => setActiveTab('source')}
|
||||
className={`px-4 py-2 ${activeTab === 'source' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Source
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setActiveTab('medium')}
|
||||
className={`px-4 py-2 ${activeTab === 'medium' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Medium
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setActiveTab('campaign')}
|
||||
className={`px-4 py-2 ${activeTab === 'campaign' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Campaign
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setActiveTab('term')}
|
||||
className={`px-4 py-2 ${activeTab === 'term' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Term
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setActiveTab('content')}
|
||||
className={`px-4 py-2 ${activeTab === 'content' ? 'border-b-2 border-blue-500 text-blue-600' : 'text-gray-500'}`}
|
||||
>
|
||||
Content
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isLoading ? (
|
||||
<div className="flex justify-center items-center py-8">
|
||||
<div className="animate-spin rounded-full h-6 w-6 border-b-2 border-blue-500"></div>
|
||||
<span className="ml-2 text-gray-500">Loading...</span>
|
||||
</div>
|
||||
) : error ? (
|
||||
<div className="text-red-500 text-center py-8">
|
||||
Error: {error}
|
||||
</div>
|
||||
) : utmData.length === 0 ? (
|
||||
<div className="text-gray-500 text-center py-8">
|
||||
No data available
|
||||
</div>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
{activeTab === 'source' ? 'Source' :
|
||||
activeTab === 'medium' ? 'Medium' :
|
||||
activeTab === 'campaign' ? 'Campaign' :
|
||||
activeTab === 'term' ? 'Term' : 'Content'}
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Clicks
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Visitors
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Avg. Time
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Bounce Rate
|
||||
</th>
|
||||
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Conversions
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="bg-white divide-y divide-gray-200">
|
||||
{utmData.map((item, index) => {
|
||||
const bounceRate = item.clicks > 0 ? (item.bounces / item.clicks) * 100 : 0;
|
||||
const conversionRate = item.clicks > 0 ? (item.conversions / item.clicks) * 100 : 0;
|
||||
|
||||
return (
|
||||
<tr key={index} className={index % 2 === 0 ? 'bg-white' : 'bg-gray-50'}>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{item.utm_value || 'Unknown'}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{formatNumber(item.clicks)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{formatNumber(item.visitors)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{item.avg_time_spent.toFixed(1)}s
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{bounceRate.toFixed(1)}%
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
|
||||
{formatNumber(item.conversions)} ({conversionRate.toFixed(1)}%)
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
211
app/components/charts/DevicePieCharts.tsx
Normal file
211
app/components/charts/DevicePieCharts.tsx
Normal file
@@ -0,0 +1,211 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { DeviceAnalytics } from '@/app/api/types';
|
||||
import { Chart, PieController, ArcElement, Tooltip, Legend, CategoryScale, LinearScale } from 'chart.js';
|
||||
|
||||
// 注册Chart.js组件
|
||||
Chart.register(PieController, ArcElement, Tooltip, Legend, CategoryScale, LinearScale);
|
||||
|
||||
interface DevicePieChartsProps {
|
||||
data: DeviceAnalytics;
|
||||
}
|
||||
|
||||
// 颜色配置
|
||||
const COLORS = {
|
||||
deviceTypes: ['rgba(59, 130, 246, 0.8)', 'rgba(96, 165, 250, 0.8)', 'rgba(147, 197, 253, 0.8)', 'rgba(191, 219, 254, 0.8)', 'rgba(219, 234, 254, 0.8)'],
|
||||
browsers: ['rgba(16, 185, 129, 0.8)', 'rgba(52, 211, 153, 0.8)', 'rgba(110, 231, 183, 0.8)', 'rgba(167, 243, 208, 0.8)', 'rgba(209, 250, 229, 0.8)'],
|
||||
os: ['rgba(239, 68, 68, 0.8)', 'rgba(248, 113, 113, 0.8)', 'rgba(252, 165, 165, 0.8)', 'rgba(254, 202, 202, 0.8)', 'rgba(254, 226, 226, 0.8)']
|
||||
};
|
||||
|
||||
export default function DevicePieCharts({ data }: DevicePieChartsProps) {
|
||||
// 创建图表引用
|
||||
const deviceTypesChartRef = useRef<HTMLCanvasElement>(null);
|
||||
const browsersChartRef = useRef<HTMLCanvasElement>(null);
|
||||
const osChartRef = useRef<HTMLCanvasElement>(null);
|
||||
|
||||
// 图表实例引用
|
||||
const deviceTypesChartInstance = useRef<Chart | null>(null);
|
||||
const browsersChartInstance = useRef<Chart | null>(null);
|
||||
const osChartInstance = useRef<Chart | null>(null);
|
||||
|
||||
// 初始化和更新图表
|
||||
useEffect(() => {
|
||||
if (!data) return;
|
||||
|
||||
// 销毁旧的图表实例
|
||||
if (deviceTypesChartInstance.current) {
|
||||
deviceTypesChartInstance.current.destroy();
|
||||
}
|
||||
if (browsersChartInstance.current) {
|
||||
browsersChartInstance.current.destroy();
|
||||
}
|
||||
if (osChartInstance.current) {
|
||||
osChartInstance.current.destroy();
|
||||
}
|
||||
|
||||
// 创建设备类型图表
|
||||
if (deviceTypesChartRef.current && data.deviceTypes.length > 0) {
|
||||
const ctx = deviceTypesChartRef.current.getContext('2d');
|
||||
if (ctx) {
|
||||
deviceTypesChartInstance.current = new Chart(ctx, {
|
||||
type: 'pie',
|
||||
data: {
|
||||
labels: data.deviceTypes.map(item => item.type),
|
||||
datasets: [{
|
||||
data: data.deviceTypes.map(item => item.count),
|
||||
backgroundColor: COLORS.deviceTypes,
|
||||
borderColor: COLORS.deviceTypes.map(color => color.replace('0.8', '1')),
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom',
|
||||
labels: {
|
||||
color: 'currentColor'
|
||||
}
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
const label = context.label || '';
|
||||
const value = context.raw as number;
|
||||
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
|
||||
const percentage = Math.round((value * 100) / total);
|
||||
return `${label}: ${value} (${percentage}%)`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 创建浏览器图表
|
||||
if (browsersChartRef.current && data.browsers.length > 0) {
|
||||
const ctx = browsersChartRef.current.getContext('2d');
|
||||
if (ctx) {
|
||||
browsersChartInstance.current = new Chart(ctx, {
|
||||
type: 'pie',
|
||||
data: {
|
||||
labels: data.browsers.map(item => item.name),
|
||||
datasets: [{
|
||||
data: data.browsers.map(item => item.count),
|
||||
backgroundColor: COLORS.browsers,
|
||||
borderColor: COLORS.browsers.map(color => color.replace('0.8', '1')),
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom',
|
||||
labels: {
|
||||
color: 'currentColor'
|
||||
}
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
const label = context.label || '';
|
||||
const value = context.raw as number;
|
||||
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
|
||||
const percentage = Math.round((value * 100) / total);
|
||||
return `${label}: ${value} (${percentage}%)`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 创建操作系统图表
|
||||
if (osChartRef.current && data.operatingSystems.length > 0) {
|
||||
const ctx = osChartRef.current.getContext('2d');
|
||||
if (ctx) {
|
||||
osChartInstance.current = new Chart(ctx, {
|
||||
type: 'pie',
|
||||
data: {
|
||||
labels: data.operatingSystems.map(item => item.name),
|
||||
datasets: [{
|
||||
data: data.operatingSystems.map(item => item.count),
|
||||
backgroundColor: COLORS.os,
|
||||
borderColor: COLORS.os.map(color => color.replace('0.8', '1')),
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom',
|
||||
labels: {
|
||||
color: 'currentColor'
|
||||
}
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
const label = context.label || '';
|
||||
const value = context.raw as number;
|
||||
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
|
||||
const percentage = Math.round((value * 100) / total);
|
||||
return `${label}: ${value} (${percentage}%)`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 清理函数
|
||||
return () => {
|
||||
if (deviceTypesChartInstance.current) {
|
||||
deviceTypesChartInstance.current.destroy();
|
||||
}
|
||||
if (browsersChartInstance.current) {
|
||||
browsersChartInstance.current.destroy();
|
||||
}
|
||||
if (osChartInstance.current) {
|
||||
osChartInstance.current.destroy();
|
||||
}
|
||||
};
|
||||
}, [data]);
|
||||
|
||||
return (
|
||||
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
|
||||
{/* 设备类型 */}
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">Device Types</h3>
|
||||
<div className="h-64">
|
||||
<canvas ref={deviceTypesChartRef} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 浏览器 */}
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">Browsers</h3>
|
||||
<div className="h-64">
|
||||
<canvas ref={browsersChartRef} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 操作系统 */}
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">Operating Systems</h3>
|
||||
<div className="h-64">
|
||||
<canvas ref={osChartRef} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
189
app/components/charts/TimeSeriesChart.tsx
Normal file
189
app/components/charts/TimeSeriesChart.tsx
Normal file
@@ -0,0 +1,189 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useRef } from 'react';
|
||||
import {
|
||||
Chart as ChartJS,
|
||||
CategoryScale,
|
||||
LinearScale,
|
||||
PointElement,
|
||||
LineElement,
|
||||
LineController,
|
||||
Title,
|
||||
Tooltip,
|
||||
Legend,
|
||||
Filler,
|
||||
ChartData,
|
||||
ChartOptions,
|
||||
TooltipItem
|
||||
} from 'chart.js';
|
||||
import { TimeSeriesData } from '@/app/api/types';
|
||||
|
||||
// 注册 Chart.js 组件
|
||||
ChartJS.register(
|
||||
CategoryScale,
|
||||
LinearScale,
|
||||
PointElement,
|
||||
LineElement,
|
||||
LineController,
|
||||
Title,
|
||||
Tooltip,
|
||||
Legend,
|
||||
Filler
|
||||
);
|
||||
|
||||
interface TimeSeriesChartProps {
|
||||
data: TimeSeriesData[];
|
||||
}
|
||||
|
||||
export default function TimeSeriesChart({ data }: TimeSeriesChartProps) {
|
||||
const chartRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const chartInstance = useRef<ChartJS | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!chartRef.current) return;
|
||||
|
||||
// 销毁旧的图表实例
|
||||
if (chartInstance.current) {
|
||||
chartInstance.current.destroy();
|
||||
}
|
||||
|
||||
const ctx = chartRef.current.getContext('2d');
|
||||
if (!ctx) return;
|
||||
|
||||
// 准备数据
|
||||
const labels = data.map(item => {
|
||||
if (!item || !item.timestamp) return '';
|
||||
const date = new Date(item.timestamp);
|
||||
return date.toLocaleDateString();
|
||||
});
|
||||
|
||||
const eventsData = data.map(item => {
|
||||
if (!item || item.events === undefined || item.events === null) return 0;
|
||||
return Number(item.events);
|
||||
});
|
||||
|
||||
const visitorsData = data.map(item => {
|
||||
if (!item || item.visitors === undefined || item.visitors === null) return 0;
|
||||
return Number(item.visitors);
|
||||
});
|
||||
|
||||
const conversionsData = data.map(item => {
|
||||
if (!item || item.conversions === undefined || item.conversions === null) return 0;
|
||||
return Number(item.conversions);
|
||||
});
|
||||
|
||||
// 创建新的图表实例
|
||||
chartInstance.current = new ChartJS(ctx, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels,
|
||||
datasets: [
|
||||
{
|
||||
label: 'Events',
|
||||
data: eventsData,
|
||||
borderColor: 'rgb(59, 130, 246)', // blue-500
|
||||
backgroundColor: 'rgba(59, 130, 246, 0.1)',
|
||||
tension: 0.4,
|
||||
fill: true
|
||||
},
|
||||
{
|
||||
label: 'Visitors',
|
||||
data: visitorsData,
|
||||
borderColor: 'rgb(16, 185, 129)', // green-500
|
||||
backgroundColor: 'rgba(16, 185, 129, 0.1)',
|
||||
tension: 0.4,
|
||||
fill: true
|
||||
},
|
||||
{
|
||||
label: 'Conversions',
|
||||
data: conversionsData,
|
||||
borderColor: 'rgb(239, 68, 68)', // red-500
|
||||
backgroundColor: 'rgba(239, 68, 68, 0.1)',
|
||||
tension: 0.4,
|
||||
fill: true
|
||||
}
|
||||
]
|
||||
} as ChartData<'line'>,
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
interaction: {
|
||||
mode: 'index',
|
||||
intersect: false
|
||||
},
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'top',
|
||||
labels: {
|
||||
usePointStyle: true,
|
||||
padding: 20,
|
||||
color: 'rgb(156, 163, 175)' // gray-400
|
||||
}
|
||||
},
|
||||
tooltip: {
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
backgroundColor: 'rgb(31, 41, 55)', // gray-800
|
||||
titleColor: 'rgb(229, 231, 235)', // gray-200
|
||||
bodyColor: 'rgb(229, 231, 235)', // gray-200
|
||||
borderColor: 'rgb(75, 85, 99)', // gray-600
|
||||
borderWidth: 1,
|
||||
padding: 12,
|
||||
displayColors: true,
|
||||
callbacks: {
|
||||
title: (items: TooltipItem<'line'>[]) => {
|
||||
if (items.length > 0) {
|
||||
const date = new Date(data[items[0].dataIndex].timestamp);
|
||||
return date.toLocaleDateString();
|
||||
}
|
||||
return '';
|
||||
},
|
||||
label: (context) => {
|
||||
const label = context.dataset.label || '';
|
||||
const value = context.parsed.y;
|
||||
return `${label}: ${Math.round(value)}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
scales: {
|
||||
x: {
|
||||
grid: {
|
||||
display: false
|
||||
},
|
||||
ticks: {
|
||||
color: 'rgb(156, 163, 175)' // gray-400
|
||||
}
|
||||
},
|
||||
y: {
|
||||
beginAtZero: true,
|
||||
grid: {
|
||||
color: 'rgb(75, 85, 99, 0.1)' // gray-600 with opacity
|
||||
},
|
||||
ticks: {
|
||||
color: 'rgb(156, 163, 175)', // gray-400
|
||||
callback: (value: number) => {
|
||||
if (!value && value !== 0) return '';
|
||||
if (value >= 1000) {
|
||||
return `${Math.round(value / 1000)}k`;
|
||||
}
|
||||
return Math.round(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} as ChartOptions<'line'>
|
||||
});
|
||||
|
||||
// 清理函数
|
||||
return () => {
|
||||
if (chartInstance.current) {
|
||||
chartInstance.current.destroy();
|
||||
}
|
||||
};
|
||||
}, [data]);
|
||||
|
||||
return (
|
||||
<canvas ref={chartRef} />
|
||||
);
|
||||
}
|
||||
100
app/components/ipLocationTest.tsx
Normal file
100
app/components/ipLocationTest.tsx
Normal file
@@ -0,0 +1,100 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { getLocationFromIP } from '@/app/utils/ipLocation';
|
||||
|
||||
interface LocationData {
|
||||
ip: string;
|
||||
country_name: string;
|
||||
country_code: string;
|
||||
city: string;
|
||||
region: string;
|
||||
continent_code: string;
|
||||
continent_name: string;
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
}
|
||||
|
||||
export default function IpLocationTest() {
|
||||
const [locationData, setLocationData] = useState<LocationData | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const testIp = "120.244.39.90";
|
||||
|
||||
useEffect(() => {
|
||||
async function fetchLocation() {
|
||||
try {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
const data = await getLocationFromIP(testIp);
|
||||
setLocationData(data);
|
||||
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Unknown error occurred');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
fetchLocation();
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="p-4 bg-white rounded-lg shadow">
|
||||
<h2 className="text-lg font-semibold mb-4">IP Location Test: {testIp}</h2>
|
||||
|
||||
{loading && (
|
||||
<div className="flex items-center text-gray-500">
|
||||
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-blue-500 mr-2"></div>
|
||||
Loading location data...
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<div className="text-red-500">
|
||||
Error: {error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loading && locationData && (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h3 className="font-medium">Location Data:</h3>
|
||||
<pre className="mt-2 p-4 bg-gray-100 rounded overflow-auto">
|
||||
{JSON.stringify(locationData, null, 2)}
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="border p-3 rounded">
|
||||
<h4 className="font-medium">Country</h4>
|
||||
<div>{locationData.country_name} ({locationData.country_code})</div>
|
||||
</div>
|
||||
|
||||
<div className="border p-3 rounded">
|
||||
<h4 className="font-medium">City</h4>
|
||||
<div>{locationData.city || 'N/A'}</div>
|
||||
</div>
|
||||
|
||||
<div className="border p-3 rounded">
|
||||
<h4 className="font-medium">Region</h4>
|
||||
<div>{locationData.region || 'N/A'}</div>
|
||||
</div>
|
||||
|
||||
<div className="border p-3 rounded">
|
||||
<h4 className="font-medium">Continent</h4>
|
||||
<div>{locationData.continent_name} ({locationData.continent_code})</div>
|
||||
</div>
|
||||
|
||||
<div className="border p-3 rounded col-span-2">
|
||||
<h4 className="font-medium">Coordinates</h4>
|
||||
<div>Latitude: {locationData.latitude}, Longitude: {locationData.longitude}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
73
app/components/layout/Header.tsx
Normal file
73
app/components/layout/Header.tsx
Normal file
@@ -0,0 +1,73 @@
|
||||
'use client';
|
||||
|
||||
import Link from 'next/link';
|
||||
import { useAuth } from '@/lib/auth';
|
||||
|
||||
export default function Header() {
|
||||
const { user, signOut } = useAuth();
|
||||
|
||||
const handleLogout = async () => {
|
||||
await signOut();
|
||||
};
|
||||
|
||||
return (
|
||||
<header className="w-full py-4 border-b border-gray-200 bg-white">
|
||||
<div className="container flex items-center justify-between px-4 mx-auto">
|
||||
<div className="flex items-center space-x-4">
|
||||
<Link href="/analytics" className="flex items-center space-x-2">
|
||||
<svg
|
||||
className="w-6 h-6 text-blue-500"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
|
||||
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
|
||||
</svg>
|
||||
<span className="text-xl font-bold text-gray-900">ShortURL Analytics</span>
|
||||
</Link>
|
||||
|
||||
{user && (
|
||||
<nav className="ml-6">
|
||||
<ul className="flex space-x-4">
|
||||
<li>
|
||||
<Link href="/analytics" className="text-sm text-gray-700 hover:text-blue-500">
|
||||
Analytics
|
||||
</Link>
|
||||
</li>
|
||||
<li>
|
||||
<Link href="/links" className="text-sm text-gray-700 hover:text-blue-500">
|
||||
Short Links
|
||||
</Link>
|
||||
</li>
|
||||
<li>
|
||||
<Link href="/create-shorturl" className="text-sm text-gray-700 hover:text-blue-500">
|
||||
Create Short URL
|
||||
</Link>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{user && (
|
||||
<div className="flex items-center space-x-4">
|
||||
<div className="text-sm text-gray-700">
|
||||
{user.email}
|
||||
</div>
|
||||
<button
|
||||
onClick={handleLogout}
|
||||
className="px-4 py-2 text-sm text-white bg-blue-500 rounded hover:bg-blue-600"
|
||||
>
|
||||
Logout
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
'use client';
|
||||
|
||||
import Link from 'next/link';
|
||||
import ThemeToggle from "../ui/ThemeToggle";
|
||||
|
||||
export default function Navbar() {
|
||||
return (
|
||||
<header className="w-full py-4 border-b border-card-border bg-background">
|
||||
<div className="container flex items-center justify-between px-4 mx-auto">
|
||||
<div className="flex items-center space-x-4">
|
||||
<Link href="/" className="flex items-center space-x-2">
|
||||
<svg
|
||||
className="w-6 h-6 text-accent-blue"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
|
||||
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
|
||||
</svg>
|
||||
<span className="text-xl font-bold text-foreground">ShortURL</span>
|
||||
</Link>
|
||||
<nav className="hidden space-x-4 md:flex">
|
||||
<Link
|
||||
href="/links"
|
||||
className="text-sm text-foreground hover:text-accent-blue transition-colors"
|
||||
>
|
||||
Links
|
||||
</Link>
|
||||
<Link
|
||||
href="/analytics"
|
||||
className="text-sm text-foreground hover:text-accent-blue transition-colors"
|
||||
>
|
||||
Analytics
|
||||
</Link>
|
||||
</nav>
|
||||
</div>
|
||||
<div className="flex items-center space-x-3">
|
||||
<ThemeToggle />
|
||||
<button className="p-2 text-sm text-foreground rounded-md gradient-border">
|
||||
Upgrade
|
||||
</button>
|
||||
<button className="p-2 text-sm text-foreground hover:text-accent-blue">
|
||||
<svg
|
||||
className="w-5 h-5"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<circle cx="12" cy="12" r="10"></circle>
|
||||
<circle cx="12" cy="10" r="3"></circle>
|
||||
<path d="M7 20.662V19a2 2 0 0 1 2-2h6a2 2 0 0 1 2 2v1.662"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
84
app/components/ui/DateRangePicker.tsx
Normal file
84
app/components/ui/DateRangePicker.tsx
Normal file
@@ -0,0 +1,84 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from 'react';
|
||||
import { format } from 'date-fns';
|
||||
|
||||
interface DateRange {
|
||||
from: Date;
|
||||
to: Date;
|
||||
}
|
||||
|
||||
interface DateRangePickerProps {
|
||||
value: DateRange;
|
||||
onChange: (value: DateRange) => void;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function DateRangePicker({
|
||||
value,
|
||||
onChange,
|
||||
className
|
||||
}: DateRangePickerProps) {
|
||||
// Internal date state for validation
|
||||
const [from, setFrom] = useState<string>(
|
||||
value.from ? format(value.from, 'yyyy-MM-dd') : ''
|
||||
);
|
||||
const [to, setTo] = useState<string>(
|
||||
value.to ? format(value.to, 'yyyy-MM-dd') : ''
|
||||
);
|
||||
|
||||
const handleFromChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const newFrom = e.target.value;
|
||||
setFrom(newFrom);
|
||||
|
||||
if (newFrom) {
|
||||
onChange({
|
||||
from: new Date(newFrom),
|
||||
to: value.to
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleToChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const newTo = e.target.value;
|
||||
setTo(newTo);
|
||||
|
||||
if (newTo) {
|
||||
onChange({
|
||||
from: value.from,
|
||||
to: new Date(newTo)
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={`flex flex-col space-y-2 sm:flex-row sm:space-y-0 sm:space-x-4 ${className}`}>
|
||||
<div>
|
||||
<label htmlFor="from" className="block text-sm font-medium text-gray-500 mb-1">
|
||||
Start Date
|
||||
</label>
|
||||
<input
|
||||
type="date"
|
||||
id="from"
|
||||
value={from}
|
||||
onChange={handleFromChange}
|
||||
max={to}
|
||||
className="block w-full px-3 py-2 bg-white border border-gray-300 rounded-md text-sm text-gray-900 focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="to" className="block text-sm font-medium text-gray-500 mb-1">
|
||||
End Date
|
||||
</label>
|
||||
<input
|
||||
type="date"
|
||||
id="to"
|
||||
value={to}
|
||||
onChange={handleToChange}
|
||||
min={from}
|
||||
className="block w-full px-3 py-2 bg-white border border-gray-300 rounded-md text-sm text-gray-900 focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
363
app/components/ui/ProjectSelector.tsx
Normal file
363
app/components/ui/ProjectSelector.tsx
Normal file
@@ -0,0 +1,363 @@
|
||||
"use client";
|
||||
|
||||
import * as React from 'react';
|
||||
import { useEffect, useState, useRef } from 'react';
|
||||
import { getSupabaseClient } from '../../utils/supabase';
|
||||
import { AuthChangeEvent, Session } from '@supabase/supabase-js';
|
||||
import { Loader2, X, Check } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
// Define our own Project type
|
||||
interface Project {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string | null;
|
||||
attributes?: Record<string, unknown>;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
deleted_at?: string | null;
|
||||
schema_version?: number | null;
|
||||
creator_id?: string | null;
|
||||
team_name?: string;
|
||||
}
|
||||
|
||||
// 添加需要的类型定义
|
||||
interface ProjectWithTeam {
|
||||
project_id: string;
|
||||
projects: Project;
|
||||
teams?: { name: string };
|
||||
}
|
||||
|
||||
// ProjectSelector component with multi-select support
|
||||
export function ProjectSelector({
|
||||
value,
|
||||
onChange,
|
||||
className,
|
||||
multiple = false,
|
||||
teamId,
|
||||
teamIds,
|
||||
}: {
|
||||
value?: string | string[];
|
||||
onChange?: (projectId: string | string[]) => void;
|
||||
className?: string;
|
||||
multiple?: boolean;
|
||||
teamId?: string; // Optional team ID to filter projects by team
|
||||
teamIds?: string[]; // Optional array of team IDs to filter projects by multiple teams
|
||||
}) {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [projects, setProjects] = useState<Project[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<string[]>([]);
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const selectorRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Normalize team IDs to ensure we're always working with an array
|
||||
const effectiveTeamIds = React.useMemo(() => {
|
||||
if (teamIds && teamIds.length > 0) {
|
||||
return teamIds;
|
||||
} else if (teamId) {
|
||||
return [teamId];
|
||||
}
|
||||
return undefined;
|
||||
}, [teamId, teamIds]);
|
||||
|
||||
// Initialize selected projects based on value prop
|
||||
useEffect(() => {
|
||||
if (value) {
|
||||
if (Array.isArray(value)) {
|
||||
setSelectedIds(value);
|
||||
} else {
|
||||
setSelectedIds(value ? [value] : []);
|
||||
}
|
||||
} else {
|
||||
setSelectedIds([]);
|
||||
}
|
||||
}, [value]);
|
||||
|
||||
// Add click outside listener to close dropdown
|
||||
useEffect(() => {
|
||||
function handleClickOutside(event: MouseEvent) {
|
||||
if (selectorRef.current && !selectorRef.current.contains(event.target as Node)) {
|
||||
setIsOpen(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Only add the event listener if the dropdown is open
|
||||
if (isOpen) {
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
return () => {
|
||||
document.removeEventListener('mousedown', handleClickOutside);
|
||||
};
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
let isMounted = true;
|
||||
|
||||
const fetchProjects = async (userId: string) => {
|
||||
if (!isMounted) return;
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
if (effectiveTeamIds && effectiveTeamIds.length > 0) {
|
||||
// If team IDs are provided, get projects for those teams
|
||||
const { data: projectsData, error: projectsError } = await supabase
|
||||
.from('team_projects')
|
||||
.select('project_id, projects:project_id(*), teams:team_id(name)')
|
||||
.in('team_id', effectiveTeamIds)
|
||||
.is('projects.deleted_at', null);
|
||||
|
||||
if (projectsError) throw projectsError;
|
||||
|
||||
if (!projectsData || projectsData.length === 0) {
|
||||
if (isMounted) setProjects([]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract projects from response with team info
|
||||
if (isMounted) {
|
||||
const projectList: Project[] = [];
|
||||
|
||||
for (const item of projectsData as ProjectWithTeam[]) {
|
||||
if (item.projects && typeof item.projects === 'object' && 'id' in item.projects && 'name' in item.projects) {
|
||||
const project = item.projects as Project;
|
||||
if (item.teams && 'name' in item.teams) {
|
||||
project.team_name = item.teams.name;
|
||||
}
|
||||
// Avoid duplicate projects from different teams
|
||||
if (!projectList.some(p => p.id === project.id)) {
|
||||
projectList.push(project);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setProjects(projectList);
|
||||
}
|
||||
} else {
|
||||
// If no team IDs, get all user's projects
|
||||
const { data: projectsData, error: projectsError } = await supabase
|
||||
.from('user_projects')
|
||||
.select('project_id, projects:project_id(*)')
|
||||
.eq('user_id', userId)
|
||||
.is('projects.deleted_at', null);
|
||||
|
||||
if (projectsError) throw projectsError;
|
||||
|
||||
if (!projectsData || projectsData.length === 0) {
|
||||
if (isMounted) setProjects([]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch team info for these projects
|
||||
const projectIds = projectsData.map(item => item.project_id);
|
||||
|
||||
// Get team info for each project
|
||||
const { data: teamProjectsData, error: teamProjectsError } = await supabase
|
||||
.from('team_projects')
|
||||
.select('project_id, teams:team_id(name)')
|
||||
.in('project_id', projectIds);
|
||||
|
||||
if (teamProjectsError) throw teamProjectsError;
|
||||
|
||||
// Create project ID to team name mapping
|
||||
const projectTeamMap: Record<string, string> = {};
|
||||
if (teamProjectsData) {
|
||||
teamProjectsData.forEach(item => {
|
||||
if (item.teams && typeof item.teams === 'object' && 'name' in item.teams) {
|
||||
projectTeamMap[item.project_id] = (item.teams as { name: string }).name;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Extract projects with team names
|
||||
if (isMounted && projectsData) {
|
||||
const projectList: Project[] = [];
|
||||
|
||||
for (const item of projectsData) {
|
||||
if (item.projects && typeof item.projects === 'object' && 'id' in item.projects && 'name' in item.projects) {
|
||||
const project = item.projects as Project;
|
||||
project.team_name = projectTeamMap[project.id];
|
||||
projectList.push(project);
|
||||
}
|
||||
}
|
||||
|
||||
setProjects(projectList);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (isMounted) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to load projects');
|
||||
}
|
||||
} finally {
|
||||
if (isMounted) {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent, session: Session | null) => {
|
||||
if (event === 'SIGNED_IN' && session?.user?.id) {
|
||||
fetchProjects(session.user.id);
|
||||
} else if (event === 'SIGNED_OUT') {
|
||||
setProjects([]);
|
||||
setError(null);
|
||||
}
|
||||
});
|
||||
|
||||
supabase.auth.getSession().then(({ data: { session } }) => {
|
||||
if (session?.user?.id) {
|
||||
fetchProjects(session.user.id);
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
isMounted = false;
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
}, [effectiveTeamIds]);
|
||||
|
||||
const handleToggle = () => {
|
||||
if (!loading && !error && projects.length > 0) {
|
||||
setIsOpen(!isOpen);
|
||||
}
|
||||
};
|
||||
|
||||
const handleProjectSelect = (projectId: string) => {
|
||||
let newSelected: string[];
|
||||
|
||||
if (multiple) {
|
||||
// For multi-select: toggle project in/out of selection
|
||||
if (selectedIds.includes(projectId)) {
|
||||
newSelected = selectedIds.filter(id => id !== projectId);
|
||||
} else {
|
||||
newSelected = [...selectedIds, projectId];
|
||||
}
|
||||
} else {
|
||||
// For single-select: replace selection with the new project
|
||||
newSelected = [projectId];
|
||||
setIsOpen(false);
|
||||
}
|
||||
|
||||
setSelectedIds(newSelected);
|
||||
|
||||
if (onChange) {
|
||||
onChange(multiple ? newSelected : newSelected[0] || '');
|
||||
}
|
||||
};
|
||||
|
||||
const removeProject = (e: React.MouseEvent, projectId: string) => {
|
||||
e.stopPropagation();
|
||||
const newSelected = selectedIds.filter(id => id !== projectId);
|
||||
setSelectedIds(newSelected);
|
||||
if (onChange) {
|
||||
onChange(multiple ? newSelected : newSelected[0] || '');
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center justify-between rounded-md border px-3 py-2",
|
||||
className
|
||||
)}>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center rounded-md border border-destructive bg-destructive/10 px-3 py-2 text-destructive",
|
||||
className
|
||||
)}>
|
||||
{error}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (projects.length === 0) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center rounded-md border px-3 py-2 text-muted-foreground",
|
||||
className
|
||||
)}>
|
||||
No projects available
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const selectedProjects = projects.filter(project => selectedIds.includes(project.id));
|
||||
|
||||
return (
|
||||
<div className="relative" ref={selectorRef}>
|
||||
<div
|
||||
className={cn(
|
||||
"flex w-full min-h-10 items-center flex-wrap rounded-md border p-1 cursor-pointer",
|
||||
isOpen && "ring-2 ring-offset-2 ring-blue-500",
|
||||
className
|
||||
)}
|
||||
onClick={handleToggle}
|
||||
>
|
||||
{selectedProjects.length > 0 ? (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{selectedProjects.map(project => (
|
||||
<div
|
||||
key={project.id}
|
||||
className="flex items-center gap-1 bg-green-100 text-green-800 rounded-md px-2 py-1 text-sm"
|
||||
>
|
||||
{project.name}
|
||||
{multiple && (
|
||||
<X
|
||||
size={14}
|
||||
className="cursor-pointer hover:text-green-900"
|
||||
onClick={(e) => removeProject(e, project.id)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="px-2 py-1 text-gray-500">Select a project</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isOpen && (
|
||||
<div className="absolute w-full mt-1 max-h-60 overflow-auto rounded-md border bg-white shadow-lg z-10">
|
||||
{projects.map(project => (
|
||||
<div
|
||||
key={project.id}
|
||||
className={cn(
|
||||
"px-3 py-2 cursor-pointer hover:bg-gray-100 flex items-center justify-between",
|
||||
selectedIds.includes(project.id) && "bg-green-50"
|
||||
)}
|
||||
onClick={() => handleProjectSelect(project.id)}
|
||||
>
|
||||
<span className="flex flex-col">
|
||||
<span className="font-medium">{project.name}</span>
|
||||
{project.team_name && (
|
||||
<span className="text-xs text-gray-500">
|
||||
{project.team_name}
|
||||
</span>
|
||||
)}
|
||||
{project.description && (
|
||||
<span className="text-xs text-gray-500 truncate max-w-[250px]">
|
||||
{project.description}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
{selectedIds.includes(project.id) && (
|
||||
<Check className="h-4 w-4 text-green-600" />
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
88
app/components/ui/Select.tsx
Normal file
88
app/components/ui/Select.tsx
Normal file
@@ -0,0 +1,88 @@
|
||||
"use client";
|
||||
|
||||
import * as React from 'react';
|
||||
import { ChevronDown } from 'lucide-react';
|
||||
|
||||
interface SelectOption {
|
||||
value: string;
|
||||
label: string;
|
||||
icon?: string;
|
||||
}
|
||||
|
||||
interface SelectProps {
|
||||
value?: string;
|
||||
onChange?: (value: string) => void;
|
||||
options: SelectOption[];
|
||||
placeholder?: string;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function Select({ value, onChange, options, placeholder, className = '' }: SelectProps) {
|
||||
const [isOpen, setIsOpen] = React.useState(false);
|
||||
const containerRef = React.useRef<HTMLDivElement>(null);
|
||||
|
||||
React.useEffect(() => {
|
||||
function handleClickOutside(event: MouseEvent) {
|
||||
if (containerRef.current && !containerRef.current.contains(event.target as Node)) {
|
||||
setIsOpen(false);
|
||||
}
|
||||
}
|
||||
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
return () => {
|
||||
document.removeEventListener('mousedown', handleClickOutside);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const selectedOption = options.find(option => option.value === value);
|
||||
|
||||
return (
|
||||
<div className={`relative ${className}`} ref={containerRef}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setIsOpen(!isOpen)}
|
||||
className="flex h-10 w-full items-center justify-between rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
<span className="flex items-center">
|
||||
{selectedOption?.icon && (
|
||||
<img
|
||||
src={selectedOption.icon}
|
||||
alt=""
|
||||
className="mr-2 h-4 w-4 rounded-full"
|
||||
/>
|
||||
)}
|
||||
{selectedOption?.label || placeholder}
|
||||
</span>
|
||||
<ChevronDown className="h-4 w-4 opacity-50" />
|
||||
</button>
|
||||
|
||||
{isOpen && (
|
||||
<div className="absolute z-50 mt-1 w-full rounded-md border bg-popover text-popover-foreground shadow-md animate-in fade-in-80">
|
||||
<div className="p-1">
|
||||
{options.map((option) => (
|
||||
<button
|
||||
key={option.value}
|
||||
onClick={() => {
|
||||
onChange?.(option.value);
|
||||
setIsOpen(false);
|
||||
}}
|
||||
className={`relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none hover:bg-accent hover:text-accent-foreground ${
|
||||
option.value === value ? 'bg-accent text-accent-foreground' : ''
|
||||
}`}
|
||||
>
|
||||
{option.icon && (
|
||||
<img
|
||||
src={option.icon}
|
||||
alt=""
|
||||
className="mr-2 h-4 w-4 rounded-full"
|
||||
/>
|
||||
)}
|
||||
{option.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
321
app/components/ui/TagSelector.tsx
Normal file
321
app/components/ui/TagSelector.tsx
Normal file
@@ -0,0 +1,321 @@
|
||||
"use client";
|
||||
|
||||
import * as React from 'react';
|
||||
import { useEffect, useState, useRef } from 'react';
|
||||
import { getSupabaseClient } from '../../utils/supabase';
|
||||
import { AuthChangeEvent } from '@supabase/supabase-js';
|
||||
import { Loader2, X, Check, Tag } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
// Define Tag type based on the database schema
|
||||
interface Tag {
|
||||
id: string;
|
||||
name: string;
|
||||
type?: string | null;
|
||||
attributes?: Record<string, unknown>;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
deleted_at?: string | null;
|
||||
parent_tag_id?: string | null;
|
||||
team_id?: string | null;
|
||||
is_shared?: boolean;
|
||||
schema_version?: number | null;
|
||||
is_system?: boolean;
|
||||
}
|
||||
|
||||
// TagSelector component with multi-select support
|
||||
export function TagSelector({
|
||||
value,
|
||||
onChange,
|
||||
className,
|
||||
multiple = false,
|
||||
teamId,
|
||||
teamIds,
|
||||
}: {
|
||||
value?: string | string[];
|
||||
onChange?: (tagIds: string | string[]) => void;
|
||||
className?: string;
|
||||
multiple?: boolean;
|
||||
teamId?: string; // Optional single team ID
|
||||
teamIds?: string[]; // Optional array of team IDs
|
||||
}) {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [tags, setTags] = useState<Tag[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<string[]>([]);
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const selectorRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Normalize team IDs to ensure we're always working with an array
|
||||
const effectiveTeamIds = React.useMemo(() => {
|
||||
if (teamIds && teamIds.length > 0) {
|
||||
return teamIds;
|
||||
} else if (teamId) {
|
||||
return [teamId];
|
||||
}
|
||||
return undefined;
|
||||
}, [teamId, teamIds]);
|
||||
|
||||
// 标签名称与ID的映射函数
|
||||
const getTagIdByName = (name: string): string | undefined => {
|
||||
const tag = tags.find(t => t.name === name);
|
||||
return tag?.id;
|
||||
};
|
||||
|
||||
const getTagNameById = (id: string): string | undefined => {
|
||||
const tag = tags.find(t => t.id === id);
|
||||
return tag?.name;
|
||||
};
|
||||
|
||||
// 从标签名称转换为标签ID
|
||||
const nameToId = (nameOrNames: string | string[] | undefined): string[] => {
|
||||
if (!nameOrNames) return [];
|
||||
if (Array.isArray(nameOrNames)) {
|
||||
return nameOrNames
|
||||
.map(name => getTagIdByName(name))
|
||||
.filter((id): id is string => !!id);
|
||||
}
|
||||
const id = getTagIdByName(nameOrNames);
|
||||
return id ? [id] : [];
|
||||
};
|
||||
|
||||
// 从标签ID转换为标签名称
|
||||
const idToName = (idOrIds: string | string[] | undefined): string[] => {
|
||||
if (!idOrIds) return [];
|
||||
if (Array.isArray(idOrIds)) {
|
||||
return idOrIds
|
||||
.map(id => getTagNameById(id))
|
||||
.filter((name): name is string => !!name);
|
||||
}
|
||||
const name = getTagNameById(idOrIds);
|
||||
return name ? [name] : [];
|
||||
};
|
||||
|
||||
// 初始化已选择的标签 - 从传入的名称转换为ID
|
||||
useEffect(() => {
|
||||
if (tags.length > 0 && value) {
|
||||
setSelectedIds(nameToId(value));
|
||||
}
|
||||
}, [value, tags]);
|
||||
|
||||
// Add click outside listener to close dropdown
|
||||
useEffect(() => {
|
||||
function handleClickOutside(event: MouseEvent) {
|
||||
if (selectorRef.current && !selectorRef.current.contains(event.target as Node)) {
|
||||
setIsOpen(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Only add the event listener if the dropdown is open
|
||||
if (isOpen) {
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
return () => {
|
||||
document.removeEventListener('mousedown', handleClickOutside);
|
||||
};
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
let isMounted = true;
|
||||
|
||||
const fetchTags = async () => {
|
||||
if (!isMounted) return;
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
let query = supabase.from('tags').select('*').is('deleted_at', null);
|
||||
|
||||
// Filter by team if teamId is provided
|
||||
if (effectiveTeamIds) {
|
||||
query = query.in('team_id', effectiveTeamIds);
|
||||
}
|
||||
|
||||
const { data: tagsData, error: tagsError } = await query;
|
||||
|
||||
if (tagsError) throw tagsError;
|
||||
|
||||
if (!tagsData || tagsData.length === 0) {
|
||||
if (isMounted) setTags([]);
|
||||
return;
|
||||
}
|
||||
|
||||
if (isMounted) {
|
||||
setTags(tagsData as Tag[]);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isMounted) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to load tags');
|
||||
}
|
||||
} finally {
|
||||
if (isMounted) {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent) => {
|
||||
if (event === 'SIGNED_IN') {
|
||||
fetchTags();
|
||||
} else if (event === 'SIGNED_OUT') {
|
||||
setTags([]);
|
||||
setError(null);
|
||||
}
|
||||
});
|
||||
|
||||
supabase.auth.getSession().then(() => {
|
||||
fetchTags();
|
||||
});
|
||||
|
||||
return () => {
|
||||
isMounted = false;
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
}, [effectiveTeamIds]);
|
||||
|
||||
const handleToggle = () => {
|
||||
if (!loading && !error && tags.length > 0) {
|
||||
setIsOpen(!isOpen);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTagSelect = (tagId: string) => {
|
||||
let newSelected: string[];
|
||||
|
||||
if (multiple) {
|
||||
// For multi-select: toggle tag in/out of selection
|
||||
if (selectedIds.includes(tagId)) {
|
||||
newSelected = selectedIds.filter(id => id !== tagId);
|
||||
} else {
|
||||
newSelected = [...selectedIds, tagId];
|
||||
}
|
||||
} else {
|
||||
// For single-select: replace selection with the new tag
|
||||
newSelected = [tagId];
|
||||
setIsOpen(false);
|
||||
}
|
||||
|
||||
setSelectedIds(newSelected);
|
||||
|
||||
// 传递标签名称而不是ID
|
||||
if (onChange) {
|
||||
const tagNames = idToName(newSelected);
|
||||
onChange(multiple ? tagNames : tagNames[0] || '');
|
||||
}
|
||||
};
|
||||
|
||||
const removeTag = (e: React.MouseEvent, tagId: string) => {
|
||||
e.stopPropagation();
|
||||
const newSelected = selectedIds.filter(id => id !== tagId);
|
||||
setSelectedIds(newSelected);
|
||||
|
||||
// 传递标签名称而不是ID
|
||||
if (onChange) {
|
||||
const tagNames = idToName(newSelected);
|
||||
onChange(multiple ? tagNames : tagNames[0] || '');
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center justify-between rounded-md border px-3 py-2",
|
||||
className
|
||||
)}>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center rounded-md border border-destructive bg-destructive/10 px-3 py-2 text-destructive",
|
||||
className
|
||||
)}>
|
||||
{error}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (tags.length === 0) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center rounded-md border px-3 py-2 text-muted-foreground",
|
||||
className
|
||||
)}>
|
||||
No tags available
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// 根据已选择的ID筛选出已选择的标签
|
||||
const selectedTags = tags.filter(tag => selectedIds.includes(tag.id));
|
||||
|
||||
return (
|
||||
<div className="relative" ref={selectorRef}>
|
||||
<div
|
||||
className={cn(
|
||||
"flex w-full min-h-10 items-center flex-wrap rounded-md border p-1 cursor-pointer",
|
||||
isOpen && "ring-2 ring-offset-2 ring-purple-500",
|
||||
className
|
||||
)}
|
||||
onClick={handleToggle}
|
||||
>
|
||||
{selectedTags.length > 0 ? (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{selectedTags.map(tag => (
|
||||
<div
|
||||
key={tag.id}
|
||||
className="flex items-center gap-1 bg-purple-100 text-purple-800 rounded-md px-2 py-1 text-sm"
|
||||
>
|
||||
{tag.name}
|
||||
{multiple && (
|
||||
<X
|
||||
size={14}
|
||||
className="cursor-pointer hover:text-purple-900"
|
||||
onClick={(e) => removeTag(e, tag.id)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="px-2 py-1 text-gray-500">Select tags</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isOpen && (
|
||||
<div className="absolute w-full mt-1 max-h-60 overflow-auto rounded-md border bg-white shadow-lg z-10">
|
||||
{tags.map(tag => (
|
||||
<div
|
||||
key={tag.id}
|
||||
className={cn(
|
||||
"px-3 py-2 cursor-pointer hover:bg-gray-100 flex items-center justify-between",
|
||||
selectedIds.includes(tag.id) && "bg-purple-50"
|
||||
)}
|
||||
onClick={() => handleTagSelect(tag.id)}
|
||||
>
|
||||
<span className="flex items-center gap-2">
|
||||
<Tag className="h-4 w-4 text-purple-500" />
|
||||
<span>{tag.name}</span>
|
||||
{tag.type && (
|
||||
<span className="text-xs text-gray-500 bg-gray-100 px-1.5 py-0.5 rounded">
|
||||
{tag.type}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
{selectedIds.includes(tag.id) && (
|
||||
<Check className="h-4 w-4 text-purple-600" />
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
267
app/components/ui/TeamSelector.tsx
Normal file
267
app/components/ui/TeamSelector.tsx
Normal file
@@ -0,0 +1,267 @@
|
||||
"use client";
|
||||
|
||||
import * as React from 'react';
|
||||
import { useEffect, useState, useRef } from 'react';
|
||||
import type { Database } from '@/types/supabase';
|
||||
import { getSupabaseClient } from '../../utils/supabase';
|
||||
import { AuthChangeEvent, Session } from '@supabase/supabase-js';
|
||||
import { Loader2, X, Check } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { limqRequest } from '@/lib/api';
|
||||
|
||||
type Team = Database['limq']['Tables']['teams']['Row'];
|
||||
|
||||
// TeamSelector component with multi-select support
|
||||
export function TeamSelector({
|
||||
value,
|
||||
onChange,
|
||||
className,
|
||||
multiple = false,
|
||||
}: {
|
||||
value?: string | string[];
|
||||
onChange?: (teamId: string | string[]) => void;
|
||||
className?: string;
|
||||
multiple?: boolean;
|
||||
}) {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [teams, setTeams] = useState<Team[]>([]);
|
||||
const [selectedIds, setSelectedIds] = useState<string[]>([]);
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const selectorRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Initialize selected teams based on value prop
|
||||
useEffect(() => {
|
||||
if (value) {
|
||||
if (Array.isArray(value)) {
|
||||
setSelectedIds(value);
|
||||
} else {
|
||||
setSelectedIds(value ? [value] : []);
|
||||
}
|
||||
} else {
|
||||
setSelectedIds([]);
|
||||
}
|
||||
}, [value]);
|
||||
|
||||
// Add click outside listener to close dropdown
|
||||
useEffect(() => {
|
||||
function handleClickOutside(event: MouseEvent) {
|
||||
if (selectorRef.current && !selectorRef.current.contains(event.target as Node)) {
|
||||
setIsOpen(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Only add the event listener if the dropdown is open
|
||||
if (isOpen) {
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
return () => {
|
||||
document.removeEventListener('mousedown', handleClickOutside);
|
||||
};
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
let isMounted = true;
|
||||
|
||||
const fetchTeams = async (userId: string) => {
|
||||
if (!isMounted) return;
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
// 尝试创建默认团队和项目(如果用户还没有)
|
||||
try {
|
||||
const response = await limqRequest('team/create-default', 'POST');
|
||||
console.log('Default team creation response:', response);
|
||||
} catch (teamError) {
|
||||
console.error('Error creating default team:', teamError);
|
||||
}
|
||||
|
||||
const { data: memberships, error: membershipError } = await supabase
|
||||
.from('team_membership')
|
||||
.select('team_id')
|
||||
.eq('user_id', userId);
|
||||
|
||||
if (membershipError) throw membershipError;
|
||||
|
||||
if (!memberships || memberships.length === 0) {
|
||||
if (isMounted) setTeams([]);
|
||||
return;
|
||||
}
|
||||
|
||||
const teamIds = memberships.map(m => m.team_id);
|
||||
const { data: teamsData, error: teamsError } = await supabase
|
||||
.from('teams')
|
||||
.select('*')
|
||||
.in('id', teamIds)
|
||||
.is('deleted_at', null);
|
||||
|
||||
if (teamsError) throw teamsError;
|
||||
|
||||
if (isMounted && teamsData) {
|
||||
setTeams(teamsData);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isMounted) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to load teams');
|
||||
}
|
||||
} finally {
|
||||
if (isMounted) {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent, session: Session | null) => {
|
||||
if (event === 'SIGNED_IN' && session?.user?.id) {
|
||||
fetchTeams(session.user.id);
|
||||
} else if (event === 'SIGNED_OUT') {
|
||||
setTeams([]);
|
||||
setError(null);
|
||||
}
|
||||
});
|
||||
|
||||
supabase.auth.getSession().then(({ data: { session } }) => {
|
||||
if (session?.user?.id) {
|
||||
fetchTeams(session.user.id);
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
isMounted = false;
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
}, []);
|
||||
|
||||
const handleToggle = () => {
|
||||
if (!loading && !error && teams.length > 0) {
|
||||
setIsOpen(!isOpen);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTeamSelect = (teamId: string) => {
|
||||
let newSelected: string[];
|
||||
|
||||
if (multiple) {
|
||||
// For multi-select: toggle team in/out of selection
|
||||
if (selectedIds.includes(teamId)) {
|
||||
newSelected = selectedIds.filter(id => id !== teamId);
|
||||
} else {
|
||||
newSelected = [...selectedIds, teamId];
|
||||
}
|
||||
} else {
|
||||
// For single-select: replace selection with the new team
|
||||
newSelected = [teamId];
|
||||
setIsOpen(false);
|
||||
}
|
||||
|
||||
setSelectedIds(newSelected);
|
||||
|
||||
if (onChange) {
|
||||
onChange(multiple ? newSelected : newSelected[0] || '');
|
||||
}
|
||||
};
|
||||
|
||||
const removeTeam = (e: React.MouseEvent, teamId: string) => {
|
||||
e.stopPropagation();
|
||||
const newSelected = selectedIds.filter(id => id !== teamId);
|
||||
setSelectedIds(newSelected);
|
||||
if (onChange) {
|
||||
onChange(multiple ? newSelected : newSelected[0] || '');
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center justify-between rounded-md border px-3 py-2",
|
||||
className
|
||||
)}>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center rounded-md border border-destructive bg-destructive/10 px-3 py-2 text-destructive",
|
||||
className
|
||||
)}>
|
||||
{error}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (teams.length === 0) {
|
||||
return (
|
||||
<div className={cn(
|
||||
"flex w-full items-center rounded-md border px-3 py-2 text-muted-foreground",
|
||||
className
|
||||
)}>
|
||||
No teams available
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const selectedTeams = teams.filter(team => selectedIds.includes(team.id));
|
||||
|
||||
return (
|
||||
<div className="relative" ref={selectorRef}>
|
||||
<div
|
||||
className={cn(
|
||||
"flex w-full min-h-10 items-center flex-wrap rounded-md border p-1 cursor-pointer",
|
||||
isOpen && "ring-2 ring-offset-2 ring-blue-500",
|
||||
className
|
||||
)}
|
||||
onClick={handleToggle}
|
||||
>
|
||||
{selectedTeams.length > 0 ? (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{selectedTeams.map(team => (
|
||||
<div
|
||||
key={team.id}
|
||||
className="flex items-center gap-1 bg-blue-100 text-blue-800 rounded-md px-2 py-1 text-sm"
|
||||
>
|
||||
{team.name}
|
||||
{multiple && (
|
||||
<X
|
||||
size={14}
|
||||
className="cursor-pointer hover:text-blue-900"
|
||||
onClick={(e) => removeTeam(e, team.id)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="px-2 py-1 text-gray-500">Select a team</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isOpen && (
|
||||
<div className="absolute w-full mt-1 max-h-60 overflow-auto rounded-md border bg-white shadow-lg z-10">
|
||||
{teams.map(team => (
|
||||
<div
|
||||
key={team.id}
|
||||
className={cn(
|
||||
"px-3 py-2 cursor-pointer hover:bg-gray-100 flex items-center justify-between",
|
||||
selectedIds.includes(team.id) && "bg-blue-50"
|
||||
)}
|
||||
onClick={() => handleTeamSelect(team.id)}
|
||||
>
|
||||
<span>{team.name}</span>
|
||||
{selectedIds.includes(team.id) && (
|
||||
<Check className="h-4 w-4 text-blue-600" />
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
export default function ThemeToggle() {
|
||||
const [darkMode, setDarkMode] = useState(false);
|
||||
|
||||
// Initialize theme on component mount
|
||||
useEffect(() => {
|
||||
const isDarkMode = localStorage.getItem('darkMode') === 'true';
|
||||
setDarkMode(isDarkMode);
|
||||
|
||||
if (isDarkMode) {
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
document.documentElement.classList.remove('dark');
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Update theme when darkMode state changes
|
||||
const toggleTheme = () => {
|
||||
const newDarkMode = !darkMode;
|
||||
setDarkMode(newDarkMode);
|
||||
localStorage.setItem('darkMode', newDarkMode.toString());
|
||||
|
||||
if (newDarkMode) {
|
||||
document.documentElement.classList.add('dark');
|
||||
} else {
|
||||
document.documentElement.classList.remove('dark');
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<button
|
||||
onClick={toggleTheme}
|
||||
className="p-2 rounded-md bg-card-bg border border-card-border hover:bg-card-bg/80 transition-colors"
|
||||
aria-label={darkMode ? "Switch to light mode" : "Switch to dark mode"}
|
||||
>
|
||||
{darkMode ? (
|
||||
<svg
|
||||
className="w-5 h-5 text-accent-yellow"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 20 20"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<path
|
||||
fillRule="evenodd"
|
||||
d="M10 2a1 1 0 011 1v1a1 1 0 11-2 0V3a1 1 0 011-1zm4 8a4 4 0 11-8 0 4 4 0 018 0zm-.464 4.95l.707.707a1 1 0 001.414-1.414l-.707-.707a1 1 0 00-1.414 1.414zm2.12-10.607a1 1 0 010 1.414l-.706.707a1 1 0 11-1.414-1.414l.707-.707a1 1 0 011.414 0zM17 11a1 1 0 100-2h-1a1 1 0 100 2h1zm-7 4a1 1 0 011 1v1a1 1 0 11-2 0v-1a1 1 0 011-1zM5.05 6.464A1 1 0 106.465 5.05l-.708-.707a1 1 0 00-1.414 1.414l.707.707zm1.414 8.486l-.707.707a1 1 0 01-1.414-1.414l.707-.707a1 1 0 011.414 1.414zM4 11a1 1 0 100-2H3a1 1 0 000 2h1z"
|
||||
clipRule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
) : (
|
||||
<svg
|
||||
className="w-5 h-5 text-foreground"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 20 20"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<path d="M17.293 13.293A8 8 0 016.707 2.707a8.001 8.001 0 1010.586 10.586z" />
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
403
app/create-shorturl/page.tsx
Normal file
403
app/create-shorturl/page.tsx
Normal file
@@ -0,0 +1,403 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useAuth } from '@/lib/auth';
|
||||
import { limqRequest } from '@/lib/api';
|
||||
import { TeamSelector } from '@/app/components/ui/TeamSelector';
|
||||
import { ProjectSelector } from '@/app/components/ui/ProjectSelector';
|
||||
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
|
||||
|
||||
interface ShortUrlData {
|
||||
originalUrl: string;
|
||||
customSlug?: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
tags?: string[];
|
||||
teamId: string;
|
||||
projectId: string;
|
||||
domain: string;
|
||||
}
|
||||
|
||||
export default function CreateShortUrlPage() {
|
||||
return (
|
||||
<ClientRouteGuard>
|
||||
<CreateShortUrlForm />
|
||||
</ClientRouteGuard>
|
||||
);
|
||||
}
|
||||
|
||||
function CreateShortUrlForm() {
|
||||
const router = useRouter();
|
||||
const { user } = useAuth();
|
||||
|
||||
const [formData, setFormData] = useState<ShortUrlData>({
|
||||
originalUrl: '',
|
||||
customSlug: '',
|
||||
title: '',
|
||||
description: '',
|
||||
tags: [],
|
||||
teamId: '',
|
||||
projectId: '',
|
||||
domain: 'googleads.link',
|
||||
});
|
||||
|
||||
const [tagInput, setTagInput] = useState('');
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [success, setSuccess] = useState(false);
|
||||
|
||||
// Use useEffect to add user information to form data on load
|
||||
useEffect(() => {
|
||||
if (user) {
|
||||
console.log('Current user:', user.email);
|
||||
// Can add user-related data to the form here
|
||||
}
|
||||
}, [user]);
|
||||
|
||||
const handleChange = (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
|
||||
const { name, value } = e.target;
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
[name]: value
|
||||
}));
|
||||
};
|
||||
|
||||
const handleTagKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === 'Enter' && tagInput.trim()) {
|
||||
e.preventDefault();
|
||||
addTag();
|
||||
}
|
||||
};
|
||||
|
||||
const addTag = () => {
|
||||
if (tagInput.trim() && !formData.tags?.includes(tagInput.trim())) {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
tags: [...(prev.tags || []), tagInput.trim()]
|
||||
}));
|
||||
setTagInput('');
|
||||
}
|
||||
};
|
||||
|
||||
const removeTag = (tagToRemove: string) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
tags: prev.tags?.filter(tag => tag !== tagToRemove)
|
||||
}));
|
||||
};
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
setIsSubmitting(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Validate required fields
|
||||
if (!formData.originalUrl) {
|
||||
throw new Error('Original URL is required');
|
||||
}
|
||||
|
||||
if (!formData.title) {
|
||||
throw new Error('Title is required');
|
||||
}
|
||||
|
||||
if (!formData.teamId) {
|
||||
throw new Error('Team is required');
|
||||
}
|
||||
|
||||
if (!formData.projectId) {
|
||||
throw new Error('Project is required');
|
||||
}
|
||||
|
||||
if (!formData.domain) {
|
||||
throw new Error('Domain is required');
|
||||
}
|
||||
|
||||
// Construct request data according to API requirements
|
||||
const requestData = {
|
||||
type: "shorturl",
|
||||
attributes: {
|
||||
// Can add any additional attributes, but attributes cannot be empty
|
||||
icon: ""
|
||||
},
|
||||
shortUrl: {
|
||||
url: formData.originalUrl,
|
||||
slug: formData.customSlug || undefined,
|
||||
title: formData.title,
|
||||
name: formData.title,
|
||||
description: formData.description || "",
|
||||
domain: formData.domain
|
||||
},
|
||||
teamId: formData.teamId,
|
||||
projectId: formData.projectId,
|
||||
tagIds: formData.tags && formData.tags.length > 0 ? formData.tags : undefined
|
||||
};
|
||||
|
||||
// Call API to create shorturl resource
|
||||
const response = await limqRequest('resource/shorturl', 'POST', requestData as unknown as Record<string, unknown>);
|
||||
|
||||
console.log('Created successfully:', response);
|
||||
setSuccess(true);
|
||||
|
||||
// Redirect to links list page after 2 seconds
|
||||
setTimeout(() => {
|
||||
router.push('/links');
|
||||
}, 2000);
|
||||
|
||||
} catch (err) {
|
||||
console.error('Failed to create short URL:', err);
|
||||
setError(err instanceof Error ? err.message : 'Failed to create short URL, please try again later');
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="container mx-auto px-4 py-8 max-w-3xl">
|
||||
<div className="bg-white rounded-lg shadow-md overflow-hidden">
|
||||
<div className="border-b border-gray-200 bg-blue-50 px-6 py-4">
|
||||
<h1 className="text-xl font-medium text-gray-900">Create Short URL</h1>
|
||||
<p className="mt-1 text-sm text-gray-600">
|
||||
Create a new short URL resource for tracking and analytics
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border-l-4 border-red-500 p-4 m-6">
|
||||
<div className="flex">
|
||||
<div className="flex-shrink-0">
|
||||
<svg className="h-5 w-5 text-red-500" fill="currentColor" viewBox="0 0 20 20">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
<div className="ml-3">
|
||||
<p className="text-sm text-red-700">{error}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{success && (
|
||||
<div className="bg-green-50 border-l-4 border-green-500 p-4 m-6">
|
||||
<div className="flex">
|
||||
<div className="flex-shrink-0">
|
||||
<svg className="h-5 w-5 text-green-500" fill="currentColor" viewBox="0 0 20 20">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
<div className="ml-3">
|
||||
<p className="text-sm text-green-700">
|
||||
Short URL created successfully! Redirecting...
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form onSubmit={handleSubmit} className="p-6 space-y-6">
|
||||
{/* Title */}
|
||||
<div>
|
||||
<label htmlFor="title" className="block text-sm font-medium text-gray-700">
|
||||
Title <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
id="title"
|
||||
name="title"
|
||||
value={formData.title}
|
||||
onChange={handleChange}
|
||||
placeholder="e.g., Product Launch Campaign"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Original URL */}
|
||||
<div>
|
||||
<label htmlFor="originalUrl" className="block text-sm font-medium text-gray-700">
|
||||
Original URL <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="url"
|
||||
id="originalUrl"
|
||||
name="originalUrl"
|
||||
value={formData.originalUrl}
|
||||
onChange={handleChange}
|
||||
placeholder="https://example.com/your-long-url"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Custom Short Link */}
|
||||
<div>
|
||||
<label htmlFor="customSlug" className="block text-sm font-medium text-gray-700">
|
||||
Custom Short Link <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<div className="flex mt-1 rounded-md shadow-sm">
|
||||
<span className="inline-flex items-center px-3 py-2 text-sm text-gray-500 border border-r-0 border-gray-300 rounded-l-md bg-gray-50">
|
||||
{formData.domain}/
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
id="customSlug"
|
||||
name="customSlug"
|
||||
value={formData.customSlug}
|
||||
onChange={handleChange}
|
||||
placeholder="custom-slug"
|
||||
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-none rounded-r-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
</div>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
Leave blank to generate a random short link
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Domain */}
|
||||
<div>
|
||||
<label htmlFor="domain" className="block text-sm font-medium text-gray-700">
|
||||
Domain <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
id="domain"
|
||||
name="domain"
|
||||
value={formData.domain}
|
||||
onChange={handleChange}
|
||||
placeholder="e.g., googleads.link"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Team Selection */}
|
||||
<div>
|
||||
<label htmlFor="teamId" className="block text-sm font-medium text-gray-700">
|
||||
Team <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<div className="mt-1">
|
||||
<TeamSelector
|
||||
value={formData.teamId}
|
||||
onChange={(teamId) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
teamId: teamId as string,
|
||||
// Clear selected project when team changes
|
||||
projectId: ''
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Project Selection */}
|
||||
<div>
|
||||
<label htmlFor="projectId" className="block text-sm font-medium text-gray-700">
|
||||
Project <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<div className="mt-1">
|
||||
<ProjectSelector
|
||||
teamId={formData.teamId}
|
||||
value={formData.projectId}
|
||||
onChange={(projectId) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
projectId: projectId as string
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Description */}
|
||||
<div>
|
||||
<label htmlFor="description" className="block text-sm font-medium text-gray-700">
|
||||
Description <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<textarea
|
||||
id="description"
|
||||
name="description"
|
||||
value={formData.description}
|
||||
onChange={handleChange}
|
||||
rows={3}
|
||||
placeholder="A brief description of this link"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Tags */}
|
||||
<div>
|
||||
<label htmlFor="tagInput" className="block text-sm font-medium text-gray-700">
|
||||
Tags <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<div className="flex mt-1 rounded-md shadow-sm">
|
||||
<input
|
||||
type="text"
|
||||
id="tagInput"
|
||||
value={tagInput}
|
||||
onChange={(e) => setTagInput(e.target.value)}
|
||||
onKeyDown={handleTagKeyDown}
|
||||
placeholder="Add a tag and press Enter"
|
||||
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-l-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={addTag}
|
||||
className="inline-flex items-center px-3 py-2 text-sm font-medium text-white border border-transparent rounded-r-md shadow-sm bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
Add
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{formData.tags && formData.tags.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2 mt-2">
|
||||
{formData.tags.map(tag => (
|
||||
<span key={tag} className="inline-flex items-center px-2 py-0.5 text-xs font-medium bg-blue-100 rounded-full text-blue-800">
|
||||
{tag}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeTag(tag)}
|
||||
className="flex-shrink-0 ml-1 text-blue-500 rounded-full hover:text-blue-700 focus:outline-none"
|
||||
>
|
||||
<span className="sr-only">Remove tag {tag}</span>
|
||||
<svg className="w-3 h-3" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Submit Button */}
|
||||
<div className="flex justify-end pt-4 border-t border-gray-200">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => router.back()}
|
||||
className="px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 mr-3"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isSubmitting}
|
||||
className="inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
{isSubmitting ? (
|
||||
<>
|
||||
<svg className="w-5 h-5 mr-2 -ml-1 animate-spin" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
||||
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
Processing...
|
||||
</>
|
||||
) : 'Create Short URL'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -30,30 +30,6 @@
|
||||
--gradient-red: linear-gradient(135deg, #f43f5e, #e11d48);
|
||||
}
|
||||
|
||||
.dark {
|
||||
/* Dark Mode */
|
||||
--background: #0f172a;
|
||||
--foreground: #ffffff;
|
||||
|
||||
/* Card colors */
|
||||
--card-bg: #1e293b;
|
||||
--card-border: #334155;
|
||||
|
||||
/* Vibrant accent colors */
|
||||
--accent-blue: #3b82f6;
|
||||
--accent-green: #10b981;
|
||||
--accent-red: #f43f5e;
|
||||
--accent-yellow: #f59e0b;
|
||||
--accent-purple: #8b5cf6;
|
||||
--accent-pink: #ec4899;
|
||||
--accent-teal: #14b8a6;
|
||||
--accent-orange: #f97316;
|
||||
|
||||
/* UI colors */
|
||||
--text-secondary: #94a3b8;
|
||||
--progress-bg: #334155;
|
||||
}
|
||||
|
||||
@theme inline {
|
||||
--color-background: var(--background);
|
||||
--color-foreground: var(--foreground);
|
||||
|
||||
@@ -1,21 +1,13 @@
|
||||
import './globals.css';
|
||||
import '@radix-ui/themes/styles.css';
|
||||
import type { Metadata } from 'next';
|
||||
import { Geist, Geist_Mono } from "next/font/google";
|
||||
import Navbar from "./components/layout/Navbar";
|
||||
|
||||
const geistSans = Geist({
|
||||
variable: "--font-geist-sans",
|
||||
subsets: ["latin"],
|
||||
});
|
||||
|
||||
const geistMono = Geist_Mono({
|
||||
variable: "--font-geist-mono",
|
||||
subsets: ["latin"],
|
||||
});
|
||||
import { AuthProvider } from '@/lib/auth';
|
||||
import { Theme } from '@radix-ui/themes';
|
||||
import Header from '@/app/components/layout/Header';
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: 'ShortURL Analytics',
|
||||
description: 'Analytics dashboard for short URL management',
|
||||
description: 'Track and analyze shortened links',
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
@@ -25,14 +17,14 @@ export default function RootLayout({
|
||||
}>) {
|
||||
return (
|
||||
<html lang="en" suppressHydrationWarning>
|
||||
<body
|
||||
className={`${geistSans.variable} ${geistMono.variable} antialiased bg-background`}
|
||||
>
|
||||
<Navbar />
|
||||
<main className="min-h-screen px-4 py-6">
|
||||
{children}
|
||||
</main>
|
||||
<body>
|
||||
<Theme>
|
||||
<AuthProvider>
|
||||
<Header />
|
||||
{children}
|
||||
</AuthProvider>
|
||||
</Theme>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import './globals.css';
|
||||
import type { Metadata } from 'next';
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: 'Link Management & Analytics',
|
||||
description: 'Track and analyze shortened links',
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
children,
|
||||
}: Readonly<{
|
||||
children: React.ReactNode;
|
||||
}>) {
|
||||
return (
|
||||
<html lang="en" suppressHydrationWarning>
|
||||
<body>
|
||||
{children}
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
1125
app/links/page.tsx
1125
app/links/page.tsx
File diff suppressed because it is too large
Load Diff
207
app/login/page.tsx
Normal file
207
app/login/page.tsx
Normal file
@@ -0,0 +1,207 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect, Suspense } from 'react';
|
||||
import { useRouter, useSearchParams } from 'next/navigation';
|
||||
import Link from 'next/link';
|
||||
import { useAuth } from '@/lib/auth';
|
||||
|
||||
// Separate component for message handling to isolate useSearchParams
|
||||
function MessageHandler({ setMessage }: { setMessage: (message: { type: string, content: string }) => void }) {
|
||||
const searchParams = useSearchParams();
|
||||
|
||||
useEffect(() => {
|
||||
const messageParam = searchParams.get('message');
|
||||
if (messageParam) {
|
||||
setMessage({ type: 'info', content: messageParam });
|
||||
}
|
||||
}, [searchParams, setMessage]);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export default function LoginPage() {
|
||||
const router = useRouter();
|
||||
const { signIn, signInWithGoogle, user } = useAuth();
|
||||
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [message, setMessage] = useState({ type: '', content: '' });
|
||||
|
||||
// 如果用户已登录,重定向到首页
|
||||
useEffect(() => {
|
||||
if (user) {
|
||||
router.push('/');
|
||||
}
|
||||
}, [user, router]);
|
||||
|
||||
const handleEmailSignIn = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
|
||||
if (!email || !password) {
|
||||
setMessage({
|
||||
type: 'error',
|
||||
content: 'Please enter both email and password'
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
setIsLoading(true);
|
||||
setMessage({ type: '', content: '' });
|
||||
|
||||
const { error } = await signIn(email, password);
|
||||
|
||||
if (error) {
|
||||
throw new Error(error.message);
|
||||
}
|
||||
|
||||
// 登录成功后,会通过 useEffect 重定向
|
||||
} catch (error) {
|
||||
console.error('Login error:', error);
|
||||
setMessage({
|
||||
type: 'error',
|
||||
content: error instanceof Error ? error.message : 'Failed to sign in'
|
||||
});
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleGoogleSignIn = async () => {
|
||||
try {
|
||||
setIsLoading(true);
|
||||
setMessage({ type: '', content: '' });
|
||||
|
||||
const { error } = await signInWithGoogle();
|
||||
|
||||
if (error) {
|
||||
throw new Error(error.message);
|
||||
}
|
||||
|
||||
// Google OAuth will handle the redirect
|
||||
} catch (error) {
|
||||
console.error('Google login error:', error);
|
||||
setMessage({
|
||||
type: 'error',
|
||||
content: error instanceof Error ? error.message : 'Failed to sign in with Google'
|
||||
});
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen bg-gray-100">
|
||||
{/* Wrap the component using useSearchParams in Suspense */}
|
||||
<Suspense fallback={null}>
|
||||
<MessageHandler setMessage={setMessage} />
|
||||
</Suspense>
|
||||
|
||||
<div className="w-full max-w-md p-8 space-y-8 bg-white rounded-lg shadow-md">
|
||||
<div className="text-center">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Login</h1>
|
||||
<p className="mt-2 text-sm text-gray-600">
|
||||
Sign in to your account to access analytics
|
||||
</p>
|
||||
<div className="mt-2 text-xs text-gray-500">
|
||||
Welcome to ShortURL Analytics
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Message display */}
|
||||
{message.content && (
|
||||
<div className={`p-4 mb-4 text-sm rounded-lg ${
|
||||
message.type === 'error'
|
||||
? 'text-red-700 bg-red-100 border border-red-200'
|
||||
: 'text-blue-700 bg-blue-50 border border-blue-200'
|
||||
}`}>
|
||||
{message.type === 'error' ? (
|
||||
<span className="font-medium">Error: </span>
|
||||
) : (
|
||||
<span className="font-medium">Notice: </span>
|
||||
)}
|
||||
{message.content}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Google Sign In Button */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleGoogleSignIn}
|
||||
disabled={isLoading}
|
||||
className="w-full flex items-center justify-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
<svg className="h-5 w-5 mr-2" viewBox="0 0 24 24" width="24" height="24">
|
||||
<g transform="matrix(1, 0, 0, 1, 27.009001, -39.238998)">
|
||||
<path fill="#4285F4" d="M -3.264 51.509 C -3.264 50.719 -3.334 49.969 -3.454 49.239 L -14.754 49.239 L -14.754 53.749 L -8.284 53.749 C -8.574 55.229 -9.424 56.479 -10.684 57.329 L -10.684 60.329 L -6.824 60.329 C -4.564 58.239 -3.264 55.159 -3.264 51.509 Z"/>
|
||||
<path fill="#34A853" d="M -14.754 63.239 C -11.514 63.239 -8.804 62.159 -6.824 60.329 L -10.684 57.329 C -11.764 58.049 -13.134 58.489 -14.754 58.489 C -17.884 58.489 -20.534 56.379 -21.484 53.529 L -25.464 53.529 L -25.464 56.619 C -23.494 60.539 -19.444 63.239 -14.754 63.239 Z"/>
|
||||
<path fill="#FBBC05" d="M -21.484 53.529 C -21.734 52.809 -21.864 52.039 -21.864 51.239 C -21.864 50.439 -21.724 49.669 -21.484 48.949 L -21.484 45.859 L -25.464 45.859 C -26.284 47.479 -26.754 49.299 -26.754 51.239 C -26.754 53.179 -26.284 54.999 -25.464 56.619 L -21.484 53.529 Z"/>
|
||||
<path fill="#EA4335" d="M -14.754 43.989 C -12.984 43.989 -11.404 44.599 -10.154 45.789 L -6.734 42.369 C -8.804 40.429 -11.514 39.239 -14.754 39.239 C -19.444 39.239 -23.494 41.939 -25.464 45.859 L -21.484 48.949 C -20.534 46.099 -17.884 43.989 -14.754 43.989 Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
{isLoading ? 'Signing in...' : 'Sign in with Google'}
|
||||
</button>
|
||||
|
||||
<div className="mt-6 flex items-center justify-center">
|
||||
<div className="border-t border-gray-300 flex-grow mr-3"></div>
|
||||
<span className="text-sm text-gray-500">or</span>
|
||||
<div className="border-t border-gray-300 flex-grow ml-3"></div>
|
||||
</div>
|
||||
|
||||
<form onSubmit={handleEmailSignIn} className="mt-6 space-y-6">
|
||||
<div>
|
||||
<label htmlFor="email" className="block text-sm font-medium text-gray-700">
|
||||
Email address
|
||||
</label>
|
||||
<input
|
||||
id="email"
|
||||
name="email"
|
||||
type="email"
|
||||
autoComplete="email"
|
||||
required
|
||||
value={email}
|
||||
onChange={(e) => setEmail(e.target.value)}
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm placeholder-gray-400 focus:outline-none focus:ring-blue-500 focus:border-blue-500"
|
||||
placeholder="your@email.com"
|
||||
disabled={isLoading}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label htmlFor="password" className="block text-sm font-medium text-gray-700">
|
||||
Password
|
||||
</label>
|
||||
<input
|
||||
id="password"
|
||||
name="password"
|
||||
type="password"
|
||||
autoComplete="current-password"
|
||||
required
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm placeholder-gray-400 focus:outline-none focus:ring-blue-500 focus:border-blue-500"
|
||||
placeholder="••••••••"
|
||||
disabled={isLoading}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isLoading}
|
||||
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
{isLoading ? 'Signing in...' : 'Sign in with Email'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<p className="text-sm mt-6 text-gray-600">
|
||||
Don't have an account?{' '}
|
||||
<Link href="/register" className="font-medium text-blue-600 hover:text-blue-500">
|
||||
Register
|
||||
</Link>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { redirect } from 'next/navigation';
|
||||
|
||||
export default function Home() {
|
||||
redirect('/links');
|
||||
}
|
||||
redirect('/analytics');
|
||||
}
|
||||
195
app/register/page.tsx
Normal file
195
app/register/page.tsx
Normal file
@@ -0,0 +1,195 @@
|
||||
'use client';
|
||||
|
||||
import { useState, FormEvent } from 'react';
|
||||
import Link from 'next/link';
|
||||
import { useAuth } from '@/lib/auth';
|
||||
|
||||
export default function RegisterPage() {
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
const [confirmPassword, setConfirmPassword] = useState('');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const { signUp, signInWithGoogle } = useAuth();
|
||||
|
||||
// Handle registration form submission
|
||||
const handleSubmit = async (e: FormEvent) => {
|
||||
e.preventDefault();
|
||||
setError(null);
|
||||
|
||||
// Validate passwords
|
||||
if (password !== confirmPassword) {
|
||||
setError('Passwords do not match');
|
||||
return;
|
||||
}
|
||||
|
||||
// Password strength validation
|
||||
if (password.length < 6) {
|
||||
setError('Password must be at least 6 characters');
|
||||
return;
|
||||
}
|
||||
|
||||
setIsLoading(true);
|
||||
try {
|
||||
await signUp(email, password);
|
||||
// After successful registration, redirect to login page with email verification prompt
|
||||
} catch (error) {
|
||||
console.error('Registration error:', error);
|
||||
setError('Registration failed. Please try again later or use a different email');
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Handle Google registration/login
|
||||
const handleGoogleSignIn = async () => {
|
||||
setError(null);
|
||||
try {
|
||||
await signInWithGoogle();
|
||||
// Login flow will redirect to Google and then back to the application
|
||||
} catch (error) {
|
||||
console.error('Google sign in error:', error);
|
||||
setError('Google login failed. Please try again later');
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen bg-gray-100 dark:bg-gray-900">
|
||||
<div className="w-full max-w-md p-8 space-y-8 bg-white dark:bg-gray-800 rounded-lg shadow-md">
|
||||
<div className="text-center">
|
||||
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">Register</h1>
|
||||
<p className="mt-2 text-sm text-gray-600 dark:text-gray-400">
|
||||
Create your account to access the analytics dashboard
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Error message */}
|
||||
{error && (
|
||||
<div className="p-4 mb-4 text-sm text-red-700 bg-red-100 dark:bg-red-900 dark:text-red-200 rounded-lg">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form className="mt-8 space-y-6" onSubmit={handleSubmit}>
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<label htmlFor="email" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
Email Address
|
||||
</label>
|
||||
<input
|
||||
id="email"
|
||||
name="email"
|
||||
type="email"
|
||||
autoComplete="email"
|
||||
required
|
||||
value={email}
|
||||
onChange={(e) => setEmail(e.target.value)}
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm placeholder-gray-400 dark:placeholder-gray-500 dark:bg-gray-700 dark:text-white focus:outline-none focus:ring-blue-500 focus:border-blue-500"
|
||||
placeholder="your@email.com"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="password" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
Password
|
||||
</label>
|
||||
<input
|
||||
id="password"
|
||||
name="password"
|
||||
type="password"
|
||||
autoComplete="new-password"
|
||||
required
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm placeholder-gray-400 dark:placeholder-gray-500 dark:bg-gray-700 dark:text-white focus:outline-none focus:ring-blue-500 focus:border-blue-500"
|
||||
placeholder="********"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="confirmPassword" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
Confirm Password
|
||||
</label>
|
||||
<input
|
||||
id="confirmPassword"
|
||||
name="confirmPassword"
|
||||
type="password"
|
||||
autoComplete="new-password"
|
||||
required
|
||||
value={confirmPassword}
|
||||
onChange={(e) => setConfirmPassword(e.target.value)}
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm placeholder-gray-400 dark:placeholder-gray-500 dark:bg-gray-700 dark:text-white focus:outline-none focus:ring-blue-500 focus:border-blue-500"
|
||||
placeholder="********"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isLoading}
|
||||
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
{isLoading ? 'Registering...' : 'Register'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="relative">
|
||||
<div className="absolute inset-0 flex items-center">
|
||||
<div className="w-full border-t border-gray-300 dark:border-gray-600"></div>
|
||||
</div>
|
||||
<div className="relative flex justify-center text-sm">
|
||||
<span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">or</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleGoogleSignIn}
|
||||
className="w-full flex justify-center items-center py-2 px-4 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm text-sm font-medium text-gray-700 dark:text-gray-300 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
<svg
|
||||
className="h-5 w-5 mr-2"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
width="24"
|
||||
height="24"
|
||||
>
|
||||
<g transform="matrix(1, 0, 0, 1, 27.009001, -39.238998)">
|
||||
<path
|
||||
fill="#4285F4"
|
||||
d="M -3.264 51.509 C -3.264 50.719 -3.334 49.969 -3.454 49.239 L -14.754 49.239 L -14.754 53.749 L -8.284 53.749 C -8.574 55.229 -9.424 56.479 -10.684 57.329 L -10.684 60.329 L -6.824 60.329 C -4.564 58.239 -3.264 55.159 -3.264 51.509 Z"
|
||||
/>
|
||||
<path
|
||||
fill="#34A853"
|
||||
d="M -14.754 63.239 C -11.514 63.239 -8.804 62.159 -6.824 60.329 L -10.684 57.329 C -11.764 58.049 -13.134 58.489 -14.754 58.489 C -17.884 58.489 -20.534 56.379 -21.484 53.529 L -25.464 53.529 L -25.464 56.619 C -23.494 60.539 -19.444 63.239 -14.754 63.239 Z"
|
||||
/>
|
||||
<path
|
||||
fill="#FBBC05"
|
||||
d="M -21.484 53.529 C -21.734 52.809 -21.864 52.039 -21.864 51.239 C -21.864 50.439 -21.724 49.669 -21.484 48.949 L -21.484 45.859 L -25.464 45.859 C -26.284 47.479 -26.754 49.299 -26.754 51.239 C -26.754 53.179 -26.284 54.999 -25.464 56.619 L -21.484 53.529 Z"
|
||||
/>
|
||||
<path
|
||||
fill="#EA4335"
|
||||
d="M -14.754 43.989 C -12.984 43.989 -11.404 44.599 -10.154 45.789 L -6.734 42.369 C -8.804 40.429 -11.514 39.239 -14.754 39.239 C -19.444 39.239 -23.494 41.939 -25.464 45.859 L -21.484 48.949 C -20.534 46.099 -17.884 43.989 -14.754 43.989 Z"
|
||||
/>
|
||||
</g>
|
||||
</svg>
|
||||
Sign up with Google
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div className="mt-6 text-center">
|
||||
<p className="text-sm text-gray-600 dark:text-gray-400">
|
||||
Already have an account?{' '}
|
||||
<Link
|
||||
href="/login"
|
||||
className="font-medium text-blue-600 hover:text-blue-500 dark:text-blue-400 dark:hover:text-blue-300"
|
||||
>
|
||||
Log in
|
||||
</Link>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
484
app/utils/ipLocation.ts
Normal file
484
app/utils/ipLocation.ts
Normal file
@@ -0,0 +1,484 @@
|
||||
interface IpLocationData {
|
||||
ip: string;
|
||||
country_name: string;
|
||||
country_code: string;
|
||||
city: string;
|
||||
region: string;
|
||||
continent_code: string;
|
||||
continent_name: string;
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
timestamp?: number; // When this data was fetched
|
||||
}
|
||||
|
||||
// In-memory cache
|
||||
let locationCache: Record<string, IpLocationData> = {};
|
||||
|
||||
// Blacklist for IPs that failed to resolve multiple times
|
||||
let failedIPs: Record<string, { attempts: number, lastAttempt: number }> = {};
|
||||
|
||||
// Cache expiration time (30 days in milliseconds)
|
||||
const CACHE_EXPIRATION = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
// Max retries for a failed IP
|
||||
const MAX_RETRY_ATTEMPTS = 3;
|
||||
|
||||
// Retry timeout (24 hours in milliseconds)
|
||||
const RETRY_TIMEOUT = 24 * 60 * 60 * 1000;
|
||||
|
||||
// Max number of IPs to batch in a single request
|
||||
const MAX_BATCH_SIZE = 10;
|
||||
|
||||
/**
|
||||
* Initialize cache from localStorage
|
||||
*/
|
||||
const initializeCache = () => {
|
||||
if (typeof window === 'undefined') return;
|
||||
|
||||
try {
|
||||
// Load location cache
|
||||
const cachedData = localStorage.getItem('ip-location-cache');
|
||||
if (cachedData) {
|
||||
const parsedCache = JSON.parse(cachedData);
|
||||
|
||||
// Filter out expired entries
|
||||
const now = Date.now();
|
||||
const validEntries = Object.entries(parsedCache).filter(([, data]) => {
|
||||
const entry = data as IpLocationData;
|
||||
return entry.timestamp && now - entry.timestamp < CACHE_EXPIRATION;
|
||||
});
|
||||
|
||||
locationCache = Object.fromEntries(validEntries) as Record<string, IpLocationData>;
|
||||
console.log(`Loaded ${validEntries.length} IP locations from cache`);
|
||||
}
|
||||
|
||||
// Load failed IPs
|
||||
const failedIPsData = localStorage.getItem('ip-location-blacklist');
|
||||
if (failedIPsData) {
|
||||
const parsedFailedIPs = JSON.parse(failedIPsData);
|
||||
|
||||
// Filter out expired blacklist entries
|
||||
const now = Date.now();
|
||||
const validFailedEntries = Object.entries(parsedFailedIPs).filter(([, data]) => {
|
||||
const entry = data as { attempts: number, lastAttempt: number };
|
||||
// Keep entries that have max attempts or haven't timed out yet
|
||||
return entry.attempts >= MAX_RETRY_ATTEMPTS ||
|
||||
now - entry.lastAttempt < RETRY_TIMEOUT;
|
||||
});
|
||||
|
||||
failedIPs = Object.fromEntries(validFailedEntries) as Record<string, { attempts: number, lastAttempt: number }>;
|
||||
console.log(`Loaded ${validFailedEntries.length} blacklisted IPs`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load IP location cache:', error);
|
||||
// Reset cache if corrupted
|
||||
localStorage.removeItem('ip-location-cache');
|
||||
localStorage.removeItem('ip-location-blacklist');
|
||||
locationCache = {};
|
||||
failedIPs = {};
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Save cache to localStorage
|
||||
*/
|
||||
const saveCache = () => {
|
||||
if (typeof window === 'undefined') return;
|
||||
|
||||
try {
|
||||
localStorage.setItem('ip-location-cache', JSON.stringify(locationCache));
|
||||
} catch (error) {
|
||||
console.error('Failed to save IP location cache:', error);
|
||||
|
||||
// If localStorage is full, clear old entries
|
||||
if (error instanceof DOMException && error.name === 'QuotaExceededError') {
|
||||
// Clear older entries - keep newest 100
|
||||
const entries = Object.entries(locationCache)
|
||||
.sort((a, b) => {
|
||||
const timestampA = (a[1].timestamp || 0);
|
||||
const timestampB = (b[1].timestamp || 0);
|
||||
return timestampB - timestampA;
|
||||
})
|
||||
.slice(0, 100);
|
||||
|
||||
locationCache = Object.fromEntries(entries);
|
||||
localStorage.setItem('ip-location-cache', JSON.stringify(locationCache));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Save failed IPs to localStorage
|
||||
*/
|
||||
const saveFailedIPs = () => {
|
||||
if (typeof window === 'undefined') return;
|
||||
|
||||
try {
|
||||
localStorage.setItem('ip-location-blacklist', JSON.stringify(failedIPs));
|
||||
} catch (error) {
|
||||
console.error('Failed to save IP blacklist:', error);
|
||||
|
||||
// If localStorage is full, limit the size
|
||||
if (error instanceof DOMException && error.name === 'QuotaExceededError') {
|
||||
// Keep only IPs with max attempts
|
||||
const entries = Object.entries(failedIPs)
|
||||
.filter(([, data]) => data.attempts >= MAX_RETRY_ATTEMPTS);
|
||||
|
||||
failedIPs = Object.fromEntries(entries);
|
||||
localStorage.setItem('ip-location-blacklist', JSON.stringify(failedIPs));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if IP is a private/local address
|
||||
*/
|
||||
const isPrivateIP = (ip: string): boolean => {
|
||||
return (
|
||||
ip.startsWith('10.') ||
|
||||
ip.startsWith('192.168.') ||
|
||||
ip.startsWith('172.16.') ||
|
||||
ip.startsWith('172.17.') ||
|
||||
ip.startsWith('172.18.') ||
|
||||
ip.startsWith('172.19.') ||
|
||||
ip.startsWith('172.20.') ||
|
||||
ip.startsWith('172.21.') ||
|
||||
ip.startsWith('172.22.') ||
|
||||
ip.startsWith('127.') ||
|
||||
ip === 'localhost' ||
|
||||
ip === '::1'
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if an IP should be skipped (blacklisted)
|
||||
*/
|
||||
const shouldSkipIP = (ip: string): boolean => {
|
||||
// If not in failed list, don't skip
|
||||
if (!failedIPs[ip]) return false;
|
||||
|
||||
const now = Date.now();
|
||||
|
||||
// If reached max attempts, skip
|
||||
if (failedIPs[ip].attempts >= MAX_RETRY_ATTEMPTS) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// If hasn't been long enough since last attempt, skip
|
||||
if (now - failedIPs[ip].lastAttempt < RETRY_TIMEOUT) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Otherwise, we can try again
|
||||
return false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Mark IP as failed
|
||||
*/
|
||||
const markIPAsFailed = (ip: string): void => {
|
||||
const now = Date.now();
|
||||
|
||||
if (failedIPs[ip]) {
|
||||
failedIPs[ip] = {
|
||||
attempts: failedIPs[ip].attempts + 1,
|
||||
lastAttempt: now
|
||||
};
|
||||
} else {
|
||||
failedIPs[ip] = {
|
||||
attempts: 1,
|
||||
lastAttempt: now
|
||||
};
|
||||
}
|
||||
|
||||
saveFailedIPs();
|
||||
};
|
||||
|
||||
/**
|
||||
* Get location data for a single IP address
|
||||
*/
|
||||
const fetchSingleIP = async (ip: string): Promise<IpLocationData | null> => {
|
||||
// Skip blacklisted IPs
|
||||
if (shouldSkipIP(ip)) {
|
||||
console.log(`Skipping blacklisted IP: ${ip}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`https://ipapi.co/${ip}/json/`);
|
||||
|
||||
if (!response.ok) {
|
||||
console.error(`Error fetching location for IP ${ip}: ${response.statusText}`);
|
||||
markIPAsFailed(ip);
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.error) {
|
||||
console.error(`Error fetching location for IP ${ip}: ${data.reason}`);
|
||||
markIPAsFailed(ip);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Reset failed attempts if successful
|
||||
if (failedIPs[ip]) {
|
||||
delete failedIPs[ip];
|
||||
saveFailedIPs();
|
||||
}
|
||||
|
||||
const locationData: IpLocationData = {
|
||||
ip: data.ip,
|
||||
country_name: data.country_name || 'Unknown',
|
||||
country_code: data.country_code || 'UN',
|
||||
city: data.city || 'Unknown',
|
||||
region: data.region || 'Unknown',
|
||||
continent_code: data.continent_code || 'UN',
|
||||
continent_name: getContinentName(data.continent_code) || 'Unknown',
|
||||
latitude: data.latitude || 0,
|
||||
longitude: data.longitude || 0,
|
||||
timestamp: Date.now()
|
||||
};
|
||||
|
||||
return locationData;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching location for IP ${ip}:`, error);
|
||||
markIPAsFailed(ip);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Batch process multiple IPs at once using our own API endpoint
|
||||
* This is a placeholder - we'll create a server API route for this
|
||||
*/
|
||||
const fetchBatchIPs = async (ips: string[]): Promise<Record<string, IpLocationData | null>> => {
|
||||
try {
|
||||
// Filter out blacklisted IPs
|
||||
const validIPs = ips.filter(ip => !shouldSkipIP(ip));
|
||||
|
||||
if (validIPs.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const response = await fetch('/api/geo/batch', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ ips: validIPs }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Batch request failed: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const results = await response.json();
|
||||
|
||||
// Mark failed IPs from results
|
||||
for (const [ip, data] of Object.entries(results.data)) {
|
||||
if (!data) {
|
||||
markIPAsFailed(ip);
|
||||
} else if (failedIPs[ip]) {
|
||||
// Reset failed attempts if successful
|
||||
delete failedIPs[ip];
|
||||
}
|
||||
}
|
||||
|
||||
saveFailedIPs();
|
||||
return results.data;
|
||||
} catch (error) {
|
||||
console.error('Error in batch IP lookup:', error);
|
||||
|
||||
// Fallback to individual requests
|
||||
const results: Record<string, IpLocationData | null> = {};
|
||||
for (const ip of ips) {
|
||||
// Skip blacklisted IPs
|
||||
if (shouldSkipIP(ip)) {
|
||||
results[ip] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Add delays between requests to avoid rate limiting
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
results[ip] = await fetchSingleIP(ip);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle private IP addresses
|
||||
*/
|
||||
const getPrivateIPData = (ip: string): IpLocationData => ({
|
||||
ip,
|
||||
country_name: 'Local Network',
|
||||
country_code: 'LO',
|
||||
city: 'Local',
|
||||
region: 'Local',
|
||||
continent_code: 'LO',
|
||||
continent_name: 'Local',
|
||||
latitude: 0,
|
||||
longitude: 0,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
/**
|
||||
* Convert an IP address to location information
|
||||
* Individual lookup for a single IP
|
||||
*/
|
||||
export async function getLocationFromIP(ip: string): Promise<IpLocationData | null> {
|
||||
// Initialize cache from localStorage if needed
|
||||
if (Object.keys(locationCache).length === 0) {
|
||||
initializeCache();
|
||||
}
|
||||
|
||||
// Handle private IP addresses
|
||||
if (isPrivateIP(ip)) {
|
||||
const privateIPData = getPrivateIPData(ip);
|
||||
locationCache[ip] = privateIPData;
|
||||
return privateIPData;
|
||||
}
|
||||
|
||||
// Skip blacklisted IPs
|
||||
if (shouldSkipIP(ip)) {
|
||||
console.log(`Skipping blacklisted IP: ${ip}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Return from cache if available and not expired
|
||||
if (locationCache[ip]) {
|
||||
const cachedData = locationCache[ip];
|
||||
const now = Date.now();
|
||||
|
||||
// Return cached data if not expired
|
||||
if (cachedData.timestamp && now - cachedData.timestamp < CACHE_EXPIRATION) {
|
||||
return cachedData;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch new data
|
||||
const locationData = await fetchSingleIP(ip);
|
||||
|
||||
// Save to cache if successful
|
||||
if (locationData) {
|
||||
locationCache[ip] = locationData;
|
||||
saveCache();
|
||||
}
|
||||
|
||||
return locationData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch lookup for multiple IPs at once
|
||||
* More efficient than calling getLocationFromIP multiple times
|
||||
*/
|
||||
export async function getLocationsFromIPs(ips: string[]): Promise<Record<string, IpLocationData | null>> {
|
||||
// Initialize cache from localStorage if needed
|
||||
if (Object.keys(locationCache).length === 0) {
|
||||
initializeCache();
|
||||
}
|
||||
|
||||
// Filter out IPs that are already in cache and not expired
|
||||
const now = Date.now();
|
||||
const cachedResults: Record<string, IpLocationData> = {};
|
||||
const ipsToFetch: string[] = [];
|
||||
|
||||
for (const ip of ips) {
|
||||
// Handle private IPs
|
||||
if (isPrivateIP(ip)) {
|
||||
cachedResults[ip] = getPrivateIPData(ip);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip blacklisted IPs
|
||||
if (shouldSkipIP(ip)) {
|
||||
console.log(`Skipping blacklisted IP: ${ip}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check cache
|
||||
if (locationCache[ip] && locationCache[ip].timestamp &&
|
||||
now - locationCache[ip].timestamp < CACHE_EXPIRATION) {
|
||||
cachedResults[ip] = locationCache[ip];
|
||||
} else {
|
||||
ipsToFetch.push(ip);
|
||||
}
|
||||
}
|
||||
|
||||
// If all IPs were cached or blacklisted, return immediately
|
||||
if (ipsToFetch.length === 0) {
|
||||
return cachedResults;
|
||||
}
|
||||
|
||||
// Process IPs in batches to avoid overwhelming the API
|
||||
const results: Record<string, IpLocationData | null> = { ...cachedResults };
|
||||
|
||||
// Process in smaller batches (e.g., 10 IPs at a time)
|
||||
for (let i = 0; i < ipsToFetch.length; i += MAX_BATCH_SIZE) {
|
||||
const batchIPs = ipsToFetch.slice(i, i + MAX_BATCH_SIZE);
|
||||
|
||||
// Batch request
|
||||
const batchResults = await fetchBatchIPs(batchIPs);
|
||||
|
||||
// Update results and cache
|
||||
for (const [ip, data] of Object.entries(batchResults)) {
|
||||
results[ip] = data;
|
||||
if (data) {
|
||||
locationCache[ip] = data;
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated cache
|
||||
saveCache();
|
||||
|
||||
// Add delay between batches
|
||||
if (i + MAX_BATCH_SIZE < ipsToFetch.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get continent name from continent code
|
||||
*/
|
||||
function getContinentName(code?: string): string {
|
||||
if (!code) return 'Unknown';
|
||||
|
||||
const continents: Record<string, string> = {
|
||||
'AF': 'Africa',
|
||||
'AN': 'Antarctica',
|
||||
'AS': 'Asia',
|
||||
'EU': 'Europe',
|
||||
'NA': 'North America',
|
||||
'OC': 'Oceania',
|
||||
'SA': 'South America'
|
||||
};
|
||||
|
||||
return continents[code] || 'Unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get location information based on view mode
|
||||
*/
|
||||
export function getLocationByType(
|
||||
locationData: IpLocationData | null,
|
||||
viewMode: 'country' | 'city' | 'region' | 'continent'
|
||||
): string {
|
||||
if (!locationData) return 'Unknown';
|
||||
|
||||
switch (viewMode) {
|
||||
case 'country':
|
||||
return locationData.country_name || 'Unknown';
|
||||
case 'city':
|
||||
return locationData.city || 'Unknown';
|
||||
case 'region':
|
||||
return locationData.region || 'Unknown';
|
||||
case 'continent':
|
||||
return locationData.continent_name || 'Unknown';
|
||||
default:
|
||||
return 'Unknown';
|
||||
}
|
||||
}
|
||||
52
app/utils/store.ts
Normal file
52
app/utils/store.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
import { create } from 'zustand';
|
||||
import { persist } from 'zustand/middleware';
|
||||
|
||||
// Define interface for team, project and tag objects
|
||||
interface TeamData {
|
||||
team_id: string;
|
||||
team_name: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface ProjectData {
|
||||
project_id: string;
|
||||
project_name: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// 定义 ShortUrl 数据类型
|
||||
export interface ShortUrlData {
|
||||
id: string;
|
||||
externalId: string;
|
||||
slug: string;
|
||||
originalUrl: string;
|
||||
title?: string;
|
||||
shortUrl: string;
|
||||
teams?: TeamData[];
|
||||
projects?: ProjectData[];
|
||||
tags?: string[];
|
||||
createdAt?: string;
|
||||
domain?: string;
|
||||
}
|
||||
|
||||
// 定义 store 类型
|
||||
interface ShortUrlStore {
|
||||
selectedShortUrl: ShortUrlData | null;
|
||||
setSelectedShortUrl: (shortUrl: ShortUrlData | null) => void;
|
||||
clearSelectedShortUrl: () => void;
|
||||
}
|
||||
|
||||
// 创建 store 并使用 persist 中间件保存到 localStorage
|
||||
export const useShortUrlStore = create<ShortUrlStore>()(
|
||||
persist(
|
||||
(set) => ({
|
||||
selectedShortUrl: null,
|
||||
setSelectedShortUrl: (shortUrl) => set({ selectedShortUrl: shortUrl }),
|
||||
clearSelectedShortUrl: () => set({ selectedShortUrl: null }),
|
||||
}),
|
||||
{
|
||||
name: 'shorturl-storage', // localStorage 中的 key 名称
|
||||
partialize: (state) => ({ selectedShortUrl: state.selectedShortUrl }), // 只持久化 selectedShortUrl
|
||||
}
|
||||
)
|
||||
);
|
||||
59
app/utils/supabase.ts
Normal file
59
app/utils/supabase.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import { createClient, SupabaseClient } from "@supabase/supabase-js";
|
||||
import type { Database } from "@/types/supabase";
|
||||
|
||||
let supabase: SupabaseClient<Database> | null = null;
|
||||
|
||||
// 简单的存储适配器,使用localStorage
|
||||
const storageAdapter = {
|
||||
getItem: async (key: string) => {
|
||||
try {
|
||||
const item = localStorage.getItem(key);
|
||||
return item;
|
||||
} catch (error) {
|
||||
console.error("Storage get error:", error);
|
||||
return null;
|
||||
}
|
||||
},
|
||||
|
||||
setItem: async (key: string, value: string) => {
|
||||
try {
|
||||
localStorage.setItem(key, value);
|
||||
} catch (error) {
|
||||
console.error("Storage set error:", error);
|
||||
}
|
||||
},
|
||||
|
||||
removeItem: async (key: string) => {
|
||||
try {
|
||||
localStorage.removeItem(key);
|
||||
} catch (error) {
|
||||
console.error("Storage remove error:", error);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
export const getSupabaseClient = (): SupabaseClient<Database> => {
|
||||
if (!supabase) {
|
||||
if (!process.env.NEXT_PUBLIC_SUPABASE_URL || !process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY) {
|
||||
throw new Error('Missing Supabase environment variables');
|
||||
}
|
||||
|
||||
supabase = createClient<Database>(
|
||||
process.env.NEXT_PUBLIC_SUPABASE_URL,
|
||||
process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY,
|
||||
{
|
||||
db: { schema: "limq" },
|
||||
auth: {
|
||||
storage: storageAdapter,
|
||||
persistSession: true,
|
||||
autoRefreshToken: true,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
return supabase;
|
||||
};
|
||||
|
||||
export const clearSupabaseInstance = () => {
|
||||
supabase = null;
|
||||
};
|
||||
151
docs/swagger-setup.md
Normal file
151
docs/swagger-setup.md
Normal file
@@ -0,0 +1,151 @@
|
||||
# Setting up Swagger UI in Next.js
|
||||
|
||||
This guide explains how to set up Swagger UI in a Next.js application using route groups.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
The recommended directory structure for Swagger documentation:
|
||||
|
||||
```
|
||||
app/
|
||||
(swagger)/ # Route group for swagger-related pages
|
||||
swagger/ # Actual swagger route
|
||||
page.tsx # Swagger UI component
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
1. Add Swagger UI dependencies to your project:
|
||||
|
||||
```json
|
||||
{
|
||||
"dependencies": {
|
||||
"swagger-ui-react": "^5.12.0",
|
||||
"swagger-ui-dist": "^5.12.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/swagger-ui-react": "^4.18.3"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. Install webpack style loaders for handling Swagger UI CSS:
|
||||
|
||||
```bash
|
||||
pnpm add -D style-loader css-loader
|
||||
```
|
||||
|
||||
## Next.js Configuration
|
||||
|
||||
Create or update `next.config.js` to handle Swagger UI CSS:
|
||||
|
||||
```javascript
|
||||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {
|
||||
transpilePackages: ['swagger-ui-react'],
|
||||
webpack: (config) => {
|
||||
config.module.rules.push({
|
||||
test: /\.css$/,
|
||||
use: ['style-loader', 'css-loader'],
|
||||
});
|
||||
return config;
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = nextConfig;
|
||||
```
|
||||
|
||||
## Swagger UI Component
|
||||
|
||||
Create `app/(swagger)/swagger/page.tsx`:
|
||||
|
||||
```typescript
|
||||
"use client";
|
||||
|
||||
import { useEffect } from 'react';
|
||||
import SwaggerUI from 'swagger-ui-react';
|
||||
import 'swagger-ui-react/swagger-ui.css';
|
||||
|
||||
export default function SwaggerPage() {
|
||||
useEffect(() => {
|
||||
document.title = 'API Documentation - ShortURL Analytics';
|
||||
}, []);
|
||||
|
||||
const swaggerConfig = {
|
||||
openapi: '3.0.0',
|
||||
info: {
|
||||
title: 'Your API Title',
|
||||
version: '1.0.0',
|
||||
description: 'API documentation',
|
||||
contact: {
|
||||
name: 'API Support',
|
||||
email: 'support@example.com',
|
||||
},
|
||||
license: {
|
||||
name: 'MIT',
|
||||
url: 'https://opensource.org/licenses/MIT',
|
||||
},
|
||||
},
|
||||
// ... your API configuration
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="container mx-auto px-4 py-8">
|
||||
<div className="mb-8">
|
||||
<h1 className="text-3xl font-bold mb-2">API Documentation</h1>
|
||||
<p className="text-gray-600">
|
||||
Explore and test the API endpoints using the interactive documentation below.
|
||||
</p>
|
||||
</div>
|
||||
<SwaggerUI spec={swaggerConfig} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Route Groups**: Use route groups `(groupname)` to organize related pages without affecting the URL structure.
|
||||
|
||||
2. **API Documentation**:
|
||||
- Add detailed descriptions for all endpoints
|
||||
- Include parameter descriptions and constraints
|
||||
- Define response schemas
|
||||
- Document error responses
|
||||
- Use appropriate data formats (UUID, URI, etc.)
|
||||
- Group related endpoints using tags
|
||||
|
||||
3. **Swagger Configuration**:
|
||||
- Add contact information
|
||||
- Include license details
|
||||
- Set appropriate servers configuration
|
||||
- Define required fields
|
||||
- Add parameter validations (min/max values)
|
||||
|
||||
## Common Issues
|
||||
|
||||
1. **Route Conflicts**: Avoid parallel routes that resolve to the same path. For example, don't have both `app/swagger/page.tsx` and `app/(group)/swagger/page.tsx` as they would conflict.
|
||||
|
||||
2. **CSS Loading**: Make sure to:
|
||||
- Import Swagger UI CSS
|
||||
- Configure webpack in `next.config.js`
|
||||
- Use the `"use client"` directive as Swagger UI is a client-side component
|
||||
|
||||
3. **React Version Compatibility**: Be aware of potential peer dependency warnings between Swagger UI React and your React version. You might need to use `--legacy-peer-deps` or adjust your React version accordingly.
|
||||
|
||||
## Accessing the Documentation
|
||||
|
||||
After setup, your Swagger documentation will be available at `/swagger` in your application. The UI provides:
|
||||
- Interactive API documentation
|
||||
- Request/response examples
|
||||
- Try-it-out functionality
|
||||
- Schema definitions
|
||||
- Error responses
|
||||
|
||||
## Maintenance
|
||||
|
||||
Keep your Swagger documentation up-to-date by:
|
||||
- Updating the OpenAPI specification when adding or modifying endpoints
|
||||
- Maintaining accurate parameter descriptions
|
||||
- Keeping example values relevant
|
||||
- Updating response schemas when data structures change
|
||||
182
lib/analytics.ts
182
lib/analytics.ts
@@ -1,8 +1,16 @@
|
||||
import { executeQuery, executeQuerySingle, buildFilter, buildPagination, buildOrderBy } from './clickhouse';
|
||||
import type { Event, EventsSummary, TimeSeriesData, GeoData, DeviceAnalytics, DeviceType } from './types';
|
||||
|
||||
// 获取事件列表
|
||||
export async function getEvents(params: {
|
||||
// 时间粒度枚举
|
||||
export enum TimeGranularity {
|
||||
HOUR = 'hour',
|
||||
DAY = 'day',
|
||||
WEEK = 'week',
|
||||
MONTH = 'month'
|
||||
}
|
||||
|
||||
// 事件查询参数类型
|
||||
export interface EventsQueryParams {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
eventType?: string;
|
||||
@@ -11,11 +19,18 @@ export async function getEvents(params: {
|
||||
userId?: string;
|
||||
teamId?: string;
|
||||
projectId?: string;
|
||||
teamIds?: string[];
|
||||
projectIds?: string[];
|
||||
tagIds?: string[];
|
||||
subpath?: string;
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
sortBy?: string;
|
||||
sortOrder?: 'asc' | 'desc';
|
||||
}): Promise<{ events: Event[]; total: number }> {
|
||||
}
|
||||
|
||||
// 获取事件列表
|
||||
export async function getEvents(params: EventsQueryParams): Promise<{ events: Event[]; total: number }> {
|
||||
const filter = buildFilter(params);
|
||||
const pagination = buildPagination(params.page, params.pageSize);
|
||||
const orderBy = buildOrderBy(params.sortBy, params.sortOrder);
|
||||
@@ -49,14 +64,20 @@ export async function getEventsSummary(params: {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
linkId?: string;
|
||||
teamIds?: string[];
|
||||
projectIds?: string[];
|
||||
tagIds?: string[];
|
||||
subpath?: string;
|
||||
}): Promise<EventsSummary> {
|
||||
console.log('getEventsSummary received params:', params);
|
||||
const filter = buildFilter(params);
|
||||
console.log('getEventsSummary built filter:', filter);
|
||||
|
||||
// 获取基本统计数据
|
||||
const baseQuery = `
|
||||
SELECT
|
||||
count() as totalEvents,
|
||||
uniq(visitor_id) as uniqueVisitors,
|
||||
uniq(ip_address) as uniqueVisitors,
|
||||
countIf(event_type = 'conversion') as totalConversions,
|
||||
avg(time_spent_sec) as averageTimeSpent,
|
||||
|
||||
@@ -91,57 +112,71 @@ export async function getEventsSummary(params: {
|
||||
ORDER BY count DESC
|
||||
`;
|
||||
|
||||
const [baseResult, browserResults, osResults] = await Promise.all([
|
||||
executeQuerySingle<{
|
||||
totalEvents: number;
|
||||
uniqueVisitors: number;
|
||||
totalConversions: number;
|
||||
averageTimeSpent: number;
|
||||
mobileCount: number;
|
||||
desktopCount: number;
|
||||
tabletCount: number;
|
||||
otherCount: number;
|
||||
}>(baseQuery),
|
||||
executeQuery<{ name: string; count: number }>(browserQuery),
|
||||
executeQuery<{ name: string; count: number }>(osQuery)
|
||||
]);
|
||||
|
||||
if (!baseResult) {
|
||||
throw new Error('Failed to get events summary');
|
||||
try {
|
||||
const [baseResult, browserResults, osResults] = await Promise.all([
|
||||
executeQuerySingle<{
|
||||
totalEvents: number;
|
||||
uniqueVisitors: number;
|
||||
totalConversions: number;
|
||||
averageTimeSpent: number;
|
||||
mobileCount: number;
|
||||
desktopCount: number;
|
||||
tabletCount: number;
|
||||
otherCount: number;
|
||||
}>(baseQuery),
|
||||
executeQuery<{ name: string; count: number }>(browserQuery),
|
||||
executeQuery<{ name: string; count: number }>(osQuery)
|
||||
]);
|
||||
|
||||
if (!baseResult) {
|
||||
throw new Error('Failed to get events summary');
|
||||
}
|
||||
|
||||
// 安全转换数字类型
|
||||
const safeNumber = (value: any): number => {
|
||||
if (value === null || value === undefined) return 0;
|
||||
const num = Number(value);
|
||||
return isNaN(num) ? 0 : num;
|
||||
};
|
||||
|
||||
// 计算百分比
|
||||
const calculatePercentage = (count: number, total: number) => {
|
||||
if (!total) return 0; // 防止除以零
|
||||
return Number(((count / total) * 100).toFixed(2));
|
||||
};
|
||||
|
||||
// 处理浏览器数据
|
||||
const browsers = browserResults.map(item => ({
|
||||
name: item.name || 'Unknown',
|
||||
count: safeNumber(item.count),
|
||||
percentage: calculatePercentage(safeNumber(item.count), safeNumber(baseResult.totalEvents))
|
||||
}));
|
||||
|
||||
// 处理操作系统数据
|
||||
const operatingSystems = osResults.map(item => ({
|
||||
name: item.name || 'Unknown',
|
||||
count: safeNumber(item.count),
|
||||
percentage: calculatePercentage(safeNumber(item.count), safeNumber(baseResult.totalEvents))
|
||||
}));
|
||||
|
||||
return {
|
||||
totalEvents: safeNumber(baseResult.totalEvents),
|
||||
uniqueVisitors: safeNumber(baseResult.uniqueVisitors),
|
||||
totalConversions: safeNumber(baseResult.totalConversions),
|
||||
averageTimeSpent: baseResult.averageTimeSpent ? Number(baseResult.averageTimeSpent.toFixed(2)) : 0,
|
||||
deviceTypes: {
|
||||
mobile: safeNumber(baseResult.mobileCount),
|
||||
desktop: safeNumber(baseResult.desktopCount),
|
||||
tablet: safeNumber(baseResult.tabletCount),
|
||||
other: safeNumber(baseResult.otherCount)
|
||||
},
|
||||
browsers,
|
||||
operatingSystems
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error in getEventsSummary:', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// 计算百分比
|
||||
const calculatePercentage = (count: number, total: number) =>
|
||||
Number(((count / total) * 100).toFixed(2));
|
||||
|
||||
// 处理浏览器数据
|
||||
const browsers = browserResults.map(item => ({
|
||||
name: item.name,
|
||||
count: item.count,
|
||||
percentage: calculatePercentage(item.count, baseResult.totalEvents)
|
||||
}));
|
||||
|
||||
// 处理操作系统数据
|
||||
const operatingSystems = osResults.map(item => ({
|
||||
name: item.name,
|
||||
count: item.count,
|
||||
percentage: calculatePercentage(item.count, baseResult.totalEvents)
|
||||
}));
|
||||
|
||||
return {
|
||||
totalEvents: baseResult.totalEvents,
|
||||
uniqueVisitors: baseResult.uniqueVisitors,
|
||||
totalConversions: baseResult.totalConversions,
|
||||
averageTimeSpent: Number(baseResult.averageTimeSpent.toFixed(2)),
|
||||
deviceTypes: {
|
||||
mobile: baseResult.mobileCount,
|
||||
desktop: baseResult.desktopCount,
|
||||
tablet: baseResult.tabletCount,
|
||||
other: baseResult.otherCount
|
||||
},
|
||||
browsers,
|
||||
operatingSystems
|
||||
};
|
||||
}
|
||||
|
||||
// 获取时间序列数据
|
||||
@@ -150,6 +185,10 @@ export async function getTimeSeriesData(params: {
|
||||
endTime: string;
|
||||
linkId?: string;
|
||||
granularity: 'hour' | 'day' | 'week' | 'month';
|
||||
teamIds?: string[];
|
||||
projectIds?: string[];
|
||||
tagIds?: string[];
|
||||
subpath?: string;
|
||||
}): Promise<TimeSeriesData[]> {
|
||||
const filter = buildFilter(params);
|
||||
|
||||
@@ -165,7 +204,7 @@ export async function getTimeSeriesData(params: {
|
||||
SELECT
|
||||
toStartOfInterval(event_time, INTERVAL ${interval}) as timestamp,
|
||||
count() as events,
|
||||
uniq(visitor_id) as visitors,
|
||||
uniq(ip_address) as visitors,
|
||||
countIf(event_type = 'conversion') as conversions
|
||||
FROM events
|
||||
${filter}
|
||||
@@ -181,23 +220,34 @@ export async function getGeoAnalytics(params: {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
linkId?: string;
|
||||
groupBy?: 'country' | 'city';
|
||||
groupBy?: 'country' | 'city' | 'region' | 'continent';
|
||||
teamIds?: string[];
|
||||
projectIds?: string[];
|
||||
tagIds?: string[];
|
||||
subpath?: string;
|
||||
}): Promise<GeoData[]> {
|
||||
const filter = buildFilter(params);
|
||||
const groupByField = 'ip_address'; // 暂时按 IP 地址分组
|
||||
|
||||
// Choose grouping field based on selected view
|
||||
let groupByField = 'country';
|
||||
if (params.groupBy === 'city') groupByField = 'city';
|
||||
else if (params.groupBy === 'region') groupByField = 'region';
|
||||
else if (params.groupBy === 'continent') groupByField = 'continent';
|
||||
else if (!params.groupBy) groupByField = 'ip_address'; // Default to IP address if no groupBy is specified
|
||||
|
||||
const query = `
|
||||
SELECT
|
||||
${groupByField} as location,
|
||||
COALESCE(${groupByField}, 'Unknown') as location,
|
||||
'' as area, /* Area column - empty for now */
|
||||
count() as visits,
|
||||
uniq(visitor_id) as visitors,
|
||||
uniq(ip_address) as visitors,
|
||||
count() * 100.0 / sum(count()) OVER () as percentage
|
||||
FROM events
|
||||
${filter}
|
||||
GROUP BY ${groupByField}
|
||||
GROUP BY location
|
||||
HAVING location != ''
|
||||
ORDER BY visits DESC
|
||||
LIMIT 10
|
||||
LIMIT 20
|
||||
`;
|
||||
|
||||
return executeQuery<GeoData>(query);
|
||||
@@ -208,6 +258,10 @@ export async function getDeviceAnalytics(params: {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
linkId?: string;
|
||||
teamIds?: string[];
|
||||
projectIds?: string[];
|
||||
tagIds?: string[];
|
||||
subpath?: string;
|
||||
}): Promise<DeviceAnalytics> {
|
||||
const filter = buildFilter(params);
|
||||
|
||||
@@ -263,8 +317,10 @@ export async function getDeviceAnalytics(params: {
|
||||
}
|
||||
|
||||
// 计算百分比
|
||||
const calculatePercentage = (count: number) =>
|
||||
Number(((count / totalResult.total) * 100).toFixed(2));
|
||||
const calculatePercentage = (count: number) => {
|
||||
if (!totalResult || totalResult.total === 0) return 0;
|
||||
return Number(((count / totalResult.total) * 100).toFixed(2));
|
||||
};
|
||||
|
||||
return {
|
||||
deviceTypes: deviceTypes.map(item => ({
|
||||
|
||||
50
lib/api.ts
Normal file
50
lib/api.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import supabase from './supabase';
|
||||
|
||||
// Define response type for API
|
||||
export interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
// Common function for authenticated API requests to LIMQ
|
||||
export async function limqRequest<T = unknown>(
|
||||
endpoint: string,
|
||||
method: 'GET' | 'POST' | 'PUT' | 'DELETE' = 'GET',
|
||||
data?: Record<string, unknown>
|
||||
): Promise<ApiResponse<T>> {
|
||||
// Get current session
|
||||
const { data: { session } } = await supabase.auth.getSession();
|
||||
|
||||
if (!session) {
|
||||
throw new Error('No active session. User must be authenticated.');
|
||||
}
|
||||
|
||||
const baseUrl = process.env.NEXT_PUBLIC_LIMQ_API;
|
||||
const url = `${baseUrl}${endpoint.startsWith('/') ? endpoint : '/' + endpoint}`;
|
||||
|
||||
const options: RequestInit = {
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${session.access_token}`
|
||||
},
|
||||
mode: 'cors'
|
||||
};
|
||||
|
||||
if (data && (method === 'POST' || method === 'PUT')) {
|
||||
options.body = JSON.stringify(data);
|
||||
}
|
||||
|
||||
const response = await fetch(url, options);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => null);
|
||||
throw new Error(
|
||||
errorData?.error || `Request failed with status ${response.status}`
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
266
lib/auth.tsx
Normal file
266
lib/auth.tsx
Normal file
@@ -0,0 +1,266 @@
|
||||
'use client';
|
||||
|
||||
import React, { createContext, useContext, useEffect, useState } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { Session, User } from '@supabase/supabase-js';
|
||||
import supabase from './supabase';
|
||||
import { limqRequest } from './api';
|
||||
|
||||
// 定义用户类型
|
||||
export type AuthUser = User | null;
|
||||
|
||||
// 定义验证上下文类型
|
||||
export type AuthContextType = {
|
||||
user: AuthUser;
|
||||
session: Session | null;
|
||||
isLoading: boolean;
|
||||
signIn: (email: string, password: string) => Promise<{ error?: unknown }>;
|
||||
signInWithGoogle: () => Promise<{ error?: unknown }>;
|
||||
signInWithGitHub: () => Promise<{ error?: unknown }>;
|
||||
signUp: (email: string, password: string) => Promise<void>;
|
||||
signOut: () => Promise<void>;
|
||||
};
|
||||
|
||||
// 创建验证上下文
|
||||
const AuthContext = createContext<AuthContextType | undefined>(undefined);
|
||||
|
||||
// 验证提供者组件
|
||||
export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
|
||||
const [user, setUser] = useState<AuthUser>(null);
|
||||
const [session, setSession] = useState<Session | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const router = useRouter();
|
||||
|
||||
// 初始化验证状态
|
||||
useEffect(() => {
|
||||
const getSession = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 尝试从Supabase获取会话
|
||||
const { data: { session }, error } = await supabase.auth.getSession();
|
||||
|
||||
if (error) {
|
||||
console.error('Error getting session:', error);
|
||||
return;
|
||||
}
|
||||
|
||||
setSession(session);
|
||||
setUser(session?.user || null);
|
||||
} catch (error) {
|
||||
console.error('Unexpected error during getSession:', error);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
getSession();
|
||||
|
||||
// 监听验证状态变化
|
||||
const { data: { subscription } } = supabase.auth.onAuthStateChange((_event, session) => {
|
||||
setSession(session);
|
||||
setUser(session?.user || null);
|
||||
});
|
||||
|
||||
// 清理函数
|
||||
return () => {
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// 登录函数
|
||||
const signIn = async (email: string, password: string) => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
console.log('尝试登录:', { email });
|
||||
|
||||
// 尝试通过Supabase登录
|
||||
const { data, error } = await supabase.auth.signInWithPassword({
|
||||
email,
|
||||
password,
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('登录出错:', error);
|
||||
return { error };
|
||||
}
|
||||
|
||||
setSession(data.session);
|
||||
setUser(data.user);
|
||||
|
||||
router.push('/analytics');
|
||||
return {};
|
||||
} catch (error) {
|
||||
console.error('登录过程出错:', error);
|
||||
return { error };
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Google登录函数
|
||||
const signInWithGoogle = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL,如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase登录Google
|
||||
const { error } = await supabase.auth.signInWithOAuth({
|
||||
provider: 'google',
|
||||
options: {
|
||||
redirectTo: `${siteUrl}/auth/callback`,
|
||||
},
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('Google登录出错:', error);
|
||||
return { error };
|
||||
}
|
||||
|
||||
return {}; // Return empty object when successful
|
||||
} catch (error) {
|
||||
console.error('Google登录过程出错:', error);
|
||||
return { error };
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
// GitHub登录函数
|
||||
const signInWithGitHub = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL,如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase登录GitHub
|
||||
const { error } = await supabase.auth.signInWithOAuth({
|
||||
provider: 'github',
|
||||
options: {
|
||||
redirectTo: `${siteUrl}/auth/callback`,
|
||||
},
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('GitHub login error:', error);
|
||||
return { error };
|
||||
}
|
||||
|
||||
return {}; // Return empty object when successful
|
||||
} catch (error) {
|
||||
console.error('GitHub login process error:', error);
|
||||
return { error };
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
// 注册函数
|
||||
const signUp = async (email: string, password: string) => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL,如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase注册
|
||||
const { error } = await supabase.auth.signUp({
|
||||
email,
|
||||
password,
|
||||
options: {
|
||||
emailRedirectTo: `${siteUrl}/auth/callback`,
|
||||
}
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('注册出错:', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// 注册成功后跳转到登录页面并显示确认消息
|
||||
router.push('/login?message=Registration successful! Please check your email to verify your account before logging in.');
|
||||
} catch (error) {
|
||||
console.error('注册过程出错:', error);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
// 登出函数
|
||||
const signOut = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 尝试通过Supabase登出
|
||||
const { error } = await supabase.auth.signOut();
|
||||
if (error) {
|
||||
console.error('登出出错:', error);
|
||||
throw error;
|
||||
}
|
||||
setSession(null);
|
||||
setUser(null);
|
||||
router.push('/login');
|
||||
} catch (error) {
|
||||
console.error('登出过程出错:', error);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const contextValue: AuthContextType = {
|
||||
user,
|
||||
session,
|
||||
isLoading,
|
||||
signIn,
|
||||
signInWithGoogle,
|
||||
signInWithGitHub,
|
||||
signUp,
|
||||
signOut,
|
||||
};
|
||||
|
||||
return (
|
||||
<AuthContext.Provider value={contextValue}>
|
||||
{children}
|
||||
</AuthContext.Provider>
|
||||
);
|
||||
};
|
||||
|
||||
// 自定义钩子
|
||||
export const useAuth = () => {
|
||||
const context = useContext(AuthContext);
|
||||
if (context === undefined) {
|
||||
throw new Error('useAuth must be used within an AuthProvider');
|
||||
}
|
||||
return context;
|
||||
};
|
||||
|
||||
// 受保护路由组件
|
||||
export const ProtectedRoute: React.FC<{ children: React.ReactNode }> = ({ children }) => {
|
||||
const { user, isLoading } = useAuth();
|
||||
const router = useRouter();
|
||||
|
||||
useEffect(() => {
|
||||
if (!isLoading && !user) {
|
||||
router.push('/login');
|
||||
}
|
||||
}, [user, isLoading, router]);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen">
|
||||
<div className="text-center">
|
||||
<div className="animate-spin rounded-full h-16 w-16 border-t-2 border-b-2 border-blue-500 mx-auto"></div>
|
||||
<p className="mt-4 text-lg text-gray-700 dark:text-gray-300">加载中...</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!user) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return <>{children}</>;
|
||||
};
|
||||
|
||||
export default AuthContext;
|
||||
@@ -1,12 +1,12 @@
|
||||
import { createClient } from '@clickhouse/client';
|
||||
import type { EventsQueryParams } from './types';
|
||||
import { EventsQueryParams } from './analytics';
|
||||
|
||||
// ClickHouse 客户端配置
|
||||
const clickhouse = createClient({
|
||||
url: process.env.CLICKHOUSE_URL || 'http://localhost:8123',
|
||||
username: process.env.CLICKHOUSE_USER || 'admin',
|
||||
password: process.env.CLICKHOUSE_PASSWORD || 'your_secure_password',
|
||||
database: process.env.CLICKHOUSE_DB || 'shorturl_analytics'
|
||||
url: process.env.CLICKHOUSE_URL,
|
||||
username: process.env.CLICKHOUSE_USER ,
|
||||
password: process.env.CLICKHOUSE_PASSWORD ,
|
||||
database: process.env.CLICKHOUSE_DATABASE
|
||||
});
|
||||
|
||||
// 构建日期过滤条件
|
||||
@@ -26,84 +26,140 @@ function buildDateFilter(startTime?: string, endTime?: string): string {
|
||||
|
||||
// 构建通用过滤条件
|
||||
export function buildFilter(params: Partial<EventsQueryParams>): string {
|
||||
console.log('buildFilter received params:', JSON.stringify(params));
|
||||
const filters = [];
|
||||
|
||||
// 时间范围过滤
|
||||
// 添加日期过滤条件
|
||||
if (params.startTime || params.endTime) {
|
||||
const dateFilter = buildDateFilter(params.startTime, params.endTime).replace('WHERE ', '');
|
||||
const dateFilter = buildDateFilter(params.startTime, params.endTime);
|
||||
if (dateFilter) {
|
||||
filters.push(dateFilter);
|
||||
filters.push(dateFilter.replace('WHERE ', ''));
|
||||
}
|
||||
}
|
||||
|
||||
// 事件类型过滤
|
||||
// 添加事件类型过滤条件
|
||||
if (params.eventType) {
|
||||
filters.push(`event_type = '${params.eventType}'`);
|
||||
}
|
||||
|
||||
// 链接ID过滤
|
||||
// 添加链接ID过滤条件
|
||||
if (params.linkId) {
|
||||
console.log('Adding link_id filter:', params.linkId);
|
||||
filters.push(`link_id = '${params.linkId}'`);
|
||||
}
|
||||
|
||||
// 链接短码过滤
|
||||
// 添加链接Slug过滤条件
|
||||
if (params.linkSlug) {
|
||||
filters.push(`link_slug = '${params.linkSlug}'`);
|
||||
}
|
||||
|
||||
// 用户ID过滤
|
||||
// 添加用户ID过滤条件
|
||||
if (params.userId) {
|
||||
filters.push(`user_id = '${params.userId}'`);
|
||||
}
|
||||
|
||||
// 团队ID过滤
|
||||
// 添加子路径过滤条件 - 使用更精确的匹配方式
|
||||
if (params.subpath && params.subpath.trim() !== '') {
|
||||
console.log('====== SUBPATH DEBUG ======');
|
||||
console.log('Raw subpath param:', params.subpath);
|
||||
|
||||
// 清理并准备subpath值
|
||||
let cleanSubpath = params.subpath.trim();
|
||||
// 移除开头的斜杠以便匹配
|
||||
if (cleanSubpath.startsWith('/')) {
|
||||
cleanSubpath = cleanSubpath.substring(1);
|
||||
}
|
||||
// 移除结尾的斜杠以便匹配
|
||||
if (cleanSubpath.endsWith('/')) {
|
||||
cleanSubpath = cleanSubpath.substring(0, cleanSubpath.length - 1);
|
||||
}
|
||||
|
||||
console.log('Cleaned subpath:', cleanSubpath);
|
||||
|
||||
// 使用正则表达式匹配URL中的第二个路径部分
|
||||
// 示例: 在 "https://abc.com/slug/subpath/" 中匹配 "subpath"
|
||||
const condition = `match(JSONExtractString(event_attributes, 'full_url'), '/[^/]+/${cleanSubpath}(/|\\\\?|$)')`;
|
||||
|
||||
console.log('Final SQL condition:', condition);
|
||||
console.log('==========================');
|
||||
|
||||
filters.push(condition);
|
||||
}
|
||||
|
||||
// 添加团队ID过滤条件
|
||||
if (params.teamId) {
|
||||
filters.push(`team_id = '${params.teamId}'`);
|
||||
}
|
||||
|
||||
// 项目ID过滤
|
||||
// 处理多个团队ID
|
||||
if (params.teamIds && params.teamIds.length > 0) {
|
||||
filters.push(`team_id IN (${params.teamIds.map(id => `'${id}'`).join(', ')})`);
|
||||
}
|
||||
|
||||
// 添加项目ID过滤条件
|
||||
if (params.projectId) {
|
||||
filters.push(`project_id = '${params.projectId}'`);
|
||||
}
|
||||
|
||||
// 处理多个项目ID
|
||||
if (params.projectIds && params.projectIds.length > 0) {
|
||||
filters.push(`project_id IN (${params.projectIds.map(id => `'${id}'`).join(', ')})`);
|
||||
}
|
||||
|
||||
// 处理标签过滤 - 使用LIKE来匹配标签字符串
|
||||
if (params.tagIds && params.tagIds.length > 0) {
|
||||
const tagConditions = params.tagIds.map(tag =>
|
||||
`link_tags LIKE '%${tag}%'`
|
||||
);
|
||||
filters.push(`(${tagConditions.join(' OR ')})`);
|
||||
}
|
||||
|
||||
return filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
|
||||
}
|
||||
|
||||
// 构建分页
|
||||
export function buildPagination(page?: number, pageSize?: number): string {
|
||||
const limit = pageSize || 20;
|
||||
const offset = ((page || 1) - 1) * limit;
|
||||
return `LIMIT ${limit} OFFSET ${offset}`;
|
||||
// 构建分页条件
|
||||
export function buildPagination(page: number = 1, pageSize: number = 20): string {
|
||||
const offset = (page - 1) * pageSize;
|
||||
return `LIMIT ${pageSize} OFFSET ${offset}`;
|
||||
}
|
||||
|
||||
// 构建排序
|
||||
export function buildOrderBy(sortBy?: string, sortOrder?: 'asc' | 'desc'): string {
|
||||
if (!sortBy) {
|
||||
return 'ORDER BY event_time DESC';
|
||||
}
|
||||
return `ORDER BY ${sortBy} ${sortOrder || 'desc'}`;
|
||||
// 构建排序条件
|
||||
export function buildOrderBy(sortBy: string = 'event_time', sortOrder: string = 'desc'): string {
|
||||
return `ORDER BY ${sortBy} ${sortOrder}`;
|
||||
}
|
||||
|
||||
// 执行查询并处理错误
|
||||
export async function executeQuery<T>(query: string): Promise<T[]> {
|
||||
// 执行查询
|
||||
export async function executeQuery(query: string) {
|
||||
console.log('Executing query:', query); // 查询日志
|
||||
try {
|
||||
const resultSet = await clickhouse.query({
|
||||
query,
|
||||
format: 'JSONEachRow'
|
||||
format: 'JSONEachRow',
|
||||
});
|
||||
|
||||
const rows = await resultSet.json<T>();
|
||||
return Array.isArray(rows) ? rows : [rows];
|
||||
const rows = await resultSet.json();
|
||||
return rows;
|
||||
} catch (error) {
|
||||
console.error('ClickHouse query error:', error);
|
||||
console.error('查询执行错误:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 执行查询并返回单个结果
|
||||
export async function executeQuerySingle<T>(query: string): Promise<T | null> {
|
||||
const results = await executeQuery<T>(query);
|
||||
return results.length > 0 ? results[0] : null;
|
||||
// 执行返回单一结果的查询
|
||||
export async function executeQuerySingle(query: string) {
|
||||
console.log('Executing single result query:', query); // 查询日志
|
||||
try {
|
||||
const resultSet = await clickhouse.query({
|
||||
query,
|
||||
format: 'JSONEachRow',
|
||||
});
|
||||
|
||||
const rows = await resultSet.json();
|
||||
return rows.length > 0 ? rows[0] : null;
|
||||
} catch (error) {
|
||||
console.error('单一结果查询执行错误:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export default clickhouse;
|
||||
65
lib/supabase.ts
Normal file
65
lib/supabase.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { createClient } from '@supabase/supabase-js';
|
||||
|
||||
// 从环境变量获取Supabase配置
|
||||
const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL || '';
|
||||
const supabaseAnonKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || process.env.SUPABASE_ANON_KEY || '';
|
||||
|
||||
console.log('Supabase Configuration Check:', {
|
||||
urlDefined: !!supabaseUrl,
|
||||
keyDefined: !!supabaseAnonKey,
|
||||
url: supabaseUrl,
|
||||
// 打印部分key以便调试
|
||||
keyPrefix: supabaseAnonKey ? supabaseAnonKey.substring(0, 20) + '...' : 'undefined',
|
||||
keyLength: supabaseAnonKey ? supabaseAnonKey.length : 0
|
||||
});
|
||||
|
||||
if (!supabaseUrl || !supabaseAnonKey) {
|
||||
console.error('Supabase URL and Anon Key are required');
|
||||
}
|
||||
|
||||
// 尝试解码JWT token并打印解码内容
|
||||
try {
|
||||
if (supabaseAnonKey) {
|
||||
const parts = supabaseAnonKey.split('.');
|
||||
if (parts.length === 3) {
|
||||
const payload = parts[1];
|
||||
const decoded = atob(payload);
|
||||
console.log('JWT Payload:', decoded);
|
||||
} else {
|
||||
console.error('Invalid JWT format, expected 3 parts but got:', parts.length);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('JWT解码失败:', error);
|
||||
}
|
||||
|
||||
// 创建Supabase客户端
|
||||
export const supabase = createClient(supabaseUrl, supabaseAnonKey, {
|
||||
auth: {
|
||||
persistSession: true,
|
||||
autoRefreshToken: true,
|
||||
detectSessionInUrl: true,
|
||||
}
|
||||
});
|
||||
|
||||
// 测试Supabase连接
|
||||
supabase.auth.onAuthStateChange((event, session) => {
|
||||
console.log(`Supabase auth event: ${event}`, session ? 'Session exists' : 'No session');
|
||||
if (session) {
|
||||
console.log('Current user:', session.user.email);
|
||||
}
|
||||
});
|
||||
|
||||
// 尝试执行健康检查
|
||||
async function checkSupabaseHealth() {
|
||||
try {
|
||||
const { data, error } = await supabase.from('_health').select('*').limit(1);
|
||||
console.log('Supabase health check:', error ? `Error: ${error.message}` : 'Success', data);
|
||||
} catch (error) {
|
||||
console.error('Supabase health check error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
checkSupabaseHealth();
|
||||
|
||||
export default supabase;
|
||||
13
lib/types.ts
13
lib/types.ts
@@ -24,6 +24,16 @@ export enum DeviceType {
|
||||
OTHER = 'other'
|
||||
}
|
||||
|
||||
// 标签类型
|
||||
export interface Tag {
|
||||
id: string;
|
||||
name: string;
|
||||
color?: string;
|
||||
type?: string;
|
||||
attributes?: Record<string, any>;
|
||||
team_id?: string;
|
||||
}
|
||||
|
||||
// API 响应基础接口
|
||||
export interface ApiResponse<T> {
|
||||
success: boolean;
|
||||
@@ -45,7 +55,10 @@ export interface EventsQueryParams {
|
||||
linkSlug?: string;
|
||||
userId?: string;
|
||||
teamId?: string;
|
||||
teamIds?: string[]; // 团队ID数组,支持多选
|
||||
projectId?: string;
|
||||
projectIds?: string[]; // 项目ID数组,支持多选
|
||||
tagIds?: string[]; // 标签ID数组,支持多选
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
sortBy?: string;
|
||||
|
||||
6
lib/utils.ts
Normal file
6
lib/utils.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
import { type ClassValue, clsx } from 'clsx';
|
||||
import { twMerge } from 'tailwind-merge';
|
||||
|
||||
export function cn(...inputs: ClassValue[]) {
|
||||
return twMerge(clsx(inputs));
|
||||
}
|
||||
22
middleware.ts
Normal file
22
middleware.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { createMiddlewareClient } from '@supabase/auth-helpers-nextjs';
|
||||
import { NextResponse } from 'next/server';
|
||||
import type { NextRequest } from 'next/server';
|
||||
|
||||
export async function middleware(req: NextRequest) {
|
||||
const res = NextResponse.next();
|
||||
|
||||
// Create a Supabase client configured to use cookies
|
||||
const supabase = createMiddlewareClient({ req, res });
|
||||
|
||||
// Refresh session if expired - required for Server Components
|
||||
await supabase.auth.getSession();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// Specify the paths where this middleware should run
|
||||
export const config = {
|
||||
matcher: [
|
||||
'/((?!_next/static|_next/image|favicon.ico).*)',
|
||||
],
|
||||
};
|
||||
@@ -3,7 +3,6 @@ import type { NextConfig } from "next";
|
||||
const nextConfig: NextConfig = {
|
||||
/* config options here */
|
||||
// 设置需要转译的包
|
||||
transpilePackages: [],
|
||||
|
||||
// 配置实验性选项
|
||||
experimental: {
|
||||
@@ -14,8 +13,8 @@ const nextConfig: NextConfig = {
|
||||
// 禁用严格模式,避免开发时重复渲染
|
||||
reactStrictMode: false,
|
||||
|
||||
// 设置输出为独立应用
|
||||
output: 'standalone',
|
||||
// 暂时禁用standalone输出模式,解决构建问题
|
||||
// output: 'standalone',
|
||||
|
||||
// 忽略ESLint错误,不会在构建时中断
|
||||
eslint: {
|
||||
|
||||
454
package-lock.json
generated
454
package-lock.json
generated
@@ -9,9 +9,16 @@
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@clickhouse/client": "^1.11.0",
|
||||
"@types/chart.js": "^2.9.41",
|
||||
"@types/recharts": "^1.8.29",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"chart.js": "^4.4.8",
|
||||
"date-fns": "^4.1.0",
|
||||
"next": "15.2.3",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0"
|
||||
"react-dom": "^19.0.0",
|
||||
"recharts": "^2.15.1",
|
||||
"uuid": "^10.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/eslintrc": "^3",
|
||||
@@ -38,6 +45,18 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/runtime": {
|
||||
"version": "7.27.0",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.0.tgz",
|
||||
"integrity": "sha512-VtPOkrdPHZsKc/clNqyi9WUA8TINkZ4cGk63UUE3u4pmB2k+ZMQRDuIOagv8UVd6j7k0T3+RRIb7beKTebNbcw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"regenerator-runtime": "^0.14.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@clickhouse/client": {
|
||||
"version": "1.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.11.0.tgz",
|
||||
@@ -654,6 +673,12 @@
|
||||
"url": "https://opencollective.com/libvips"
|
||||
}
|
||||
},
|
||||
"node_modules/@kurkle/color": {
|
||||
"version": "0.3.4",
|
||||
"resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz",
|
||||
"integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "0.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.7.tgz",
|
||||
@@ -1136,6 +1161,78 @@
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/chart.js": {
|
||||
"version": "2.9.41",
|
||||
"resolved": "https://registry.npmjs.org/@types/chart.js/-/chart.js-2.9.41.tgz",
|
||||
"integrity": "sha512-3dvkDvueckY83UyUXtJMalYoH6faOLkWQoaTlJgB4Djde3oORmNP0Jw85HtzTuXyliUHcdp704s0mZFQKio/KQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"moment": "^2.10.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/d3-array": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz",
|
||||
"integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/d3-color": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz",
|
||||
"integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/d3-ease": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz",
|
||||
"integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/d3-interpolate": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz",
|
||||
"integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/d3-color": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/d3-path": {
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-1.0.11.tgz",
|
||||
"integrity": "sha512-4pQMp8ldf7UaB/gR8Fvvy69psNHkTpD/pVw3vmEi8iZAB9EPMBruB1JvHO4BIq9QkUUd2lV1F5YXpMNj7JPBpw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/d3-scale": {
|
||||
"version": "4.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz",
|
||||
"integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/d3-time": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/d3-shape": {
|
||||
"version": "1.3.12",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-1.3.12.tgz",
|
||||
"integrity": "sha512-8oMzcd4+poSLGgV0R1Q1rOlx/xdmozS4Xab7np0eamFFUYq71AU9pOCJEFnkXW2aI/oXdVYJzw6pssbSut7Z9Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/d3-path": "^1"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/d3-time": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz",
|
||||
"integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/d3-timer": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz",
|
||||
"integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
|
||||
@@ -1171,7 +1268,6 @@
|
||||
"version": "19.0.12",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.12.tgz",
|
||||
"integrity": "sha512-V6Ar115dBDrjbtXSrS+/Oruobc+qVbbUxDFC1RSbRqLt5SYvxxyIDrSC85RWml54g+jfNeEMZhEj7wW07ONQhA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"csstype": "^3.0.2"
|
||||
@@ -1187,6 +1283,22 @@
|
||||
"@types/react": "^19.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/recharts": {
|
||||
"version": "1.8.29",
|
||||
"resolved": "https://registry.npmjs.org/@types/recharts/-/recharts-1.8.29.tgz",
|
||||
"integrity": "sha512-ulKklaVsnFIIhTQsQw226TnOibrddW1qUQNFVhoQEyY1Z7FRQrNecFCGt7msRuJseudzE9czVawZb17dK/aPXw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/d3-shape": "^1",
|
||||
"@types/react": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/uuid": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
|
||||
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@typescript-eslint/eslint-plugin": {
|
||||
"version": "8.27.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.27.0.tgz",
|
||||
@@ -2003,12 +2115,33 @@
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/chart.js": {
|
||||
"version": "4.4.8",
|
||||
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.8.tgz",
|
||||
"integrity": "sha512-IkGZlVpXP+83QpMm4uxEiGqSI7jFizwVtF3+n5Pc3k7sMO+tkd0qxh2OzLhenM0K80xtmAONWGBn082EiBQSDA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@kurkle/color": "^0.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"pnpm": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/client-only": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
|
||||
"integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/clsx": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
|
||||
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/color": {
|
||||
"version": "4.2.3",
|
||||
"resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
|
||||
@@ -2080,9 +2213,129 @@
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
|
||||
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/d3-array": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
|
||||
"integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"internmap": "1 - 2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-color": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
|
||||
"integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-ease": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz",
|
||||
"integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==",
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-format": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz",
|
||||
"integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-interpolate": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
|
||||
"integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"d3-color": "1 - 3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-path": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
|
||||
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-scale": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
|
||||
"integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"d3-array": "2.10.0 - 3",
|
||||
"d3-format": "1 - 3",
|
||||
"d3-interpolate": "1.2.0 - 3",
|
||||
"d3-time": "2.1.1 - 3",
|
||||
"d3-time-format": "2 - 4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-shape": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
|
||||
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"d3-path": "^3.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-time": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz",
|
||||
"integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"d3-array": "2 - 3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-time-format": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz",
|
||||
"integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"d3-time": "1 - 3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/d3-timer": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz",
|
||||
"integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/damerau-levenshtein": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",
|
||||
@@ -2144,6 +2397,16 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/date-fns": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz",
|
||||
"integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/kossnocorp"
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.0",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
|
||||
@@ -2162,6 +2425,12 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/decimal.js-light": {
|
||||
"version": "2.5.1",
|
||||
"resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz",
|
||||
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/deep-is": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
|
||||
@@ -2228,6 +2497,16 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/dom-helpers": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz",
|
||||
"integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.8.7",
|
||||
"csstype": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
@@ -2877,6 +3156,12 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/eventemitter3": {
|
||||
"version": "4.0.7",
|
||||
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
|
||||
"integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
||||
@@ -2884,6 +3169,15 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-equals": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.2.2.tgz",
|
||||
"integrity": "sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-glob": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz",
|
||||
@@ -3345,6 +3639,15 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/internmap": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
|
||||
"integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/is-array-buffer": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz",
|
||||
@@ -3786,7 +4089,6 @@
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
@@ -4151,6 +4453,12 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash": {
|
||||
"version": "4.17.21",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
|
||||
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.merge": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
|
||||
@@ -4162,7 +4470,6 @@
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
|
||||
"integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"js-tokens": "^3.0.0 || ^4.0.0"
|
||||
@@ -4228,6 +4535,15 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/moment": {
|
||||
"version": "2.30.1",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
|
||||
"integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
@@ -4346,7 +4662,6 @@
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
@@ -4645,7 +4960,6 @@
|
||||
"version": "15.8.1",
|
||||
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
|
||||
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"loose-envify": "^1.4.0",
|
||||
@@ -4709,7 +5023,75 @@
|
||||
"version": "16.13.1",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
|
||||
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/react-smooth": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz",
|
||||
"integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fast-equals": "^5.0.1",
|
||||
"prop-types": "^15.8.1",
|
||||
"react-transition-group": "^4.4.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
|
||||
"react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/react-transition-group": {
|
||||
"version": "4.4.5",
|
||||
"resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
|
||||
"integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.5.5",
|
||||
"dom-helpers": "^5.0.1",
|
||||
"loose-envify": "^1.4.0",
|
||||
"prop-types": "^15.6.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=16.6.0",
|
||||
"react-dom": ">=16.6.0"
|
||||
}
|
||||
},
|
||||
"node_modules/recharts": {
|
||||
"version": "2.15.1",
|
||||
"resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.1.tgz",
|
||||
"integrity": "sha512-v8PUTUlyiDe56qUj82w/EDVuzEFXwEHp9/xOowGAZwfLjB9uAy3GllQVIYMWF6nU+qibx85WF75zD7AjqoT54Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"clsx": "^2.0.0",
|
||||
"eventemitter3": "^4.0.1",
|
||||
"lodash": "^4.17.21",
|
||||
"react-is": "^18.3.1",
|
||||
"react-smooth": "^4.0.4",
|
||||
"recharts-scale": "^0.4.4",
|
||||
"tiny-invariant": "^1.3.1",
|
||||
"victory-vendor": "^36.6.8"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
|
||||
"react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/recharts-scale": {
|
||||
"version": "0.4.5",
|
||||
"resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz",
|
||||
"integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"decimal.js-light": "^2.4.1"
|
||||
}
|
||||
},
|
||||
"node_modules/recharts/node_modules/react-is": {
|
||||
"version": "18.3.1",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
|
||||
"integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/reflect.getprototypeof": {
|
||||
@@ -4735,6 +5117,12 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/regenerator-runtime": {
|
||||
"version": "0.14.1",
|
||||
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
|
||||
"integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/regexp.prototype.flags": {
|
||||
"version": "1.5.4",
|
||||
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz",
|
||||
@@ -5353,6 +5741,12 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/tiny-invariant": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
|
||||
"integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tinyglobby": {
|
||||
"version": "0.2.12",
|
||||
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.12.tgz",
|
||||
@@ -5584,6 +5978,50 @@
|
||||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
|
||||
"integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/victory-vendor": {
|
||||
"version": "36.9.2",
|
||||
"resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz",
|
||||
"integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==",
|
||||
"license": "MIT AND ISC",
|
||||
"dependencies": {
|
||||
"@types/d3-array": "^3.0.3",
|
||||
"@types/d3-ease": "^3.0.0",
|
||||
"@types/d3-interpolate": "^3.0.1",
|
||||
"@types/d3-scale": "^4.0.2",
|
||||
"@types/d3-shape": "^3.1.0",
|
||||
"@types/d3-time": "^3.0.0",
|
||||
"@types/d3-timer": "^3.0.0",
|
||||
"d3-array": "^3.1.6",
|
||||
"d3-ease": "^3.0.1",
|
||||
"d3-interpolate": "^3.0.1",
|
||||
"d3-scale": "^4.0.2",
|
||||
"d3-shape": "^3.1.0",
|
||||
"d3-time": "^3.0.0",
|
||||
"d3-timer": "^3.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/victory-vendor/node_modules/@types/d3-shape": {
|
||||
"version": "3.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.7.tgz",
|
||||
"integrity": "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/d3-path": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
|
||||
23
package.json
23
package.json
@@ -3,7 +3,7 @@
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev --turbo",
|
||||
"dev": "next dev -p 3007",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
@@ -24,20 +24,39 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@clickhouse/client": "^1.11.0",
|
||||
"@radix-ui/react-icons": "^1.3.2",
|
||||
"@radix-ui/react-popover": "^1.1.6",
|
||||
"@radix-ui/react-select": "^2.1.6",
|
||||
"@radix-ui/themes": "^3.2.1",
|
||||
"@supabase/auth-helpers-nextjs": "^0.10.0",
|
||||
"@types/chart.js": "^2.9.41",
|
||||
"@types/recharts": "^1.8.29",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"chart.js": "^4.4.8",
|
||||
"clsx": "^2.1.1",
|
||||
"date-fns": "^4.1.0",
|
||||
"lucide-react": "^0.486.0",
|
||||
"next": "15.2.3",
|
||||
"process": "^0.11.10",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"uuid": "^10.0.0"
|
||||
"recharts": "^2.15.1",
|
||||
"tailwind-merge": "^3.1.0",
|
||||
"uuid": "^10.0.0",
|
||||
"zustand": "^5.0.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/eslintrc": "^3",
|
||||
"@supabase/supabase-js": "^2.49.4",
|
||||
"@tailwindcss/postcss": "^4",
|
||||
"@types/node": "^20",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"css-loader": "^7.1.2",
|
||||
"dotenv": "^16.4.7",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "15.2.3",
|
||||
"style-loader": "^4.0.0",
|
||||
"tailwindcss": "^4",
|
||||
"typescript": "^5"
|
||||
},
|
||||
|
||||
2799
pnpm-lock.yaml
generated
2799
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -1,225 +0,0 @@
|
||||
|
||||
获取所有表...
|
||||
数据库 limq 中找到以下表:
|
||||
- .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb
|
||||
- .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc
|
||||
- .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1
|
||||
- .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0
|
||||
- .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024
|
||||
- .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea
|
||||
- link_daily_stats
|
||||
- link_events
|
||||
- link_hourly_patterns
|
||||
- links
|
||||
- platform_distribution
|
||||
- project_daily_stats
|
||||
- projects
|
||||
- qr_scans
|
||||
- qrcode_daily_stats
|
||||
- qrcodes
|
||||
- sessions
|
||||
- team_daily_stats
|
||||
- team_members
|
||||
- teams
|
||||
|
||||
所有ClickHouse表:
|
||||
.inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb, .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc, .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1, .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0, .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024, .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea, link_daily_stats, link_events, link_hourly_patterns, links, platform_distribution, project_daily_stats, projects, qr_scans, qrcode_daily_stats, qrcodes, sessions, team_daily_stats, team_members, teams
|
||||
|
||||
获取表 .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb 的结构...
|
||||
|
||||
获取表 .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc 的结构...
|
||||
|
||||
获取表 .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1 的结构...
|
||||
|
||||
获取表 .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0 的结构...
|
||||
|
||||
获取表 .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024 的结构...
|
||||
|
||||
获取表 .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea 的结构...
|
||||
|
||||
获取表 link_daily_stats 的结构...
|
||||
表 link_daily_stats 的列:
|
||||
- date (Date, 无默认值)
|
||||
- link_id (String, 无默认值)
|
||||
- total_clicks (UInt64, 无默认值)
|
||||
- unique_visitors (UInt64, 无默认值)
|
||||
- unique_sessions (UInt64, 无默认值)
|
||||
- total_time_spent (UInt64, 无默认值)
|
||||
- avg_time_spent (Float64, 无默认值)
|
||||
- bounce_count (UInt64, 无默认值)
|
||||
- conversion_count (UInt64, 无默认值)
|
||||
- unique_referrers (UInt64, 无默认值)
|
||||
- mobile_count (UInt64, 无默认值)
|
||||
- tablet_count (UInt64, 无默认值)
|
||||
- desktop_count (UInt64, 无默认值)
|
||||
- qr_scan_count (UInt64, 无默认值)
|
||||
- total_conversion_value (Float64, 无默认值)
|
||||
|
||||
获取表 link_events 的结构...
|
||||
表 link_events 的列:
|
||||
- event_id (UUID, 默认值: generateUUIDv4())
|
||||
- event_time (DateTime64(3), 默认值: now64())
|
||||
- date (Date, 默认值: toDate(event_time))
|
||||
- link_id (String, 无默认值)
|
||||
- channel_id (String, 无默认值)
|
||||
- visitor_id (String, 无默认值)
|
||||
- session_id (String, 无默认值)
|
||||
- event_type (Enum8('click' = 1, 'redirect' = 2, 'conversion' = 3, 'error' = 4), 无默认值)
|
||||
- ip_address (String, 无默认值)
|
||||
- country (String, 无默认值)
|
||||
- city (String, 无默认值)
|
||||
- referrer (String, 无默认值)
|
||||
- utm_source (String, 无默认值)
|
||||
- utm_medium (String, 无默认值)
|
||||
- utm_campaign (String, 无默认值)
|
||||
- user_agent (String, 无默认值)
|
||||
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
|
||||
- browser (String, 无默认值)
|
||||
- os (String, 无默认值)
|
||||
- time_spent_sec (UInt32, 默认值: 0)
|
||||
- is_bounce (Bool, 默认值: true)
|
||||
- is_qr_scan (Bool, 默认值: false)
|
||||
- qr_code_id (String, 默认值: '')
|
||||
- conversion_type (Enum8('visit' = 1, 'stay' = 2, 'interact' = 3, 'signup' = 4, 'subscription' = 5, 'purchase' = 6), 默认值: 'visit')
|
||||
- conversion_value (Float64, 默认值: 0)
|
||||
- custom_data (String, 默认值: '{}')
|
||||
|
||||
获取表 link_hourly_patterns 的结构...
|
||||
表 link_hourly_patterns 的列:
|
||||
- date (Date, 无默认值)
|
||||
- hour (UInt8, 无默认值)
|
||||
- link_id (String, 无默认值)
|
||||
- visits (UInt64, 无默认值)
|
||||
- unique_visitors (UInt64, 无默认值)
|
||||
|
||||
获取表 links 的结构...
|
||||
表 links 的列:
|
||||
- link_id (String, 无默认值)
|
||||
- original_url (String, 无默认值)
|
||||
- created_at (DateTime64(3), 无默认值)
|
||||
- created_by (String, 无默认值)
|
||||
- title (String, 无默认值)
|
||||
- description (String, 无默认值)
|
||||
- tags (Array(String), 无默认值)
|
||||
- is_active (Bool, 默认值: true)
|
||||
- expires_at (Nullable(DateTime), 无默认值)
|
||||
- team_id (String, 默认值: '')
|
||||
- project_id (String, 默认值: '')
|
||||
|
||||
获取表 platform_distribution 的结构...
|
||||
表 platform_distribution 的列:
|
||||
- date (Date, 无默认值)
|
||||
- utm_source (String, 无默认值)
|
||||
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
|
||||
- visits (UInt64, 无默认值)
|
||||
- unique_visitors (UInt64, 无默认值)
|
||||
|
||||
获取表 project_daily_stats 的结构...
|
||||
表 project_daily_stats 的列:
|
||||
- date (Date, 无默认值)
|
||||
- project_id (String, 无默认值)
|
||||
- total_clicks (UInt64, 无默认值)
|
||||
- unique_visitors (UInt64, 无默认值)
|
||||
- conversion_count (UInt64, 无默认值)
|
||||
- links_used (UInt64, 无默认值)
|
||||
- qr_scan_count (UInt64, 无默认值)
|
||||
|
||||
获取表 projects 的结构...
|
||||
表 projects 的列:
|
||||
- project_id (String, 无默认值)
|
||||
- team_id (String, 无默认值)
|
||||
- name (String, 无默认值)
|
||||
- created_at (DateTime, 无默认值)
|
||||
- created_by (String, 无默认值)
|
||||
- description (String, 默认值: '')
|
||||
- is_archived (Bool, 默认值: false)
|
||||
- links_count (UInt32, 默认值: 0)
|
||||
- total_clicks (UInt64, 默认值: 0)
|
||||
- last_updated (DateTime, 默认值: now())
|
||||
|
||||
获取表 qr_scans 的结构...
|
||||
表 qr_scans 的列:
|
||||
- scan_id (UUID, 默认值: generateUUIDv4())
|
||||
- qr_code_id (String, 无默认值)
|
||||
- link_id (String, 无默认值)
|
||||
- scan_time (DateTime64(3), 无默认值)
|
||||
- visitor_id (String, 无默认值)
|
||||
- location (String, 无默认值)
|
||||
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
|
||||
- led_to_conversion (Bool, 默认值: false)
|
||||
|
||||
获取表 qrcode_daily_stats 的结构...
|
||||
表 qrcode_daily_stats 的列:
|
||||
- date (Date, 无默认值)
|
||||
- qr_code_id (String, 无默认值)
|
||||
- total_scans (UInt64, 无默认值)
|
||||
- unique_scanners (UInt64, 无默认值)
|
||||
- conversions (UInt64, 无默认值)
|
||||
- mobile_scans (UInt64, 无默认值)
|
||||
- tablet_scans (UInt64, 无默认值)
|
||||
- desktop_scans (UInt64, 无默认值)
|
||||
- unique_locations (UInt64, 无默认值)
|
||||
|
||||
获取表 qrcodes 的结构...
|
||||
表 qrcodes 的列:
|
||||
- qr_code_id (String, 无默认值)
|
||||
- link_id (String, 无默认值)
|
||||
- team_id (String, 无默认值)
|
||||
- project_id (String, 默认值: '')
|
||||
- name (String, 无默认值)
|
||||
- description (String, 默认值: '')
|
||||
- created_at (DateTime, 无默认值)
|
||||
- created_by (String, 无默认值)
|
||||
- updated_at (DateTime, 默认值: now())
|
||||
- qr_type (Enum8('standard' = 1, 'custom' = 2, 'dynamic' = 3), 默认值: 'standard')
|
||||
- image_url (String, 默认值: '')
|
||||
- design_config (String, 默认值: '{}')
|
||||
- is_active (Bool, 默认值: true)
|
||||
- total_scans (UInt64, 默认值: 0)
|
||||
- unique_scanners (UInt32, 默认值: 0)
|
||||
|
||||
获取表 sessions 的结构...
|
||||
表 sessions 的列:
|
||||
- session_id (String, 无默认值)
|
||||
- visitor_id (String, 无默认值)
|
||||
- link_id (String, 无默认值)
|
||||
- started_at (DateTime64(3), 无默认值)
|
||||
- last_activity (DateTime64(3), 无默认值)
|
||||
- ended_at (Nullable(DateTime64(3)), 无默认值)
|
||||
- duration_sec (UInt32, 默认值: 0)
|
||||
- session_pages (UInt8, 默认值: 1)
|
||||
- is_completed (Bool, 默认值: false)
|
||||
|
||||
获取表 team_daily_stats 的结构...
|
||||
表 team_daily_stats 的列:
|
||||
- date (Date, 无默认值)
|
||||
- team_id (String, 无默认值)
|
||||
- total_clicks (UInt64, 无默认值)
|
||||
- unique_visitors (UInt64, 无默认值)
|
||||
- conversion_count (UInt64, 无默认值)
|
||||
- links_used (UInt64, 无默认值)
|
||||
- qr_scan_count (UInt64, 无默认值)
|
||||
|
||||
获取表 team_members 的结构...
|
||||
表 team_members 的列:
|
||||
- team_id (String, 无默认值)
|
||||
- user_id (String, 无默认值)
|
||||
- role (Enum8('owner' = 1, 'admin' = 2, 'editor' = 3, 'viewer' = 4), 无默认值)
|
||||
- joined_at (DateTime, 默认值: now())
|
||||
- invited_by (String, 无默认值)
|
||||
- is_active (Bool, 默认值: true)
|
||||
- last_active (DateTime, 默认值: now())
|
||||
|
||||
获取表 teams 的结构...
|
||||
表 teams 的列:
|
||||
- team_id (String, 无默认值)
|
||||
- name (String, 无默认值)
|
||||
- created_at (DateTime, 无默认值)
|
||||
- created_by (String, 无默认值)
|
||||
- description (String, 默认值: '')
|
||||
- avatar_url (String, 默认值: '')
|
||||
- is_active (Bool, 默认值: true)
|
||||
- plan_type (Enum8('free' = 1, 'pro' = 2, 'enterprise' = 3), 无默认值)
|
||||
- members_count (UInt32, 默认值: 1)
|
||||
|
||||
ClickHouse数据库结构检查完成
|
||||
5
scripts/db/sql/clickhouse/add_domain_column.sql
Normal file
5
scripts/db/sql/clickhouse/add_domain_column.sql
Normal file
@@ -0,0 +1,5 @@
|
||||
-- 添加domain列到shorturl_analytics.shorturl表
|
||||
ALTER TABLE
|
||||
shorturl_analytics.shorturl
|
||||
ADD
|
||||
COLUMN IF NOT EXISTS domain Nullable(String) COMMENT '域名';
|
||||
9
scripts/db/sql/clickhouse/add_req_full_path.sql
Normal file
9
scripts/db/sql/clickhouse/add_req_full_path.sql
Normal file
@@ -0,0 +1,9 @@
|
||||
-- add_req_full_path.sql
|
||||
-- Add req_full_path column to the shorturl_analytics.events table
|
||||
ALTER TABLE
|
||||
shorturl_analytics.events
|
||||
ADD
|
||||
COLUMN IF NOT EXISTS req_full_path String COMMENT 'Full request path including query parameters';
|
||||
|
||||
-- Display the updated table structure
|
||||
DESCRIBE TABLE shorturl_analytics.events;
|
||||
41
scripts/db/sql/clickhouse/add_utm_fields.sql
Normal file
41
scripts/db/sql/clickhouse/add_utm_fields.sql
Normal file
@@ -0,0 +1,41 @@
|
||||
-- 添加缺失的UTM参数字段到shorturl_analytics.events表
|
||||
-- 创建日期: 2024-07-02
|
||||
-- 用途: 增强UTM参数追踪能力
|
||||
-- 添加utm_term字段 (用于跟踪付费搜索关键词)
|
||||
ALTER TABLE
|
||||
shorturl_analytics.events
|
||||
ADD
|
||||
COLUMN utm_term String DEFAULT '' AFTER utm_campaign;
|
||||
|
||||
-- 添加utm_content字段 (用于区分相同广告的不同版本或A/B测试)
|
||||
ALTER TABLE
|
||||
shorturl_analytics.events
|
||||
ADD
|
||||
COLUMN utm_content String DEFAULT '' AFTER utm_term;
|
||||
|
||||
-- 验证字段添加成功
|
||||
DESCRIBE TABLE shorturl_analytics.events;
|
||||
|
||||
-- 示例查询: 查看UTM参数分析数据
|
||||
SELECT
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
utm_term,
|
||||
utm_content,
|
||||
COUNT(*) as clicks
|
||||
FROM
|
||||
shorturl_analytics.events
|
||||
WHERE
|
||||
event_type = 'click'
|
||||
AND utm_source != ''
|
||||
GROUP BY
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
utm_term,
|
||||
utm_content
|
||||
ORDER BY
|
||||
clicks DESC
|
||||
LIMIT
|
||||
10;
|
||||
46
scripts/db/sql/clickhouse/create_shorturl_table.sql
Normal file
46
scripts/db/sql/clickhouse/create_shorturl_table.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
-- 使用shorturl_analytics数据库
|
||||
USE shorturl_analytics;
|
||||
|
||||
-- 删除已存在的shorturl表
|
||||
DROP TABLE IF EXISTS shorturl_analytics.shorturl;
|
||||
|
||||
-- 创建shorturl表
|
||||
CREATE TABLE IF NOT EXISTS shorturl_analytics.shorturl (
|
||||
-- 短链接基本信息(来源于resources表)
|
||||
id String COMMENT '资源ID (resources.id)',
|
||||
external_id String COMMENT '外部ID (resources.external_id)',
|
||||
type String COMMENT '类型,值为shorturl',
|
||||
slug String COMMENT '短链接slug (存储在attributes中)',
|
||||
original_url String COMMENT '原始URL (存储在attributes中)',
|
||||
title String COMMENT '标题 (存储在attributes中)',
|
||||
description String COMMENT '描述 (存储在attributes中)',
|
||||
attributes String DEFAULT '{}' COMMENT '资源属性JSON',
|
||||
schema_version Int32 COMMENT 'Schema版本',
|
||||
-- 创建者信息
|
||||
creator_id String COMMENT '创建者ID (resources.creator_id)',
|
||||
creator_email String COMMENT '创建者邮箱 (来自users表)',
|
||||
creator_name String COMMENT '创建者名称 (来自users表)',
|
||||
-- 时间信息
|
||||
created_at DateTime64(3) COMMENT '创建时间 (resources.created_at)',
|
||||
updated_at DateTime64(3) COMMENT '更新时间 (resources.updated_at)',
|
||||
deleted_at Nullable(DateTime64(3)) COMMENT '删除时间 (resources.deleted_at)',
|
||||
-- 项目关联 (project_resources表)
|
||||
projects String DEFAULT '[]' COMMENT '项目关联信息数组。结构: [{project_id: String, project_name: String, project_description: String, assigned_at: DateTime64}]',
|
||||
-- 团队关联 (通过项目关联到团队)
|
||||
teams String DEFAULT '[]' COMMENT '团队关联信息数组。结构: [{team_id: String, team_name: String, team_description: String, via_project_id: String}]',
|
||||
-- 标签关联 (resource_tags表)
|
||||
tags String DEFAULT '[]' COMMENT '标签关联信息数组。结构: [{tag_id: String, tag_name: String, tag_type: String, created_at: DateTime64}]',
|
||||
-- QR码关联 (qr_code表)
|
||||
qr_codes String DEFAULT '[]' COMMENT 'QR码信息数组。结构: [{qr_id: String, scan_count: Int32, url: String, template_name: String, created_at: DateTime64}]',
|
||||
-- 渠道关联 (channel表)
|
||||
channels String DEFAULT '[]' COMMENT '渠道信息数组。结构: [{channel_id: String, channel_name: String, channel_path: String, is_user_created: Boolean}]',
|
||||
-- 收藏关联 (favorite表)
|
||||
favorites String DEFAULT '[]' COMMENT '收藏信息数组。结构: [{favorite_id: String, user_id: String, user_name: String, created_at: DateTime64}]',
|
||||
-- 自定义过期时间 (存储在attributes中)
|
||||
expires_at Nullable(DateTime64(3)) COMMENT '过期时间',
|
||||
-- 统计信息 (分析时聚合计算)
|
||||
click_count UInt32 DEFAULT 0 COMMENT '点击次数',
|
||||
unique_visitors UInt32 DEFAULT 0 COMMENT '唯一访问者数'
|
||||
) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY
|
||||
(id, created_at) SETTINGS index_granularity = 8192 COMMENT '用于存储所有shorturl类型资源的统一表,集成了相关联的项目、团队、标签、QR码、渠道和收藏信息';
|
||||
1
scripts/db/sql/clickhouse/truncate_events.sh
Normal file
1
scripts/db/sql/clickhouse/truncate_events.sh
Normal file
@@ -0,0 +1 @@
|
||||
./ch-query.sh -q "TRUNCATE TABLE shorturl_analytics.events"
|
||||
1
scripts/db/sql/clickhouse/truncate_shorturl.sh
Normal file
1
scripts/db/sql/clickhouse/truncate_shorturl.sh
Normal file
@@ -0,0 +1 @@
|
||||
./ch-query.sh -q "TRUNCATE TABLE shorturl_analytics.shorturl"
|
||||
@@ -1,364 +0,0 @@
|
||||
// Sync data from MongoDB trace table to ClickHouse events table
|
||||
import { getVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
interface MongoConfig {
|
||||
host: string;
|
||||
port: string;
|
||||
db: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
interface ClickHouseConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_database: string;
|
||||
clickhouse_url: string;
|
||||
}
|
||||
|
||||
interface TraceRecord {
|
||||
_id: ObjectId;
|
||||
slugId: ObjectId;
|
||||
label: string | null;
|
||||
ip: string;
|
||||
type: number;
|
||||
platform: string;
|
||||
platformOS: string;
|
||||
browser: string;
|
||||
browserVersion: string;
|
||||
url: string;
|
||||
createTime: number;
|
||||
}
|
||||
|
||||
export async function main(
|
||||
batch_size = 1000,
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
|
||||
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
|
||||
|
||||
// Set timeout
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// Get MongoDB and ClickHouse connection info
|
||||
let mongoConfig: MongoConfig;
|
||||
let clickhouseConfig: ClickHouseConfig;
|
||||
|
||||
try {
|
||||
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
|
||||
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
|
||||
|
||||
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
|
||||
} catch (error) {
|
||||
console.error("Failed to get config:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Build MongoDB connection URL
|
||||
let mongoUrl = "mongodb://";
|
||||
if (mongoConfig.username && mongoConfig.password) {
|
||||
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
|
||||
}
|
||||
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
|
||||
|
||||
// Connect to MongoDB
|
||||
const client = new MongoClient();
|
||||
try {
|
||||
await client.connect(mongoUrl);
|
||||
console.log("MongoDB connected successfully");
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const traceCollection = db.collection<TraceRecord>("trace");
|
||||
|
||||
// Build query conditions
|
||||
const query: Record<string, unknown> = {
|
||||
type: 1 // Only sync records with type 1
|
||||
};
|
||||
|
||||
// Count total records
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`Found ${totalRecords} records to sync`);
|
||||
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`Will process ${recordsToProcess} records`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("No records to sync, task completed");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "No records to sync"
|
||||
};
|
||||
}
|
||||
|
||||
// Check ClickHouse connection
|
||||
const checkClickHouseConnection = async (): Promise<boolean> => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("Skipping ClickHouse connection check");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("Testing ClickHouse connection...");
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
|
||||
},
|
||||
body: "SELECT 1",
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse connection test successful");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Check if records exist in ClickHouse
|
||||
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
|
||||
return records;
|
||||
}
|
||||
|
||||
try {
|
||||
const recordIds = records.map(record => record._id.toString());
|
||||
|
||||
const query = `
|
||||
SELECT event_id
|
||||
FROM ${clickhouseConfig.clickhouse_database}.events
|
||||
WHERE event_attributes LIKE '%"mongo_id":"%'
|
||||
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
const existingIds = new Set(result.data.map((row: any) => {
|
||||
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
|
||||
return matches ? matches[1] : null;
|
||||
}).filter(Boolean));
|
||||
|
||||
return records.filter(record => !existingIds.has(record._id.toString()));
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
|
||||
return skip_clickhouse_check ? records : [];
|
||||
}
|
||||
};
|
||||
|
||||
// Process records function
|
||||
const processRecords = async (records: TraceRecord[]) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
const newRecords = await checkExistingRecords(records);
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("All records already exist, skipping");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare ClickHouse insert data
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const eventTime = new Date(record.createTime).toISOString();
|
||||
return {
|
||||
event_time: eventTime,
|
||||
event_type: "click",
|
||||
event_attributes: JSON.stringify({
|
||||
mongo_id: record._id.toString(),
|
||||
original_type: record.type
|
||||
}),
|
||||
|
||||
// Link information
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: "",
|
||||
link_label: record.label || "",
|
||||
link_title: "",
|
||||
link_original_url: record.url || "",
|
||||
link_attributes: "{}",
|
||||
link_created_at: eventTime,
|
||||
link_expires_at: null,
|
||||
link_tags: "[]",
|
||||
|
||||
// User information (empty as not available in trace)
|
||||
user_id: "",
|
||||
user_name: "",
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
|
||||
// Team information (empty as not available in trace)
|
||||
team_id: "",
|
||||
team_name: "",
|
||||
team_attributes: "{}",
|
||||
|
||||
// Project information (empty as not available in trace)
|
||||
project_id: "",
|
||||
project_name: "",
|
||||
project_attributes: "{}",
|
||||
|
||||
// QR code information (empty as not available in trace)
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
|
||||
// Visitor information
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: `${record._id.toString()}-${record.createTime}`,
|
||||
ip_address: record.ip || "",
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "",
|
||||
os: record.platformOS || "",
|
||||
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
|
||||
|
||||
// Source information
|
||||
referrer: record.url || "",
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
|
||||
// Interaction information
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
conversion_type: "visit",
|
||||
conversion_value: 0
|
||||
};
|
||||
});
|
||||
|
||||
// Generate ClickHouse insert SQL
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
|
||||
FORMAT JSONEachRow
|
||||
${JSON.stringify(clickhouseData)}
|
||||
`;
|
||||
|
||||
try {
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: insertSQL,
|
||||
signal: AbortSignal.timeout(20000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// Check ClickHouse connection before processing
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
throw new Error("ClickHouse connection failed, cannot continue sync");
|
||||
}
|
||||
|
||||
// Process records in batches
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
|
||||
break;
|
||||
}
|
||||
|
||||
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
const records = await traceCollection.find(
|
||||
query,
|
||||
{
|
||||
allowDiskUse: true,
|
||||
sort: { createTime: 1 },
|
||||
skip: page * batch_size,
|
||||
limit: batch_size
|
||||
}
|
||||
).toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("No more records found, sync complete");
|
||||
break;
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
message: "Data sync completed"
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("Error during sync:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
stack: err instanceof Error ? err.stack : undefined
|
||||
};
|
||||
} finally {
|
||||
await client.close();
|
||||
console.log("MongoDB connection closed");
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ const config: Config = {
|
||||
"./components/**/*.{js,ts,jsx,tsx,mdx}",
|
||||
"./app/**/*.{js,ts,jsx,tsx,mdx}",
|
||||
],
|
||||
darkMode: 'class',
|
||||
darkMode: false,
|
||||
theme: {
|
||||
extend: {
|
||||
colors: {
|
||||
|
||||
72
test-supabase-login.mjs
Normal file
72
test-supabase-login.mjs
Normal file
@@ -0,0 +1,72 @@
|
||||
// 测试Supabase登录功能
|
||||
import { config } from 'dotenv';
|
||||
import { createClient } from '@supabase/supabase-js';
|
||||
|
||||
// 加载环境变量
|
||||
config({ path: '.env.local' });
|
||||
|
||||
async function testSupabaseLogin() {
|
||||
// 获取Supabase配置
|
||||
const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL;
|
||||
const supabaseKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || process.env.SUPABASE_ANON_KEY;
|
||||
|
||||
console.log('Supabase Configuration:');
|
||||
console.log('- URL defined:', !!supabaseUrl);
|
||||
console.log('- Key defined:', !!supabaseKey);
|
||||
console.log('- URL:', supabaseUrl);
|
||||
|
||||
if (!supabaseUrl || !supabaseKey) {
|
||||
console.error('缺少Supabase配置信息,请检查.env.local文件');
|
||||
return;
|
||||
}
|
||||
|
||||
// 创建Supabase客户端
|
||||
const supabase = createClient(supabaseUrl, supabaseKey);
|
||||
console.log('Supabase客户端创建成功');
|
||||
|
||||
try {
|
||||
// 尝试获取会话状态
|
||||
console.log('检查当前会话...');
|
||||
const { data: sessionData, error: sessionError } = await supabase.auth.getSession();
|
||||
|
||||
if (sessionError) {
|
||||
console.error('获取会话失败:', sessionError.message);
|
||||
} else {
|
||||
console.log('会话状态:', sessionData.session ? '已登录' : '未登录');
|
||||
}
|
||||
|
||||
// 尝试使用测试账户登录
|
||||
const testEmail = 'test@example.com';
|
||||
const testPassword = 'password123';
|
||||
|
||||
console.log(`\n尝试使用测试账户登录: ${testEmail}`);
|
||||
const { data, error } = await supabase.auth.signInWithPassword({
|
||||
email: testEmail,
|
||||
password: testPassword
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('登录失败:', error.message);
|
||||
|
||||
// 如果登录失败,尝试注册账户
|
||||
console.log('\n尝试注册测试账户...');
|
||||
const { data: signUpData, error: signUpError } = await supabase.auth.signUp({
|
||||
email: testEmail,
|
||||
password: testPassword
|
||||
});
|
||||
|
||||
if (signUpError) {
|
||||
console.error('注册失败:', signUpError.message);
|
||||
} else {
|
||||
console.log('注册成功:', signUpData);
|
||||
}
|
||||
} else {
|
||||
console.log('登录成功!');
|
||||
console.log('用户信息:', data.user);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('发生错误:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
testSupabaseLogin();
|
||||
1
types/react-simple-maps.d.ts
vendored
Normal file
1
types/react-simple-maps.d.ts
vendored
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
146
types/supabase.ts
Normal file
146
types/supabase.ts
Normal file
@@ -0,0 +1,146 @@
|
||||
export type Json =
|
||||
| string
|
||||
| number
|
||||
| boolean
|
||||
| null
|
||||
| { [key: string]: Json | undefined }
|
||||
| Json[]
|
||||
|
||||
export interface Database {
|
||||
public: {
|
||||
Tables: {
|
||||
teams: {
|
||||
Row: {
|
||||
id: string
|
||||
name: string
|
||||
description: string | null
|
||||
attributes: Json | null
|
||||
created_at: string | null
|
||||
updated_at: string | null
|
||||
deleted_at: string | null
|
||||
schema_version: number | null
|
||||
avatar_url: string | null
|
||||
}
|
||||
Insert: {
|
||||
id?: string
|
||||
name: string
|
||||
description?: string | null
|
||||
attributes?: Json | null
|
||||
created_at?: string | null
|
||||
updated_at?: string | null
|
||||
deleted_at?: string | null
|
||||
schema_version?: number | null
|
||||
avatar_url?: string | null
|
||||
}
|
||||
Update: {
|
||||
id?: string
|
||||
name?: string
|
||||
description?: string | null
|
||||
attributes?: Json | null
|
||||
created_at?: string | null
|
||||
updated_at?: string | null
|
||||
deleted_at?: string | null
|
||||
schema_version?: number | null
|
||||
avatar_url?: string | null
|
||||
}
|
||||
}
|
||||
team_membership: {
|
||||
Row: {
|
||||
id: string
|
||||
team_id: string
|
||||
user_id: string
|
||||
is_creator: boolean
|
||||
role: string
|
||||
}
|
||||
Insert: {
|
||||
id?: string
|
||||
team_id: string
|
||||
user_id: string
|
||||
is_creator?: boolean
|
||||
role: string
|
||||
}
|
||||
Update: {
|
||||
id?: string
|
||||
team_id?: string
|
||||
user_id?: string
|
||||
is_creator?: boolean
|
||||
role?: string
|
||||
}
|
||||
}
|
||||
}
|
||||
Views: {
|
||||
[_ in never]: never
|
||||
}
|
||||
Functions: {
|
||||
[_ in never]: never
|
||||
}
|
||||
Enums: {
|
||||
[_ in never]: never
|
||||
}
|
||||
}
|
||||
limq: {
|
||||
Tables: {
|
||||
teams: {
|
||||
Row: {
|
||||
id: string
|
||||
name: string
|
||||
description: string | null
|
||||
avatar_url: string | null
|
||||
attributes: Json | null
|
||||
created_at: string
|
||||
updated_at: string
|
||||
deleted_at: string | null
|
||||
}
|
||||
Insert: {
|
||||
id?: string
|
||||
name: string
|
||||
description?: string | null
|
||||
avatar_url?: string | null
|
||||
attributes?: Json | null
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
deleted_at?: string | null
|
||||
}
|
||||
Update: {
|
||||
id?: string
|
||||
name?: string
|
||||
description?: string | null
|
||||
avatar_url?: string | null
|
||||
attributes?: Json | null
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
deleted_at?: string | null
|
||||
}
|
||||
}
|
||||
team_membership: {
|
||||
Row: {
|
||||
id: string
|
||||
team_id: string
|
||||
user_id: string
|
||||
role: string
|
||||
created_at: string
|
||||
updated_at: string
|
||||
deleted_at: string | null
|
||||
}
|
||||
Insert: {
|
||||
id?: string
|
||||
team_id: string
|
||||
user_id: string
|
||||
role: string
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
deleted_at?: string | null
|
||||
}
|
||||
Update: {
|
||||
id?: string
|
||||
team_id?: string
|
||||
user_id?: string
|
||||
role?: string
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
deleted_at?: string | null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
2
windmill/scripts/.gitignore
vendored
Normal file
2
windmill/scripts/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
/node_modules
|
||||
/package-lock.json
|
||||
19
windmill/scripts/package.json
Normal file
19
windmill/scripts/package.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "scripts",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"date-fns": "^4.1.0",
|
||||
"dotenv": "^16.5.0",
|
||||
"fs-extra": "^11.3.0",
|
||||
"mongodb": "^6.16.0",
|
||||
"node-fetch": "^2.7.0"
|
||||
}
|
||||
}
|
||||
714
windmill/scripts/sync_mongo_to_clickhouse.js
Normal file
714
windmill/scripts/sync_mongo_to_clickhouse.js
Normal file
@@ -0,0 +1,714 @@
|
||||
// 从MongoDB的trace表同步数据到ClickHouse的events表
|
||||
//
|
||||
// 支持以下同步模式:
|
||||
// 1. 增量同步:基于上次同步状态,只同步新数据(默认模式)
|
||||
// 2. 自定义时间范围同步:通过指定开始时间和结束时间,同步特定时间范围内的数据
|
||||
// - 可以通过时间戳参数(startTime/endTime)指定范围
|
||||
// - 也可以通过日期字符串参数(startDate/endDate)指定范围,支持ISO格式或yyyy-MM-dd格式
|
||||
|
||||
const { MongoClient, ObjectId } = require('mongodb');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
// 同步状态键名和保存路径
|
||||
const SYNC_STATE_FILE = path.join(__dirname, 'mongo_sync_state.json');
|
||||
|
||||
// 直接使用配置值
|
||||
const mongoConfig = {
|
||||
url: "mongodb://10.0.1.41:27017",
|
||||
db: "main" // 注意:请替换为您的实际数据库名称
|
||||
};
|
||||
|
||||
const clickhouseConfig = {
|
||||
clickhouse_host: "10.0.1.60",
|
||||
clickhouse_port: "8123",
|
||||
clickhouse_user: "admin",
|
||||
clickhouse_password: "your_secure_password",
|
||||
clickhouse_database: "shorturl_analytics",
|
||||
clickhouse_url: "http://10.0.1.60:8123"
|
||||
};
|
||||
|
||||
// 封装本地读取变量函数
|
||||
async function getVariable(key) {
|
||||
try {
|
||||
if (key === 'f/shorturl_analytics/mongodb') {
|
||||
return mongoConfig;
|
||||
} else if (key === 'f/shorturl_analytics/clickhouse') {
|
||||
return clickhouseConfig;
|
||||
} else if (key === 'f/shorturl_analytics/mongo_sync_state') {
|
||||
if (fs.existsSync(SYNC_STATE_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(SYNC_STATE_FILE, 'utf8'));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error(`获取变量失败: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 封装本地保存变量函数
|
||||
async function setVariable(key, value) {
|
||||
try {
|
||||
if (key === 'f/shorturl_analytics/mongo_sync_state') {
|
||||
fs.writeFileSync(SYNC_STATE_FILE, JSON.stringify(value, null, 2));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`保存变量失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 日期字符串转时间戳工具函数(接受ISO字符串或yyyy-MM-dd格式)
|
||||
function dateToTimestamp(dateStr) {
|
||||
try {
|
||||
// 尝试直接解析完整的ISO日期字符串
|
||||
const date = new Date(dateStr);
|
||||
|
||||
// 检查是否为有效日期
|
||||
if (isNaN(date.getTime())) {
|
||||
// 尝试解析yyyy-MM-dd格式,默认设置为当天的00:00:00
|
||||
const parts = dateStr.split('-');
|
||||
if (parts.length === 3) {
|
||||
const year = parseInt(parts[0], 10);
|
||||
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
|
||||
const day = parseInt(parts[2], 10);
|
||||
|
||||
const dateObj = new Date(year, month, day, 0, 0, 0);
|
||||
return dateObj.getTime();
|
||||
}
|
||||
throw new Error(`无法解析日期字符串: ${dateStr}`);
|
||||
}
|
||||
|
||||
return date.getTime();
|
||||
} catch (err) {
|
||||
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从URL中提取UTM参数的函数,增强版
|
||||
function extractUtmParams(url, debug = false) {
|
||||
const defaultUtmParams = {
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
utm_term: "",
|
||||
utm_content: ""
|
||||
};
|
||||
|
||||
if (!url) return defaultUtmParams;
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 原始URL: ${url}`);
|
||||
}
|
||||
|
||||
// 准备一个解析后的参数对象
|
||||
const params = { ...defaultUtmParams };
|
||||
|
||||
// 尝试多种方法提取UTM参数
|
||||
|
||||
// 方法1: 使用URL对象解析
|
||||
try {
|
||||
// 先处理URL,确保是完整的URL格式
|
||||
let normalizedUrl = url;
|
||||
if (!url.match(/^https?:\/\//i)) {
|
||||
normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
|
||||
}
|
||||
|
||||
const urlObj = new URL(normalizedUrl);
|
||||
|
||||
// 读取URL参数
|
||||
if (urlObj.searchParams.has('utm_source'))
|
||||
params.utm_source = urlObj.searchParams.get('utm_source') || "";
|
||||
if (urlObj.searchParams.has('utm_medium'))
|
||||
params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
|
||||
if (urlObj.searchParams.has('utm_campaign'))
|
||||
params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
|
||||
if (urlObj.searchParams.has('utm_term'))
|
||||
params.utm_term = urlObj.searchParams.get('utm_term') || "";
|
||||
if (urlObj.searchParams.has('utm_content'))
|
||||
params.utm_content = urlObj.searchParams.get('utm_content') || "";
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
// 如果至少找到一个UTM参数,则返回
|
||||
if (params.utm_source || params.utm_medium || params.utm_campaign ||
|
||||
params.utm_term || params.utm_content) {
|
||||
return params;
|
||||
}
|
||||
} catch (err) {
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析失败,尝试正则表达式`);
|
||||
}
|
||||
}
|
||||
|
||||
// 方法2: 使用正则表达式提取参数
|
||||
// 使用正则表达式(最安全的方法,适用于任何格式)
|
||||
const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
|
||||
if (sourceMatch && sourceMatch[1]) {
|
||||
try {
|
||||
params.utm_source = decodeURIComponent(sourceMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_source = sourceMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
|
||||
if (mediumMatch && mediumMatch[1]) {
|
||||
try {
|
||||
params.utm_medium = decodeURIComponent(mediumMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_medium = mediumMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) {
|
||||
try {
|
||||
params.utm_campaign = decodeURIComponent(campaignMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_campaign = campaignMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
|
||||
if (termMatch && termMatch[1]) {
|
||||
try {
|
||||
params.utm_term = decodeURIComponent(termMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_term = termMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
|
||||
if (contentMatch && contentMatch[1]) {
|
||||
try {
|
||||
params.utm_content = decodeURIComponent(contentMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_content = contentMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
// 解析命令行参数
|
||||
function parseCommandLineArgs() {
|
||||
const args = {};
|
||||
process.argv.slice(2).forEach(arg => {
|
||||
if (arg.startsWith('--')) {
|
||||
const [key, value] = arg.substring(2).split('=');
|
||||
args[key] = value || true;
|
||||
}
|
||||
});
|
||||
return args;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseCommandLineArgs();
|
||||
|
||||
// 参数设置
|
||||
const batch_size = parseInt(args['batch-size'] || '1000');
|
||||
const max_records = parseInt(args['max-records'] || '9999999');
|
||||
const timeout_minutes = parseInt(args['timeout'] || '60');
|
||||
const skip_clickhouse_check = args['skip-clickhouse-check'] === 'true';
|
||||
const force_insert = args['force-insert'] !== 'false';
|
||||
const database_override = args['database'] || 'shorturl_analytics';
|
||||
const reset_sync_state = args['reset-sync-state'] === 'true';
|
||||
const debug_utm = args['debug-utm'] === 'true';
|
||||
const start_time = args['start-time'] ? parseInt(args['start-time']) : undefined;
|
||||
const end_time = args['end-time'] ? parseInt(args['end-time']) : undefined;
|
||||
const use_custom_time_range = args['use-custom-time-range'] === 'true';
|
||||
const start_date = args['start-date'];
|
||||
const end_date = args['end-date'];
|
||||
|
||||
const logWithTimestamp = (message) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
|
||||
let customStartTime = start_time;
|
||||
let customEndTime = end_time;
|
||||
let useCustomTimeRange = use_custom_time_range;
|
||||
|
||||
// 处理日期字符串参数,转换为时间戳
|
||||
if (start_date) {
|
||||
try {
|
||||
customStartTime = dateToTimestamp(start_date);
|
||||
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${customStartTime}`);
|
||||
useCustomTimeRange = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (end_date) {
|
||||
try {
|
||||
customEndTime = dateToTimestamp(end_date);
|
||||
// 如果是日期格式,设置为当天结束时间 (23:59:59.999)
|
||||
if (end_date.split('-').length === 3 && end_date.length <= 10) {
|
||||
customEndTime += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${customEndTime}`);
|
||||
} else {
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${customEndTime}`);
|
||||
}
|
||||
useCustomTimeRange = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式,不会检查记录是否已存在");
|
||||
}
|
||||
if (force_insert) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
|
||||
}
|
||||
if (reset_sync_state) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用重置同步状态,将从头开始同步数据");
|
||||
}
|
||||
if (debug_utm) {
|
||||
logWithTimestamp("已启用UTM参数调试日志");
|
||||
}
|
||||
if (useCustomTimeRange) {
|
||||
if (customStartTime) {
|
||||
logWithTimestamp(`已启用自定义时间范围:开始时间 ${new Date(customStartTime).toISOString()}`);
|
||||
}
|
||||
if (customEndTime) {
|
||||
logWithTimestamp(`已启用自定义时间范围:结束时间 ${new Date(customEndTime).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
// 检查是否超时
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`运行时间超过${timeout_minutes}分钟,暂停执行`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// 获取上次同步状态
|
||||
let lastSyncState = null;
|
||||
if (!reset_sync_state) {
|
||||
try {
|
||||
const rawSyncState = await getVariable("f/shorturl_analytics/mongo_sync_state");
|
||||
if (rawSyncState) {
|
||||
lastSyncState = rawSyncState;
|
||||
}
|
||||
} catch (error) {
|
||||
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
|
||||
}
|
||||
}
|
||||
|
||||
if (lastSyncState) {
|
||||
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
|
||||
if (lastSyncState.last_sync_id) {
|
||||
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("没有找到上次同步状态,将从头开始同步");
|
||||
}
|
||||
|
||||
// 连接MongoDB
|
||||
const client = new MongoClient(mongoConfig.url);
|
||||
try {
|
||||
await client.connect();
|
||||
console.log("MongoDB连接成功");
|
||||
|
||||
const db = client.db(mongoConfig.db);
|
||||
const traceCollection = db.collection("trace");
|
||||
const shortCollection = db.collection("short");
|
||||
|
||||
// 构建查询条件
|
||||
const query = {
|
||||
type: 1 // 只同步type为1的记录
|
||||
};
|
||||
|
||||
// 根据时间范围参数构建查询条件
|
||||
if (useCustomTimeRange) {
|
||||
// 使用自定义时间范围
|
||||
const timeQuery = {};
|
||||
|
||||
if (customStartTime) {
|
||||
timeQuery.$gte = customStartTime;
|
||||
logWithTimestamp(`将只同步createTime >= ${customStartTime} (${new Date(customStartTime).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
if (customEndTime) {
|
||||
timeQuery.$lte = customEndTime;
|
||||
logWithTimestamp(`将只同步createTime <= ${customEndTime} (${new Date(customEndTime).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 只有当至少指定了一个时间限制时才添加时间查询条件
|
||||
if (Object.keys(timeQuery).length > 0) {
|
||||
query.createTime = timeQuery;
|
||||
}
|
||||
}
|
||||
// 如果不使用自定义时间范围,且有上次同步状态,则只获取更新的记录
|
||||
else if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 使用上次同步时间作为过滤条件
|
||||
query.createTime = { $gt: lastSyncState.last_sync_time };
|
||||
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 计算总记录数
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`找到 ${totalRecords} 条新记录需要同步`);
|
||||
|
||||
// 限制此次处理的记录数量
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`本次将处理 ${recordsToProcess} 条记录`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("没有新记录需要同步,任务完成");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "没有新记录需要同步"
|
||||
};
|
||||
}
|
||||
|
||||
// 检查ClickHouse连接状态
|
||||
const checkClickHouseConnection = async () => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查,不测试连接");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("测试ClickHouse连接...");
|
||||
const clickhouseUrl = clickhouseConfig.clickhouse_url;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`,
|
||||
},
|
||||
body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse连接测试成功");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ ClickHouse连接测试失败,请启用skip_clickhouse_check=true参数来跳过连接检查");
|
||||
throw new Error("ClickHouse连接失败,无法继续同步");
|
||||
}
|
||||
|
||||
// 处理记录的函数
|
||||
const processRecords = async (records) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条记录...`);
|
||||
|
||||
// 强制使用所有记录,不检查重复
|
||||
const newRecords = records;
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
|
||||
|
||||
// 获取链接信息
|
||||
const slugIds = newRecords.map(record => new ObjectId(record.slugId));
|
||||
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
|
||||
const shortLinks = await shortCollection.find({
|
||||
_id: { $in: slugIds }
|
||||
}).toArray();
|
||||
|
||||
// 创建映射用于快速查找
|
||||
const shortLinksMap = new Map(shortLinks.map((link) => [link._id.toString(), link]));
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息,${newRecords.length - shortLinks.length} 条数据将使用占位符`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const eventTime = new Date(record.createTime);
|
||||
|
||||
// 获取对应的短链接信息
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString());
|
||||
|
||||
// 提取URL中的UTM参数
|
||||
if (debug_utm && record.url) {
|
||||
logWithTimestamp(`======== UTM参数调试 ========`);
|
||||
logWithTimestamp(`记录ID: ${record._id.toString()}`);
|
||||
logWithTimestamp(`原始URL: ${record.url}`);
|
||||
}
|
||||
|
||||
const utmParams = extractUtmParams(record.url || "", debug_utm);
|
||||
|
||||
if (debug_utm) {
|
||||
logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
|
||||
logWithTimestamp(`===========================`);
|
||||
}
|
||||
|
||||
// 保存提取的UTM参数和URL到event_attributes
|
||||
const eventAttributes = {
|
||||
mongo_id: record._id.toString(),
|
||||
url: record.url || "",
|
||||
...(record.url ? { raw_url: record.url } : {})
|
||||
};
|
||||
|
||||
// 转换MongoDB记录为ClickHouse格式,匹配ClickHouse表结构
|
||||
return {
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
event_type: record.type === 1 ? "visit" : "custom",
|
||||
event_attributes: JSON.stringify(eventAttributes),
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
|
||||
link_label: record.label || "",
|
||||
link_title: shortLink?.title || "unknown_title", // 使用占位符
|
||||
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
|
||||
link_created_at: shortLink?.createTime
|
||||
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
link_expires_at: shortLink?.expiresAt
|
||||
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: null,
|
||||
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
|
||||
user_id: shortLink?.user || "unknown_user", // 使用占位符
|
||||
user_name: "unknown_user", // 使用占位符
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
team_id: shortLink?.teamId || "unknown_team", // 使用占位符
|
||||
team_name: "unknown_team", // 使用占位符
|
||||
team_attributes: "{}",
|
||||
project_id: shortLink?.projectId || "unknown_project", // 使用占位符
|
||||
project_name: "unknown_project", // 使用占位符
|
||||
project_attributes: "{}",
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: record._id.toString() + "-" + record.createTime,
|
||||
ip_address: record.ip || "0.0.0.0", // 使用占位符
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "unknown", // 使用占位符
|
||||
os: record.platformOS || "unknown", // 使用占位符
|
||||
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
|
||||
referrer: record.url || "",
|
||||
utm_source: utmParams.utm_source || "",
|
||||
utm_medium: utmParams.utm_medium || "",
|
||||
utm_campaign: utmParams.utm_campaign || "",
|
||||
utm_term: utmParams.utm_term || "",
|
||||
utm_content: utmParams.utm_content || "",
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
conversion_type: "visit",
|
||||
conversion_value: 0,
|
||||
req_full_path: record.url || ""
|
||||
};
|
||||
});
|
||||
|
||||
// 生成ClickHouse插入SQL
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
|
||||
(event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title,
|
||||
link_original_url, link_attributes, link_created_at, link_expires_at, link_tags,
|
||||
user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes,
|
||||
project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes,
|
||||
visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent,
|
||||
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
|
||||
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
|
||||
VALUES ${clickhouseData.map(record => {
|
||||
// 确保所有字符串值都是字符串类型,并安全处理替换
|
||||
const safeReplace = (val) => {
|
||||
// 确保值是字符串,如果是null或undefined则使用空字符串
|
||||
const str = val === null || val === undefined ? "" : String(val);
|
||||
// 安全替换单引号
|
||||
return str.replace(/'/g, "''");
|
||||
};
|
||||
|
||||
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
|
||||
'${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}',
|
||||
'${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}',
|
||||
${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}',
|
||||
'${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}',
|
||||
'${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}',
|
||||
'${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}',
|
||||
'${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}',
|
||||
'${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}',
|
||||
'${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}',
|
||||
'${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}',
|
||||
'${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}',
|
||||
'${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}',
|
||||
'${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan},
|
||||
'${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
|
||||
}).join(", ")}
|
||||
`;
|
||||
|
||||
if (insertSQL.length === 0) {
|
||||
console.log("没有新记录需要插入");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 发送请求到ClickHouse
|
||||
const clickhouseUrl = clickhouseConfig.clickhouse_url;
|
||||
try {
|
||||
logWithTimestamp("发送插入请求到ClickHouse...");
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`
|
||||
},
|
||||
body: insertSQL,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`向ClickHouse插入数据失败: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// 批量处理记录
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
let lastSyncTime = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
|
||||
break;
|
||||
}
|
||||
|
||||
// 每批次都输出进度
|
||||
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
|
||||
|
||||
// 查询MongoDB数据
|
||||
const records = await traceCollection.find(query)
|
||||
.sort({ createTime: 1 })
|
||||
.skip(page * batch_size)
|
||||
.limit(batch_size)
|
||||
.toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("没有找到更多数据,同步结束");
|
||||
break;
|
||||
}
|
||||
|
||||
// 找到数据,开始处理
|
||||
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
|
||||
// 输出当前批次的部分数据信息
|
||||
if (records.length > 0) {
|
||||
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
|
||||
if (records.length > 1) {
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
|
||||
}
|
||||
|
||||
// 如果开启了调试,输出一些URL样本
|
||||
if (debug_utm) {
|
||||
const sampleSize = Math.min(5, records.length);
|
||||
logWithTimestamp(`URL样本 (前${sampleSize}条):`);
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
if (records[i].url) {
|
||||
logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
// 更新最后处理的记录时间和ID
|
||||
if (records.length > 0) {
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
|
||||
}
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
if (processedRecords > 0 && lastSyncTime > 0) {
|
||||
// 只在非自定义时间范围模式下更新同步状态
|
||||
if (!useCustomTimeRange) {
|
||||
// 创建新的同步状态,简化对象结构
|
||||
const newSyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords
|
||||
};
|
||||
|
||||
try {
|
||||
// 保存同步状态
|
||||
await setVariable("f/shorturl_analytics/mongo_sync_state", newSyncState);
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
logWithTimestamp(`更新同步状态失败: ${err.message},将继续执行`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("使用自定义时间范围模式,不更新全局同步状态");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
|
||||
message: useCustomTimeRange ? "自定义时间范围数据同步完成" : "数据同步完成",
|
||||
custom_time_range_used: useCustomTimeRange
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("同步过程中发生错误:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err.message,
|
||||
stack: err.stack
|
||||
};
|
||||
} finally {
|
||||
// 关闭MongoDB连接
|
||||
await client.close();
|
||||
console.log("MongoDB连接已关闭");
|
||||
}
|
||||
}
|
||||
|
||||
// 执行主函数
|
||||
main().then(result => {
|
||||
console.log("任务执行结果:", result);
|
||||
process.exit(result.success ? 0 : 1);
|
||||
}).catch(err => {
|
||||
console.error("执行出错:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
522
windmill/sync_mongo_short_to_postgres_short_url_shorturl.ts
Normal file
522
windmill/sync_mongo_short_to_postgres_short_url_shorturl.ts
Normal file
@@ -0,0 +1,522 @@
|
||||
// 从MongoDB的main.short表同步数据到PostgreSQL的short_url.shorturl表
|
||||
import { getVariable, setVariable, getResource } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
|
||||
|
||||
interface MongoConfig {
|
||||
host: string;
|
||||
port: string;
|
||||
db: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
interface PostgresConfig {
|
||||
host: string;
|
||||
port: number;
|
||||
database: string;
|
||||
user: string;
|
||||
password: string;
|
||||
schema: string;
|
||||
}
|
||||
|
||||
// 扩展ShortRecord接口以包含更多可能的字段
|
||||
interface ShortRecord {
|
||||
_id: ObjectId;
|
||||
origin: string;
|
||||
slug: string;
|
||||
domain: string | null;
|
||||
createTime: number | { $numberLong: string } | string;
|
||||
// 可选字段
|
||||
expiredAt?: number | { $numberLong: string } | string | null;
|
||||
expiredUrl?: string | null;
|
||||
password?: string | null;
|
||||
image?: string | null;
|
||||
title?: string | null;
|
||||
description?: string | null;
|
||||
}
|
||||
|
||||
interface SyncState {
|
||||
last_sync_time: number;
|
||||
records_synced: number;
|
||||
last_sync_id?: string;
|
||||
}
|
||||
|
||||
// 同步状态键名
|
||||
const SYNC_STATE_KEY = "f/limq/mongo_short_to_postgres_shorturl_shorturl_state";
|
||||
|
||||
export async function main(
|
||||
batch_size = 1000,
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_duplicate_check = false,
|
||||
force_insert = false,
|
||||
reset_sync_state = false,
|
||||
postgres_schema = "short_url", // 添加schema参数,允许运行时指定
|
||||
postgres_database = "postgres", // 添加数据库名称参数,默认为postgres
|
||||
domain = "upj.to" // 添加domain参数,允许用户指定域名
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到PostgreSQL的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
logWithTimestamp(`使用域名: ${domain}`);
|
||||
if (skip_duplicate_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过重复检查模式,不会检查记录是否已存在");
|
||||
}
|
||||
if (force_insert) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
|
||||
}
|
||||
if (reset_sync_state) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用重置同步状态,将从头开始同步数据");
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
// 检查是否超时
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
logWithTimestamp(`运行时间超过${timeout_minutes}分钟,暂停执行`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// 日期解析函数,处理不同格式的日期
|
||||
const parseDate = (dateValue: any): Date | null => {
|
||||
if (!dateValue) return null;
|
||||
|
||||
// 处理 MongoDB $numberLong 格式
|
||||
if (dateValue.$numberLong) {
|
||||
return new Date(Number(dateValue.$numberLong));
|
||||
}
|
||||
|
||||
// 处理普通时间戳
|
||||
if (typeof dateValue === 'number') {
|
||||
return new Date(dateValue);
|
||||
}
|
||||
|
||||
// 处理 ISO 字符串格式
|
||||
if (typeof dateValue === 'string') {
|
||||
const date = new Date(dateValue);
|
||||
return isNaN(date.getTime()) ? null : date;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
// 获取MongoDB和PostgreSQL的连接信息
|
||||
let mongoConfig: MongoConfig;
|
||||
let postgresConfig: PostgresConfig;
|
||||
|
||||
try {
|
||||
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
|
||||
if (typeof rawMongoConfig === "string") {
|
||||
try {
|
||||
mongoConfig = JSON.parse(rawMongoConfig);
|
||||
} catch (e) {
|
||||
console.error("MongoDB配置解析失败:", e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
mongoConfig = rawMongoConfig as MongoConfig;
|
||||
}
|
||||
|
||||
// 使用getResource获取PostgreSQL资源
|
||||
try {
|
||||
logWithTimestamp("正在获取PostgreSQL资源...");
|
||||
const resourceConfig = await getResource("f/limq/production_supabase");
|
||||
|
||||
// 将resource转换为PostgresConfig
|
||||
postgresConfig = {
|
||||
host: resourceConfig.host || "",
|
||||
port: Number(resourceConfig.port) || 5432,
|
||||
user: resourceConfig.user || "",
|
||||
password: resourceConfig.password || "",
|
||||
database: resourceConfig.database || postgres_database, // 使用提供的数据库名称作为备选
|
||||
schema: resourceConfig.schema || postgres_schema // 使用提供的schema作为备选
|
||||
};
|
||||
|
||||
// 检查并记录配置信息
|
||||
if (!postgresConfig.database || postgresConfig.database === "undefined") {
|
||||
postgresConfig.database = postgres_database;
|
||||
logWithTimestamp(`数据库名称未指定或为"undefined",使用提供的值: ${postgresConfig.database}`);
|
||||
}
|
||||
|
||||
if (!postgresConfig.schema || postgresConfig.schema === "undefined") {
|
||||
postgresConfig.schema = postgres_schema;
|
||||
logWithTimestamp(`Schema未指定或为"undefined",使用提供的值: ${postgresConfig.schema}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`PostgreSQL配置: 数据库=${postgresConfig.database}, Schema=${postgresConfig.schema}`);
|
||||
} catch (e) {
|
||||
console.error("获取PostgreSQL资源失败:", e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
console.log("MongoDB配置:", JSON.stringify({
|
||||
...mongoConfig,
|
||||
password: "****" // 隐藏密码
|
||||
}));
|
||||
console.log("PostgreSQL配置:", JSON.stringify({
|
||||
...postgresConfig,
|
||||
password: "****" // 隐藏密码
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error("获取配置失败:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// 获取上次同步状态
|
||||
let lastSyncState: SyncState | null = null;
|
||||
if (!reset_sync_state) {
|
||||
try {
|
||||
const rawSyncState = await getVariable(SYNC_STATE_KEY);
|
||||
if (rawSyncState) {
|
||||
if (typeof rawSyncState === "string") {
|
||||
try {
|
||||
lastSyncState = JSON.parse(rawSyncState);
|
||||
} catch (e) {
|
||||
logWithTimestamp(`解析上次同步状态失败: ${e}, 将从头开始同步`);
|
||||
}
|
||||
} else {
|
||||
lastSyncState = rawSyncState as SyncState;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
|
||||
}
|
||||
}
|
||||
|
||||
if (lastSyncState) {
|
||||
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
|
||||
if (lastSyncState.last_sync_id) {
|
||||
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("没有找到上次同步状态,将从头开始同步");
|
||||
}
|
||||
|
||||
// 构建MongoDB连接URL
|
||||
let mongoUrl = "mongodb://";
|
||||
if (mongoConfig.username && mongoConfig.password) {
|
||||
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
|
||||
}
|
||||
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
|
||||
|
||||
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
|
||||
|
||||
// 构建PostgreSQL连接URL
|
||||
const pgConnectionString = `postgres://${postgresConfig.user}:${postgresConfig.password}@${postgresConfig.host}:${postgresConfig.port}/${postgresConfig.database}`;
|
||||
console.log(`PostgreSQL连接URL: ${pgConnectionString.replace(/:[^:]*@/, ":****@")}`);
|
||||
|
||||
// 连接MongoDB
|
||||
const mongoClient = new MongoClient();
|
||||
let pgClient: Client | null = null;
|
||||
|
||||
try {
|
||||
await mongoClient.connect(mongoUrl);
|
||||
logWithTimestamp("MongoDB连接成功");
|
||||
|
||||
// 连接PostgreSQL
|
||||
pgClient = new Client(pgConnectionString);
|
||||
await pgClient.connect();
|
||||
logWithTimestamp("PostgreSQL连接成功");
|
||||
|
||||
// 确认PostgreSQL schema存在
|
||||
try {
|
||||
await pgClient.queryArray(`SELECT 1 FROM information_schema.schemata WHERE schema_name = '${postgresConfig.schema}'`);
|
||||
logWithTimestamp(`PostgreSQL schema '${postgresConfig.schema}' 已确认存在`);
|
||||
} catch (error) {
|
||||
logWithTimestamp(`检查PostgreSQL schema失败: ${error}`);
|
||||
throw new Error(`Schema '${postgresConfig.schema}' 可能不存在`);
|
||||
}
|
||||
|
||||
const db = mongoClient.database(mongoConfig.db);
|
||||
const shortCollection = db.collection<ShortRecord>("short");
|
||||
|
||||
// 构建查询条件,根据上次同步状态获取新记录
|
||||
const query: Record<string, unknown> = {};
|
||||
|
||||
// 如果有上次同步状态,则只获取更新的记录
|
||||
if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 使用上次同步时间作为过滤条件
|
||||
query.createTime = { $gt: lastSyncState.last_sync_time };
|
||||
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 计算总记录数
|
||||
const totalRecords = await shortCollection.countDocuments(query);
|
||||
logWithTimestamp(`找到 ${totalRecords} 条新记录需要同步`);
|
||||
|
||||
// 限制此次处理的记录数量
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
logWithTimestamp(`本次将处理 ${recordsToProcess} 条记录`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
logWithTimestamp("没有新记录需要同步,任务完成");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "没有新记录需要同步"
|
||||
};
|
||||
}
|
||||
|
||||
// 检查记录是否已经存在于PostgreSQL中
|
||||
const checkExistingRecords = async (records: ShortRecord[]): Promise<ShortRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
// 如果跳过重复检查或强制插入,则直接返回所有记录
|
||||
if (skip_duplicate_check || force_insert) {
|
||||
logWithTimestamp(`已跳过重复检查,准备处理所有 ${records.length} 条记录`);
|
||||
return records;
|
||||
}
|
||||
|
||||
logWithTimestamp(`正在检查 ${records.length} 条记录是否已存在于PostgreSQL中...`);
|
||||
|
||||
try {
|
||||
// 提取所有记录的slugs
|
||||
const slugs = records.map(record => record.slug);
|
||||
|
||||
// 查询PostgreSQL中是否已存在这些slugs
|
||||
const result = await pgClient!.queryArray(`
|
||||
SELECT slug FROM ${postgresConfig.schema}.shorturl
|
||||
WHERE slug = ANY($1::text[])
|
||||
`, [slugs]);
|
||||
|
||||
// 将已存在的slugs加入到集合中
|
||||
const existingSlugs = new Set<string>();
|
||||
for (const row of result.rows) {
|
||||
existingSlugs.add(row[0] as string);
|
||||
}
|
||||
|
||||
logWithTimestamp(`检测到 ${existingSlugs.size} 条记录已存在于PostgreSQL中`);
|
||||
|
||||
// 过滤出不存在的记录
|
||||
const newRecords = records.filter(record => !existingSlugs.has(record.slug));
|
||||
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
|
||||
|
||||
return newRecords;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`PostgreSQL查询出错: ${error.message}`);
|
||||
if (skip_duplicate_check) {
|
||||
logWithTimestamp("已启用跳过重复检查,将继续处理所有记录");
|
||||
return records;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 处理记录的函数
|
||||
const processRecords = async (records: ShortRecord[]) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条记录...`);
|
||||
|
||||
// 检查记录是否已存在
|
||||
let newRecords;
|
||||
try {
|
||||
newRecords = await checkExistingRecords(records);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
|
||||
if (!skip_duplicate_check && !force_insert) {
|
||||
throw error;
|
||||
}
|
||||
// 如果跳过检查或强制插入,则使用所有记录
|
||||
logWithTimestamp("将使用所有记录进行处理");
|
||||
newRecords = records;
|
||||
}
|
||||
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("所有记录都已存在,跳过处理");
|
||||
return 0;
|
||||
}
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
|
||||
|
||||
// 批量插入PostgreSQL
|
||||
try {
|
||||
// 开始事务
|
||||
await pgClient!.queryArray('BEGIN');
|
||||
|
||||
let insertedCount = 0;
|
||||
|
||||
// 由于参数可能很多,按小批次处理
|
||||
const smallBatchSize = 100;
|
||||
for (let i = 0; i < newRecords.length; i += smallBatchSize) {
|
||||
const batchRecords = newRecords.slice(i, i + smallBatchSize);
|
||||
|
||||
// 构造批量插入语句
|
||||
const placeholders = [];
|
||||
const values = [];
|
||||
let valueIndex = 1;
|
||||
|
||||
for (const record of batchRecords) {
|
||||
// 参考提供的字段处理方式处理数据
|
||||
const createdAt = parseDate(record.createTime);
|
||||
const updatedAt = createdAt; // 设置更新时间等于创建时间
|
||||
const fullShortUrl = `${domain}/${record.slug}`;
|
||||
|
||||
placeholders.push(`($${valueIndex}, $${valueIndex+1}, $${valueIndex+2}, $${valueIndex+3}, $${valueIndex+4}, $${valueIndex+5}, $${valueIndex+6}, $${valueIndex+7}, $${valueIndex+8}, $${valueIndex+9}, $${valueIndex+10}, $${valueIndex+11}, $${valueIndex+12})`);
|
||||
|
||||
values.push(
|
||||
record._id.toString(), // id
|
||||
record.slug, // slug
|
||||
domain, // domain (使用提供的域名)
|
||||
record.slug, // name (使用slug作为name)
|
||||
record.slug, // title (使用slug作为title)
|
||||
record.origin || '', // origin
|
||||
createdAt, // created_at
|
||||
updatedAt, // updated_at
|
||||
fullShortUrl, // full_short_url
|
||||
record.image || null, // image
|
||||
record.description || null, // description
|
||||
record.expiredUrl || null, // expired_url
|
||||
parseDate(record.expiredAt) // expired_at
|
||||
);
|
||||
|
||||
valueIndex += 13;
|
||||
}
|
||||
|
||||
const query = `
|
||||
INSERT INTO ${postgresConfig.schema}.shorturl
|
||||
(id, slug, domain, name, title, origin, created_at, updated_at, full_short_url, image, description, expired_url, expired_at)
|
||||
VALUES ${placeholders.join(', ')}
|
||||
`;
|
||||
|
||||
await pgClient!.queryArray(query, values);
|
||||
insertedCount += batchRecords.length;
|
||||
logWithTimestamp(`已插入 ${insertedCount}/${newRecords.length} 条记录`);
|
||||
}
|
||||
|
||||
// 提交事务
|
||||
await pgClient!.queryArray('COMMIT');
|
||||
|
||||
logWithTimestamp(`成功插入 ${insertedCount} 条记录到PostgreSQL`);
|
||||
return insertedCount;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
// 发生错误,回滚事务
|
||||
await pgClient!.queryArray('ROLLBACK');
|
||||
logWithTimestamp(`向PostgreSQL插入数据失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// 批量处理记录
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
let lastSyncTime = 0;
|
||||
let lastSyncId = "";
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
|
||||
break;
|
||||
}
|
||||
|
||||
// 每批次都输出进度
|
||||
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
|
||||
const records = await shortCollection.find(
|
||||
query,
|
||||
{
|
||||
sort: { createTime: 1 },
|
||||
skip: page * batch_size,
|
||||
limit: batch_size
|
||||
}
|
||||
).toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("没有找到更多数据,同步结束");
|
||||
break;
|
||||
}
|
||||
|
||||
// 找到数据,开始处理
|
||||
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
|
||||
|
||||
// 输出当前批次的部分数据信息
|
||||
if (records.length > 0) {
|
||||
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, slug=${records[0].slug}, 时间=${new Date(typeof records[0].createTime === 'number' ? records[0].createTime : 0).toISOString()}`);
|
||||
if (records.length > 1) {
|
||||
const lastRec = records[records.length-1];
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${lastRec._id}, slug=${lastRec.slug}, 时间=${new Date(typeof lastRec.createTime === 'number' ? lastRec.createTime : 0).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
// 更新最后处理的记录时间和ID
|
||||
if (records.length > 0) {
|
||||
const lastRecord = records[records.length - 1];
|
||||
// 提取数字时间戳
|
||||
let lastCreateTime = 0;
|
||||
if (typeof lastRecord.createTime === 'number') {
|
||||
lastCreateTime = lastRecord.createTime;
|
||||
} else if (lastRecord.createTime && lastRecord.createTime.$numberLong) {
|
||||
lastCreateTime = Number(lastRecord.createTime.$numberLong);
|
||||
}
|
||||
|
||||
lastSyncTime = Math.max(lastSyncTime, lastCreateTime);
|
||||
lastSyncId = lastRecord._id.toString();
|
||||
}
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
if (processedRecords > 0 && lastSyncTime > 0) {
|
||||
// 创建新的同步状态
|
||||
const newSyncState: SyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + totalBatchRecords,
|
||||
last_sync_id: lastSyncId
|
||||
};
|
||||
|
||||
try {
|
||||
// 保存同步状态
|
||||
await setVariable(SYNC_STATE_KEY, newSyncState);
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`更新同步状态失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
|
||||
message: "数据同步完成"
|
||||
};
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.error("同步过程中发生错误:", error);
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
};
|
||||
} finally {
|
||||
// 关闭连接
|
||||
if (pgClient) {
|
||||
await pgClient.end();
|
||||
logWithTimestamp("PostgreSQL连接已关闭");
|
||||
}
|
||||
await mongoClient.close();
|
||||
logWithTimestamp("MongoDB连接已关闭");
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,13 @@
|
||||
// Sync data from MongoDB trace table to ClickHouse events table
|
||||
import { getVariable } from "npm:windmill-client@1";
|
||||
// 从MongoDB的trace表同步数据到ClickHouse的events表
|
||||
//
|
||||
// 支持以下同步模式:
|
||||
// 1. 增量同步:基于上次同步状态,只同步新数据(默认模式)
|
||||
// 2. 自定义时间范围同步:通过指定开始时间和结束时间,同步特定时间范围内的数据
|
||||
// - 可以通过时间戳参数(start_time/end_time)指定范围
|
||||
// - 也可以通过日期字符串参数(start_date/end_date)指定范围,支持ISO格式或yyyy-MM-dd格式
|
||||
//
|
||||
// 使用自定义时间范围时,将不会更新同步状态,避免干扰增量同步进度
|
||||
import { getVariable, setVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
interface MongoConfig {
|
||||
@@ -15,6 +23,7 @@ interface ClickHouseConfig {
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_database: string;
|
||||
clickhouse_url: string;
|
||||
}
|
||||
|
||||
@@ -32,6 +41,7 @@ interface TraceRecord {
|
||||
createTime: number;
|
||||
}
|
||||
|
||||
// 添加 ShortRecord 接口定义
|
||||
interface ShortRecord {
|
||||
_id: ObjectId;
|
||||
slug: string; // 短链接的slug部分
|
||||
@@ -48,9 +58,161 @@ interface ShortRecord {
|
||||
projectId?: string; // 项目ID
|
||||
}
|
||||
|
||||
interface ClickHouseRow {
|
||||
event_id: string;
|
||||
event_attributes: string;
|
||||
interface SyncState {
|
||||
last_sync_time: number;
|
||||
records_synced: number;
|
||||
last_sync_id?: string;
|
||||
}
|
||||
|
||||
// 定义UTM参数接口
|
||||
interface UtmParams {
|
||||
utm_source: string;
|
||||
utm_medium: string;
|
||||
utm_campaign: string;
|
||||
utm_term: string;
|
||||
utm_content: string;
|
||||
}
|
||||
|
||||
// 同步状态键名
|
||||
const SYNC_STATE_KEY = "f/shorturl_analytics/mongo_sync_state";
|
||||
|
||||
// 日期字符串转时间戳工具函数(接受ISO字符串或yyyy-MM-dd格式)
|
||||
function dateToTimestamp(dateStr: string): number {
|
||||
try {
|
||||
// 尝试直接解析完整的ISO日期字符串
|
||||
const date = new Date(dateStr);
|
||||
|
||||
// 检查是否为有效日期
|
||||
if (isNaN(date.getTime())) {
|
||||
// 尝试解析yyyy-MM-dd格式,默认设置为当天的00:00:00
|
||||
const parts = dateStr.split('-');
|
||||
if (parts.length === 3) {
|
||||
const year = parseInt(parts[0], 10);
|
||||
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
|
||||
const day = parseInt(parts[2], 10);
|
||||
|
||||
const dateObj = new Date(year, month, day, 0, 0, 0);
|
||||
return dateObj.getTime();
|
||||
}
|
||||
throw new Error(`无法解析日期字符串: ${dateStr}`);
|
||||
}
|
||||
|
||||
return date.getTime();
|
||||
} catch (err) {
|
||||
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从URL中提取UTM参数的函数,增强版
|
||||
function extractUtmParams(url: string, debug = false): UtmParams {
|
||||
const defaultUtmParams: UtmParams = {
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
utm_term: "",
|
||||
utm_content: ""
|
||||
};
|
||||
|
||||
if (!url) return defaultUtmParams;
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 原始URL: ${url}`);
|
||||
}
|
||||
|
||||
// 准备一个解析后的参数对象
|
||||
const params: UtmParams = { ...defaultUtmParams };
|
||||
|
||||
// 尝试多种方法提取UTM参数
|
||||
|
||||
// 方法1: 使用URL对象解析
|
||||
try {
|
||||
// 先处理URL,确保是完整的URL格式
|
||||
let normalizedUrl = url;
|
||||
if (!url.match(/^https?:\/\//i)) {
|
||||
normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
|
||||
}
|
||||
|
||||
const urlObj = new URL(normalizedUrl);
|
||||
|
||||
// 读取URL参数
|
||||
if (urlObj.searchParams.has('utm_source'))
|
||||
params.utm_source = urlObj.searchParams.get('utm_source') || "";
|
||||
if (urlObj.searchParams.has('utm_medium'))
|
||||
params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
|
||||
if (urlObj.searchParams.has('utm_campaign'))
|
||||
params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
|
||||
if (urlObj.searchParams.has('utm_term'))
|
||||
params.utm_term = urlObj.searchParams.get('utm_term') || "";
|
||||
if (urlObj.searchParams.has('utm_content'))
|
||||
params.utm_content = urlObj.searchParams.get('utm_content') || "";
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
// 如果至少找到一个UTM参数,则返回
|
||||
if (params.utm_source || params.utm_medium || params.utm_campaign ||
|
||||
params.utm_term || params.utm_content) {
|
||||
return params;
|
||||
}
|
||||
} catch (_err) {
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析失败,尝试正则表达式`);
|
||||
}
|
||||
}
|
||||
|
||||
// 方法2: 使用正则表达式提取参数
|
||||
// 使用正则表达式(最安全的方法,适用于任何格式)
|
||||
const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
|
||||
if (sourceMatch && sourceMatch[1]) {
|
||||
try {
|
||||
params.utm_source = decodeURIComponent(sourceMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_source = sourceMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
|
||||
if (mediumMatch && mediumMatch[1]) {
|
||||
try {
|
||||
params.utm_medium = decodeURIComponent(mediumMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_medium = mediumMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) {
|
||||
try {
|
||||
params.utm_campaign = decodeURIComponent(campaignMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_campaign = campaignMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
|
||||
if (termMatch && termMatch[1]) {
|
||||
try {
|
||||
params.utm_term = decodeURIComponent(termMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_term = termMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
|
||||
if (contentMatch && contentMatch[1]) {
|
||||
try {
|
||||
params.utm_content = decodeURIComponent(contentMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_content = contentMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
export async function main(
|
||||
@@ -58,90 +220,246 @@ export async function main(
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false
|
||||
force_insert = true,
|
||||
database_override = "shorturl_analytics", // 添加数据库名称参数,默认为shorturl_analytics
|
||||
reset_sync_state = false, // 添加参数用于重置同步状态
|
||||
debug_utm = false, // 添加参数控制UTM调试日志输出
|
||||
start_time?: number, // 添加参数指定同步的开始时间戳,可选
|
||||
end_time?: number, // 添加参数指定同步的结束时间戳,可选
|
||||
use_custom_time_range = false, // 添加参数控制是否使用自定义时间范围
|
||||
start_date?: string, // 添加开始日期字符串参数(ISO格式或yyyy-MM-dd格式)
|
||||
end_date?: string // 添加结束日期字符串参数(ISO格式或yyyy-MM-dd格式)
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
|
||||
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
|
||||
// 处理日期字符串参数,转换为时间戳
|
||||
if (start_date) {
|
||||
try {
|
||||
start_time = dateToTimestamp(start_date);
|
||||
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${start_time}`);
|
||||
use_custom_time_range = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (end_date) {
|
||||
try {
|
||||
end_time = dateToTimestamp(end_date);
|
||||
// 如果是日期格式,设置为当天结束时间 (23:59:59.999)
|
||||
if (end_date.split('-').length === 3 && end_date.length <= 10) {
|
||||
end_time += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${end_time}`);
|
||||
} else {
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${end_time}`);
|
||||
}
|
||||
use_custom_time_range = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式,不会检查记录是否已存在");
|
||||
}
|
||||
if (force_insert) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
|
||||
}
|
||||
if (reset_sync_state) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用重置同步状态,将从头开始同步数据");
|
||||
}
|
||||
if (debug_utm) {
|
||||
logWithTimestamp("已启用UTM参数调试日志");
|
||||
}
|
||||
if (use_custom_time_range) {
|
||||
if (start_time) {
|
||||
logWithTimestamp(`已启用自定义时间范围:开始时间 ${new Date(start_time).toISOString()}`);
|
||||
}
|
||||
if (end_time) {
|
||||
logWithTimestamp(`已启用自定义时间范围:结束时间 ${new Date(end_time).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Set timeout
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
// 检查是否超时
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
|
||||
console.log(`运行时间超过${timeout_minutes}分钟,暂停执行`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// Get MongoDB and ClickHouse connection info
|
||||
// 获取MongoDB和ClickHouse的连接信息
|
||||
let mongoConfig: MongoConfig;
|
||||
let clickhouseConfig: ClickHouseConfig;
|
||||
|
||||
try {
|
||||
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
|
||||
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
|
||||
console.log("原始MongoDB配置:", JSON.stringify(rawMongoConfig));
|
||||
|
||||
// 尝试解析配置,如果是字符串形式
|
||||
if (typeof rawMongoConfig === "string") {
|
||||
try {
|
||||
mongoConfig = JSON.parse(rawMongoConfig);
|
||||
} catch (e) {
|
||||
console.error("MongoDB配置解析失败:", e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
mongoConfig = rawMongoConfig as MongoConfig;
|
||||
}
|
||||
|
||||
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
|
||||
console.log("原始ClickHouse配置:", JSON.stringify(rawClickhouseConfig));
|
||||
|
||||
// 尝试解析配置,如果是字符串形式
|
||||
if (typeof rawClickhouseConfig === "string") {
|
||||
try {
|
||||
clickhouseConfig = JSON.parse(rawClickhouseConfig);
|
||||
} catch (e) {
|
||||
console.error("ClickHouse配置解析失败:", e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
clickhouseConfig = rawClickhouseConfig as ClickHouseConfig;
|
||||
}
|
||||
|
||||
// 检查并修复数据库配置
|
||||
if (!clickhouseConfig.clickhouse_database || clickhouseConfig.clickhouse_database === "undefined") {
|
||||
logWithTimestamp(`⚠️ 警告: 数据库名称未定义或为'undefined',使用提供的默认值: ${database_override}`);
|
||||
clickhouseConfig.clickhouse_database = database_override;
|
||||
}
|
||||
|
||||
console.log("MongoDB配置解析为:", JSON.stringify(mongoConfig));
|
||||
console.log("ClickHouse配置解析为:", JSON.stringify({
|
||||
...clickhouseConfig,
|
||||
clickhouse_password: "****" // 隐藏密码
|
||||
}));
|
||||
|
||||
logWithTimestamp(`将使用ClickHouse数据库: ${clickhouseConfig.clickhouse_database}`);
|
||||
} catch (error) {
|
||||
console.error("Failed to get config:", error);
|
||||
console.error("获取配置失败:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Build MongoDB connection URL
|
||||
// 获取上次同步状态
|
||||
let lastSyncState: SyncState | null = null;
|
||||
if (!reset_sync_state) {
|
||||
try {
|
||||
const rawSyncState = await getVariable(SYNC_STATE_KEY);
|
||||
if (rawSyncState) {
|
||||
if (typeof rawSyncState === "string") {
|
||||
try {
|
||||
lastSyncState = JSON.parse(rawSyncState);
|
||||
} catch (e) {
|
||||
logWithTimestamp(`解析上次同步状态失败: ${e}, 将从头开始同步`);
|
||||
}
|
||||
} else {
|
||||
lastSyncState = rawSyncState as SyncState;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
|
||||
}
|
||||
}
|
||||
|
||||
if (lastSyncState) {
|
||||
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
|
||||
if (lastSyncState.last_sync_id) {
|
||||
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("没有找到上次同步状态,将从头开始同步");
|
||||
}
|
||||
|
||||
// 构建MongoDB连接URL
|
||||
let mongoUrl = "mongodb://";
|
||||
if (mongoConfig.username && mongoConfig.password) {
|
||||
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
|
||||
}
|
||||
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
|
||||
|
||||
// Connect to MongoDB
|
||||
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
|
||||
|
||||
// 连接MongoDB
|
||||
const client = new MongoClient();
|
||||
try {
|
||||
await client.connect(mongoUrl);
|
||||
console.log("MongoDB connected successfully");
|
||||
console.log("MongoDB连接成功");
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const traceCollection = db.collection<TraceRecord>("trace");
|
||||
// 添加对short集合的引用
|
||||
const shortCollection = db.collection<ShortRecord>("short");
|
||||
|
||||
// Build query conditions
|
||||
// 构建查询条件,根据上次同步状态获取新记录
|
||||
const query: Record<string, unknown> = {
|
||||
type: 1 // Only sync records with type 1
|
||||
// 删除了 type: 1 的条件,将同步所有数据
|
||||
};
|
||||
|
||||
// Count total records
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`Found ${totalRecords} records to sync`);
|
||||
// 根据时间范围参数构建查询条件
|
||||
if (use_custom_time_range) {
|
||||
// 使用自定义时间范围
|
||||
const timeQuery: Record<string, number> = {};
|
||||
|
||||
if (start_time) {
|
||||
timeQuery.$gte = start_time;
|
||||
logWithTimestamp(`将只同步createTime >= ${start_time} (${new Date(start_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
if (end_time) {
|
||||
timeQuery.$lte = end_time;
|
||||
logWithTimestamp(`将只同步createTime <= ${end_time} (${new Date(end_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 只有当至少指定了一个时间限制时才添加时间查询条件
|
||||
if (Object.keys(timeQuery).length > 0) {
|
||||
query.createTime = timeQuery;
|
||||
}
|
||||
}
|
||||
// 如果不使用自定义时间范围,且有上次同步状态,则只获取更新的记录
|
||||
else if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 使用上次同步时间作为过滤条件
|
||||
query.createTime = { $gt: lastSyncState.last_sync_time };
|
||||
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 计算总记录数
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`找到 ${totalRecords} 条新记录需要同步`);
|
||||
|
||||
// 限制此次处理的记录数量
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`Will process ${recordsToProcess} records`);
|
||||
console.log(`本次将处理 ${recordsToProcess} 条记录`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("No records to sync, task completed");
|
||||
console.log("没有新记录需要同步,任务完成");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "No records to sync"
|
||||
message: "没有新记录需要同步"
|
||||
};
|
||||
}
|
||||
|
||||
// Check ClickHouse connection
|
||||
// 检查ClickHouse连接状态
|
||||
const checkClickHouseConnection = async (): Promise<boolean> => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("Skipping ClickHouse connection check");
|
||||
logWithTimestamp("已启用跳过ClickHouse检查,不测试连接");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("Testing ClickHouse connection...");
|
||||
logWithTimestamp("测试ClickHouse连接...");
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
@@ -149,184 +467,197 @@ export async function main(
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
|
||||
},
|
||||
body: "SELECT 1",
|
||||
body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
|
||||
// 设置5秒超时
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse connection test successful");
|
||||
logWithTimestamp("ClickHouse连接测试成功");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Check if records exist in ClickHouse
|
||||
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
|
||||
return records;
|
||||
}
|
||||
|
||||
try {
|
||||
const recordIds = records.map(record => record._id.toString());
|
||||
|
||||
const query = `
|
||||
SELECT event_id
|
||||
FROM shorturl_analytics.events
|
||||
WHERE event_attributes LIKE '%"mongo_id":"%'
|
||||
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
const existingIds = new Set(result.data.map((row: ClickHouseRow) => {
|
||||
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
|
||||
return matches ? matches[1] : null;
|
||||
}).filter(Boolean));
|
||||
|
||||
return records.filter(record => !existingIds.has(record._id.toString()));
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
|
||||
return skip_clickhouse_check ? records : [];
|
||||
}
|
||||
};
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ ClickHouse连接测试失败,请启用skip_clickhouse_check=true参数来跳过连接检查");
|
||||
throw new Error("ClickHouse连接失败,无法继续同步");
|
||||
}
|
||||
|
||||
// Process records function
|
||||
// 处理记录的函数
|
||||
const processRecords = async (records: TraceRecord[]) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
const newRecords = await checkExistingRecords(records);
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("All records already exist, skipping");
|
||||
return 0;
|
||||
}
|
||||
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条记录...`);
|
||||
|
||||
// Get link information for all records
|
||||
// 强制使用所有记录,不检查重复
|
||||
const newRecords = records;
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
|
||||
|
||||
// 获取链接信息 - 新增代码
|
||||
const slugIds = newRecords.map(record => record.slugId);
|
||||
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
|
||||
const shortLinks = await shortCollection.find({
|
||||
_id: { $in: slugIds }
|
||||
}).toArray();
|
||||
|
||||
// Create a map for quick lookup
|
||||
const shortLinksMap = new Map(shortLinks.map(link => [link._id.toString(), link]));
|
||||
|
||||
// Prepare ClickHouse insert data
|
||||
// 创建映射用于快速查找 - 新增代码
|
||||
const shortLinksMap = new Map(shortLinks.map((link: ShortRecord) => [link._id.toString(), link]));
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息,${newRecords.length - shortLinks.length} 条数据将使用占位符`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString());
|
||||
const eventTime = new Date(record.createTime);
|
||||
|
||||
// 将毫秒时间戳转换为 DateTime64(3) 格式
|
||||
const formatDateTime = (timestamp: number) => {
|
||||
return new Date(timestamp).toISOString().replace('T', ' ').replace('Z', '');
|
||||
// 获取对应的短链接信息 - 新增代码
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString()) as ShortRecord | undefined;
|
||||
|
||||
// 提取URL中的UTM参数 - 增加调试日志
|
||||
if (debug_utm && record.url) {
|
||||
logWithTimestamp(`======== UTM参数调试 ========`);
|
||||
logWithTimestamp(`记录ID: ${record._id.toString()}`);
|
||||
logWithTimestamp(`原始URL: ${record.url}`);
|
||||
}
|
||||
|
||||
const utmParams = extractUtmParams(record.url || "", debug_utm);
|
||||
|
||||
if (debug_utm) {
|
||||
logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
|
||||
logWithTimestamp(`===========================`);
|
||||
}
|
||||
|
||||
// 保存提取的UTM参数和URL到event_attributes
|
||||
const eventAttributes = {
|
||||
mongo_id: record._id.toString(),
|
||||
url: record.url || "",
|
||||
...(record.url ? { raw_url: record.url } : {})
|
||||
};
|
||||
|
||||
// 转换MongoDB记录为ClickHouse格式,匹配ClickHouse表结构
|
||||
return {
|
||||
// Event base information
|
||||
event_id: record._id.toString(),
|
||||
event_time: formatDateTime(record.createTime),
|
||||
event_type: "click",
|
||||
event_attributes: JSON.stringify({
|
||||
original_type: record.type
|
||||
}),
|
||||
|
||||
// Link information from short collection
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
event_type: "click", // 将所有event_type都设置为click
|
||||
event_attributes: JSON.stringify(eventAttributes),
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: shortLink?.slug || "",
|
||||
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
|
||||
link_label: record.label || "",
|
||||
link_title: "",
|
||||
link_original_url: shortLink?.origin || "",
|
||||
link_attributes: JSON.stringify({
|
||||
domain: shortLink?.domain || null
|
||||
}),
|
||||
link_created_at: shortLink?.createTime ? formatDateTime(shortLink.createTime) : formatDateTime(record.createTime),
|
||||
link_expires_at: shortLink?.expiresAt ? formatDateTime(shortLink.expiresAt) : null,
|
||||
link_tags: "[]", // Empty array as default
|
||||
|
||||
// User information
|
||||
user_id: shortLink?.user || "",
|
||||
user_name: "",
|
||||
link_title: shortLink?.title || "unknown_title", // 使用占位符
|
||||
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
|
||||
link_created_at: shortLink?.createTime
|
||||
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
link_expires_at: shortLink?.expiresAt
|
||||
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: null,
|
||||
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
|
||||
user_id: "3680f452-e404-4339-a3d2-2a8e1ff92102", // 使用占位符
|
||||
user_name: "unknown_user", // 使用占位符
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
|
||||
// Team information
|
||||
team_id: shortLink?.teamId || "",
|
||||
team_name: "",
|
||||
team_id: "e02251eb-eb98-47c8-b5dd-4f6e4fdb1f49", // 使用占位符
|
||||
team_name: "", // 使用占位符
|
||||
team_attributes: "{}",
|
||||
|
||||
// Project information
|
||||
project_id: shortLink?.projectId || "",
|
||||
project_name: "",
|
||||
project_id: "34cdb8b9-8b8e-4033-876a-0632002ef1f9", // 使用占位符
|
||||
project_name: "", // 使用占位符
|
||||
project_attributes: "{}",
|
||||
|
||||
// QR code information
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
|
||||
// Visitor information
|
||||
visitor_id: "", // Empty string as default
|
||||
session_id: `${record.slugId.toString()}-${record.createTime}`,
|
||||
ip_address: record.ip || "",
|
||||
country: "",
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: record._id.toString() + "-" + record.createTime,
|
||||
ip_address: record.ip || "0.0.0.0", // 使用占位符
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "",
|
||||
browser: record.browser || "",
|
||||
os: record.platformOS || "",
|
||||
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
|
||||
|
||||
// Source information
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "unknown", // 使用占位符
|
||||
os: record.platformOS || "unknown", // 使用占位符
|
||||
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
|
||||
referrer: record.url || "",
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
|
||||
// Interaction information
|
||||
utm_source: utmParams.utm_source || "",
|
||||
utm_medium: utmParams.utm_medium || "",
|
||||
utm_campaign: utmParams.utm_campaign || "",
|
||||
utm_term: utmParams.utm_term || "",
|
||||
utm_content: utmParams.utm_content || "",
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
conversion_type: "visit",
|
||||
conversion_value: 0
|
||||
conversion_value: 0,
|
||||
req_full_path: record.url || ""
|
||||
};
|
||||
});
|
||||
|
||||
// Generate ClickHouse insert SQL
|
||||
const rows = clickhouseData.map(row => {
|
||||
// 只需要处理JSON字符串的转义
|
||||
const formattedRow = {
|
||||
...row,
|
||||
event_attributes: row.event_attributes.replace(/\\/g, '\\\\'),
|
||||
link_attributes: row.link_attributes.replace(/\\/g, '\\\\')
|
||||
};
|
||||
return JSON.stringify(formattedRow);
|
||||
}).join('\n');
|
||||
// 生成ClickHouse插入SQL
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
|
||||
(event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title,
|
||||
link_original_url, link_attributes, link_created_at, link_expires_at, link_tags,
|
||||
user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes,
|
||||
project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes,
|
||||
visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent,
|
||||
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
|
||||
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
|
||||
VALUES ${clickhouseData.map(record => {
|
||||
// 增强版安全替换函数,处理所有特殊字符
|
||||
const safeReplace = (val: unknown): string => {
|
||||
// 确保值是字符串,如果是null或undefined则使用空字符串
|
||||
const str = val === null || val === undefined ? "" : String(val);
|
||||
|
||||
// 转义所有可能导致SQL注入或格式错误的字符
|
||||
// 1. 先替换所有反斜杠
|
||||
// 2. 再替换单引号
|
||||
// 3. 替换所有控制字符和特殊字符
|
||||
return str
|
||||
.replace(/\\/g, "\\\\") // 转义反斜杠
|
||||
.replace(/'/g, "\\'") // 转义单引号
|
||||
.replace(/\r/g, "\\r") // 转义回车
|
||||
.replace(/\n/g, "\\n") // 转义换行
|
||||
.replace(/\t/g, "\\t") // 转义制表符
|
||||
.replace(/\0/g, "") // 移除空字符
|
||||
.replace(/[\x00-\x1F\x7F-\x9F]/g, ""); // 移除所有控制字符
|
||||
};
|
||||
|
||||
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
|
||||
'${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}',
|
||||
'${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}',
|
||||
${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}',
|
||||
'${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}',
|
||||
'${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}',
|
||||
'${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}',
|
||||
'${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}',
|
||||
'${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}',
|
||||
'${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}',
|
||||
'${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}',
|
||||
'${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}',
|
||||
'${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}',
|
||||
'${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan},
|
||||
'${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
|
||||
}).join(", ")}
|
||||
`;
|
||||
|
||||
const insertSQL = `INSERT INTO shorturl_analytics.events FORMAT JSONEachRow\n${rows}`;
|
||||
if (insertSQL.length === 0) {
|
||||
console.log("没有新记录需要插入");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 发送请求到ClickHouse,添加20秒超时
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
try {
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
logWithTimestamp("发送插入请求到ClickHouse...");
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
@@ -338,35 +669,34 @@ export async function main(
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
|
||||
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
|
||||
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
|
||||
throw err;
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`向ClickHouse插入数据失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// Check ClickHouse connection before processing
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
throw new Error("ClickHouse connection failed, cannot continue sync");
|
||||
}
|
||||
|
||||
// Process records in batches
|
||||
// 批量处理记录
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
let lastSyncTime = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
|
||||
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
|
||||
break;
|
||||
}
|
||||
|
||||
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
// 每批次都输出进度
|
||||
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
|
||||
const records = await traceCollection.find(
|
||||
query,
|
||||
{
|
||||
@@ -378,32 +708,86 @@ export async function main(
|
||||
).toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("No more records found, sync complete");
|
||||
logWithTimestamp("没有找到更多数据,同步结束");
|
||||
break;
|
||||
}
|
||||
|
||||
// 找到数据,开始处理
|
||||
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
|
||||
// 输出当前批次的部分数据信息
|
||||
if (records.length > 0) {
|
||||
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
|
||||
if (records.length > 1) {
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
|
||||
}
|
||||
|
||||
// 如果开启了调试,输出一些URL样本
|
||||
if (debug_utm) {
|
||||
const sampleSize = Math.min(5, records.length);
|
||||
logWithTimestamp(`URL样本 (前${sampleSize}条):`);
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
if (records[i].url) {
|
||||
logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
// 更新最后处理的记录时间和ID
|
||||
if (records.length > 0) {
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
|
||||
}
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
if (processedRecords > 0 && lastSyncTime > 0) {
|
||||
// 只在非自定义时间范围模式下更新同步状态
|
||||
if (!use_custom_time_range) {
|
||||
// 创建新的同步状态,简化对象结构
|
||||
const newSyncState: SyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords, // 使用处理的总记录数,而不是实际插入数
|
||||
};
|
||||
|
||||
try {
|
||||
// 保存同步状态
|
||||
await setVariable(SYNC_STATE_KEY, newSyncState);
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`更新同步状态失败: ${error.message},将继续执行`);
|
||||
// 不抛出错误,继续执行
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("使用自定义时间范围模式,不更新全局同步状态");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
message: "Data sync completed"
|
||||
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
|
||||
message: use_custom_time_range ? "自定义时间范围数据同步完成" : "数据同步完成",
|
||||
custom_time_range_used: use_custom_time_range
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("Error during sync:", err);
|
||||
console.error("同步过程中发生错误:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
stack: err instanceof Error ? err.stack : undefined
|
||||
};
|
||||
} finally {
|
||||
// 关闭MongoDB连接
|
||||
await client.close();
|
||||
console.log("MongoDB connection closed");
|
||||
console.log("MongoDB连接已关闭");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
527
windmill/sync_shorturl_schema_to_clickhouse.ts
Normal file
527
windmill/sync_shorturl_schema_to_clickhouse.ts
Normal file
@@ -0,0 +1,527 @@
|
||||
// 文件名: sync_shorturl_schema_to_clickhouse.ts
|
||||
// 描述: 此脚本用于同步PostgreSQL中的short_url.shorturl表数据到ClickHouse
|
||||
// 创建日期: 2023-11-21
|
||||
|
||||
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
|
||||
import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
|
||||
|
||||
// 同步状态接口
|
||||
interface SyncState {
|
||||
last_sync_time: string; // 上次同步的结束时间
|
||||
records_synced: number; // 累计同步的记录数
|
||||
last_run: string; // 上次运行的时间
|
||||
}
|
||||
|
||||
// 同步状态键名
|
||||
const SYNC_STATE_KEY = "f/shorturl_analytics/shorturl_to_clickhouse_state";
|
||||
|
||||
// PostgreSQL配置接口
|
||||
interface PgConfig {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
password: string;
|
||||
dbname?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// ClickHouse配置接口
|
||||
interface ChConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_url?: string;
|
||||
}
|
||||
|
||||
// Shorturl数据接口
|
||||
interface ShortUrlData {
|
||||
id: string;
|
||||
slug: string;
|
||||
origin: string; // 对应ClickHouse中的original_url
|
||||
title?: string;
|
||||
description?: string;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
deleted_at?: string;
|
||||
expires_at?: string; // 注意这里已更正为expires_at
|
||||
domain?: string; // 添加domain字段
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步PostgreSQL short_url.shorturl表数据到ClickHouse
|
||||
*/
|
||||
export async function main(
|
||||
/** 是否为测试模式(不执行实际更新) */
|
||||
dry_run = false,
|
||||
/** 是否显示详细日志 */
|
||||
verbose = false,
|
||||
/** 是否重置同步状态(从头开始同步) */
|
||||
reset_sync_state = false,
|
||||
/** 如果没有同步状态,往前查询多少小时的数据(默认1小时) */
|
||||
default_hours_back = 1
|
||||
) {
|
||||
// 初始化日志函数
|
||||
const log = (message: string, isVerbose = false) => {
|
||||
if (!isVerbose || verbose) {
|
||||
console.log(message);
|
||||
}
|
||||
};
|
||||
|
||||
// 获取同步状态
|
||||
let syncState: SyncState | null = null;
|
||||
if (!reset_sync_state) {
|
||||
try {
|
||||
log("获取同步状态...", true);
|
||||
const rawState = await getVariable(SYNC_STATE_KEY);
|
||||
if (rawState) {
|
||||
if (typeof rawState === "string") {
|
||||
syncState = JSON.parse(rawState);
|
||||
} else {
|
||||
syncState = rawState as SyncState;
|
||||
}
|
||||
log(`找到上次同步状态: 最后同步时间 ${syncState.last_sync_time}, 已同步记录数 ${syncState.records_synced}`, true);
|
||||
}
|
||||
} catch (error) {
|
||||
log(`获取同步状态失败: ${error}, 将使用默认设置`, true);
|
||||
}
|
||||
} else {
|
||||
log("重置同步状态,从头开始同步", true);
|
||||
}
|
||||
|
||||
// 设置时间范围
|
||||
const oneHourAgo = new Date(Date.now() - default_hours_back * 60 * 60 * 1000).toISOString();
|
||||
// 如果有同步状态,使用上次同步时间作为开始时间;否则使用默认时间
|
||||
const start_time = syncState ? syncState.last_sync_time : oneHourAgo;
|
||||
const end_time = new Date().toISOString();
|
||||
|
||||
log(`开始同步shorturl表数据: ${start_time} 至 ${end_time}`);
|
||||
|
||||
let pgPool: Pool | null = null;
|
||||
|
||||
try {
|
||||
// 1. 获取数据库配置
|
||||
log("获取PostgreSQL数据库配置...", true);
|
||||
const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
|
||||
|
||||
// 2. 创建PostgreSQL连接池
|
||||
pgPool = new Pool({
|
||||
hostname: pgConfig.host,
|
||||
port: pgConfig.port,
|
||||
user: pgConfig.user,
|
||||
password: pgConfig.password,
|
||||
database: pgConfig.dbname || 'postgres'
|
||||
}, 3);
|
||||
|
||||
// 3. 获取需要更新的数据
|
||||
const shorturlData = await getShortUrlData(pgPool, start_time, end_time, log);
|
||||
log(`成功获取 ${shorturlData.length} 条shorturl数据`);
|
||||
|
||||
if (shorturlData.length === 0) {
|
||||
// 更新同步状态,即使没有新数据
|
||||
if (!dry_run) {
|
||||
await updateSyncState(end_time, syncState ? syncState.records_synced : 0, log);
|
||||
}
|
||||
return { success: true, message: "没有找到需要更新的数据", updated: 0 };
|
||||
}
|
||||
|
||||
// 4. 获取ClickHouse配置
|
||||
const chConfig = await getClickHouseConfig();
|
||||
|
||||
// 5. 执行更新
|
||||
if (!dry_run) {
|
||||
const shorturlUpdated = await updateClickHouseShortUrl(shorturlData, chConfig, log);
|
||||
|
||||
// 更新同步状态
|
||||
const totalSynced = (syncState ? syncState.records_synced : 0) + shorturlUpdated;
|
||||
await updateSyncState(end_time, totalSynced, log);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: "shorturl表数据同步完成",
|
||||
shorturl_updated: shorturlUpdated,
|
||||
total_synced: totalSynced,
|
||||
sync_state: {
|
||||
last_sync_time: end_time,
|
||||
records_synced: totalSynced
|
||||
}
|
||||
};
|
||||
} else {
|
||||
log("测试模式: 不执行实际更新");
|
||||
return {
|
||||
success: true,
|
||||
dry_run: true,
|
||||
shorturl_count: shorturlData.length,
|
||||
shorturl_sample: shorturlData.slice(0, 1)
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = `同步过程中发生错误: ${(error as Error).message}`;
|
||||
log(errorMessage);
|
||||
if ((error as Error).stack) {
|
||||
log(`错误堆栈: ${(error as Error).stack}`, true);
|
||||
}
|
||||
return { success: false, message: errorMessage };
|
||||
} finally {
|
||||
if (pgPool) {
|
||||
await pgPool.end();
|
||||
log("PostgreSQL连接池已关闭", true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新同步状态
|
||||
*/
|
||||
async function updateSyncState(lastSyncTime: string, recordsSynced: number, log: (message: string, isVerbose?: boolean) => void): Promise<void> {
|
||||
try {
|
||||
const newState: SyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: recordsSynced,
|
||||
last_run: new Date().toISOString()
|
||||
};
|
||||
|
||||
await setVariable(SYNC_STATE_KEY, newState);
|
||||
log(`同步状态已更新: 最后同步时间 ${lastSyncTime}, 累计同步记录数 ${recordsSynced}`, true);
|
||||
} catch (error) {
|
||||
log(`更新同步状态失败: ${error}`, true);
|
||||
// 继续执行,不中断同步过程
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从PostgreSQL获取shorturl数据
|
||||
*/
|
||||
async function getShortUrlData(
|
||||
pgPool: Pool,
|
||||
startTime: string,
|
||||
endTime: string,
|
||||
log: (message: string, isVerbose?: boolean) => void
|
||||
): Promise<ShortUrlData[]> {
|
||||
const client = await pgPool.connect();
|
||||
|
||||
try {
|
||||
log(`获取shorturl数据 (${startTime} 至 ${endTime})`, true);
|
||||
|
||||
const query = `
|
||||
SELECT
|
||||
id,
|
||||
slug,
|
||||
origin,
|
||||
title,
|
||||
description,
|
||||
domain,
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
expired_at as expires_at
|
||||
FROM
|
||||
short_url.shorturl
|
||||
WHERE
|
||||
(created_at >= $1 AND created_at <= $2)
|
||||
OR (updated_at >= $1 AND updated_at <= $2)
|
||||
`;
|
||||
|
||||
const result = await client.queryObject(query, [startTime, endTime]);
|
||||
return result.rows as ShortUrlData[];
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化日期时间为ClickHouse可接受的格式
|
||||
*/
|
||||
function formatDateTime(dateStr: string | null | undefined): string {
|
||||
if (!dateStr) return 'NULL';
|
||||
|
||||
try {
|
||||
// 将日期字符串转换为ISO格式
|
||||
const date = new Date(dateStr);
|
||||
if (isNaN(date.getTime())) {
|
||||
return 'NULL';
|
||||
}
|
||||
|
||||
// 返回ISO格式的日期字符串,ClickHouse可以解析
|
||||
return `parseDateTimeBestEffort('${date.toISOString()}')`;
|
||||
} catch (error) {
|
||||
console.error(`日期格式化错误: ${error}`);
|
||||
return 'NULL';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化进度显示
|
||||
*/
|
||||
function formatProgress(current: number, total: number): string {
|
||||
const percent = Math.round((current / total) * 100);
|
||||
const progressBar = '[' + '='.repeat(Math.floor(percent / 5)) + ' '.repeat(20 - Math.floor(percent / 5)) + ']';
|
||||
return `${progressBar} ${percent}% (${current}/${total})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新ClickHouse中的shorturl表数据
|
||||
*/
|
||||
async function updateClickHouseShortUrl(
|
||||
shorturls: ShortUrlData[],
|
||||
chConfig: ChConfig,
|
||||
log: (message: string, isVerbose?: boolean) => void
|
||||
): Promise<number> {
|
||||
if (shorturls.length === 0) {
|
||||
log('没有找到shorturl数据,跳过shorturl表更新');
|
||||
return 0;
|
||||
}
|
||||
|
||||
log(`准备更新 ${shorturls.length} 条shorturl数据`);
|
||||
|
||||
// 检查ClickHouse中是否存在shorturl表
|
||||
const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.shorturl');
|
||||
|
||||
if (!tableExists) {
|
||||
log('ClickHouse中未找到shorturl表,请先创建表');
|
||||
return 0;
|
||||
}
|
||||
|
||||
let updatedCount = 0;
|
||||
const startTime = Date.now();
|
||||
|
||||
// 使用批量插入更高效
|
||||
const batchSize = 50; // 降低批次大小,使查询更稳定
|
||||
for (let i = 0; i < shorturls.length; i += batchSize) {
|
||||
const batch = shorturls.slice(i, i + batchSize);
|
||||
let successCount = 0;
|
||||
|
||||
// 显示批处理进度信息
|
||||
const batchNumber = Math.floor(i / batchSize) + 1;
|
||||
const totalBatches = Math.ceil(shorturls.length / batchSize);
|
||||
log(`处理批次 ${batchNumber}/${totalBatches}: ${formatProgress(i, shorturls.length)}`);
|
||||
|
||||
// 对每条记录使用单独的INSERT ... SELECT ... WHERE NOT EXISTS语句
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const shorturl = batch[j];
|
||||
// 显示记录处理细节进度
|
||||
const overallProgress = i + j + 1;
|
||||
if (overallProgress % 10 === 0 || overallProgress === shorturls.length) {
|
||||
// 每10条记录或最后一条记录显示一次进度
|
||||
const elapsedSeconds = (Date.now() - startTime) / 1000;
|
||||
const recordsPerSecond = overallProgress / elapsedSeconds;
|
||||
const remainingRecords = shorturls.length - overallProgress;
|
||||
const estimatedSecondsRemaining = remainingRecords / recordsPerSecond;
|
||||
|
||||
log(`总进度: ${formatProgress(overallProgress, shorturls.length)} - 速率: ${recordsPerSecond.toFixed(1)}条/秒 - 预计剩余时间: ${formatTime(estimatedSecondsRemaining)}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const insertQuery = `
|
||||
INSERT INTO shorturl_analytics.shorturl
|
||||
SELECT
|
||||
'${escapeString(shorturl.id)}' AS id,
|
||||
'${escapeString(shorturl.id)}' AS external_id,
|
||||
'shorturl' AS type,
|
||||
'${escapeString(shorturl.slug)}' AS slug,
|
||||
'${escapeString(shorturl.origin)}' AS original_url,
|
||||
${shorturl.title ? `'${escapeString(shorturl.title)}'` : 'NULL'} AS title,
|
||||
${shorturl.description ? `'${escapeString(shorturl.description)}'` : 'NULL'} AS description,
|
||||
'{}' AS attributes,
|
||||
1 AS schema_version,
|
||||
'' AS creator_id,
|
||||
'' AS creator_email,
|
||||
'' AS creator_name,
|
||||
${formatDateTime(shorturl.created_at)} AS created_at,
|
||||
${formatDateTime(shorturl.updated_at)} AS updated_at,
|
||||
${formatDateTime(shorturl.deleted_at)} AS deleted_at,
|
||||
'[]' AS projects,
|
||||
'[]' AS teams,
|
||||
'[]' AS tags,
|
||||
'[]' AS qr_codes,
|
||||
'[]' AS channels,
|
||||
'[]' AS favorites,
|
||||
${formatDateTime(shorturl.expires_at)} AS expires_at,
|
||||
0 AS click_count,
|
||||
0 AS unique_visitors,
|
||||
${shorturl.domain ? `'${escapeString(shorturl.domain)}'` : 'NULL'} AS domain
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM shorturl_analytics.shorturl WHERE id = '${escapeString(shorturl.id)}'
|
||||
)
|
||||
`;
|
||||
|
||||
await executeClickHouseQuery(chConfig, insertQuery);
|
||||
successCount++;
|
||||
log(`成功处理shorturl: ${shorturl.id}`, true);
|
||||
} catch (error) {
|
||||
log(`处理shorturl ${shorturl.id} 失败: ${(error as Error).message}`);
|
||||
|
||||
// 尝试使用简单插入作为备选方案
|
||||
try {
|
||||
log(`尝试替代方法更新: ${shorturl.id}`, true);
|
||||
|
||||
// 先检查记录是否存在
|
||||
const checkQuery = `SELECT count() FROM shorturl_analytics.shorturl WHERE id = '${escapeString(shorturl.id)}'`;
|
||||
const existsResult = await executeClickHouseQuery(chConfig, checkQuery);
|
||||
const exists = parseInt(existsResult.trim()) > 0;
|
||||
|
||||
if (!exists) {
|
||||
const fallbackQuery = `
|
||||
INSERT INTO shorturl_analytics.shorturl (
|
||||
id, external_id, type, slug, original_url,
|
||||
title, description, attributes, schema_version,
|
||||
creator_id, creator_email, creator_name,
|
||||
created_at, updated_at, deleted_at,
|
||||
projects, teams, tags, qr_codes, channels, favorites,
|
||||
expires_at, click_count, unique_visitors, domain
|
||||
) VALUES (
|
||||
'${escapeString(shorturl.id)}',
|
||||
'${escapeString(shorturl.id)}',
|
||||
'shorturl',
|
||||
'${escapeString(shorturl.slug)}',
|
||||
'${escapeString(shorturl.origin)}',
|
||||
${shorturl.title ? `'${escapeString(shorturl.title)}'` : 'NULL'},
|
||||
${shorturl.description ? `'${escapeString(shorturl.description)}'` : 'NULL'},
|
||||
'{}',
|
||||
1,
|
||||
'',
|
||||
'',
|
||||
'',
|
||||
${formatDateTime(shorturl.created_at)},
|
||||
${formatDateTime(shorturl.updated_at)},
|
||||
${formatDateTime(shorturl.deleted_at)},
|
||||
'[]',
|
||||
'[]',
|
||||
'[]',
|
||||
'[]',
|
||||
'[]',
|
||||
'[]',
|
||||
${formatDateTime(shorturl.expires_at)},
|
||||
0,
|
||||
0,
|
||||
${shorturl.domain ? `'${escapeString(shorturl.domain)}'` : 'NULL'}
|
||||
)
|
||||
`;
|
||||
|
||||
await executeClickHouseQuery(chConfig, fallbackQuery);
|
||||
successCount++;
|
||||
log(`备选方式插入成功: ${shorturl.id}`, true);
|
||||
} else {
|
||||
log(`记录已存在,跳过: ${shorturl.id}`, true);
|
||||
}
|
||||
} catch (fallbackError) {
|
||||
log(`备选方式失败 ${shorturl.id}: ${(fallbackError as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
updatedCount += successCount;
|
||||
log(`批次 ${batchNumber}/${totalBatches} 完成: ${successCount}/${batch.length} 条成功 (总计: ${updatedCount}/${shorturls.length})`);
|
||||
}
|
||||
|
||||
const totalTime = (Date.now() - startTime) / 1000;
|
||||
log(`同步完成! 总计处理: ${updatedCount}/${shorturls.length} 条记录, 耗时: ${formatTime(totalTime)}, 平均速率: ${(updatedCount / totalTime).toFixed(1)}条/秒`);
|
||||
|
||||
return updatedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取ClickHouse配置
|
||||
*/
|
||||
async function getClickHouseConfig(): Promise<ChConfig> {
|
||||
try {
|
||||
const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
|
||||
// 确保配置不为空
|
||||
if (!chConfigJson) {
|
||||
throw new Error("未找到ClickHouse配置");
|
||||
}
|
||||
|
||||
// 解析JSON字符串为对象
|
||||
let chConfig: ChConfig;
|
||||
if (typeof chConfigJson === 'string') {
|
||||
try {
|
||||
chConfig = JSON.parse(chConfigJson);
|
||||
} catch {
|
||||
throw new Error("ClickHouse配置不是有效的JSON");
|
||||
}
|
||||
} else {
|
||||
chConfig = chConfigJson as ChConfig;
|
||||
}
|
||||
|
||||
// 验证并构建URL
|
||||
if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
|
||||
chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
|
||||
}
|
||||
|
||||
if (!chConfig.clickhouse_url) {
|
||||
throw new Error("ClickHouse配置缺少URL");
|
||||
}
|
||||
|
||||
return chConfig;
|
||||
} catch (error) {
|
||||
throw new Error(`获取ClickHouse配置失败: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查ClickHouse中是否存在指定表
|
||||
*/
|
||||
async function checkClickHouseTable(chConfig: ChConfig, tableName: string): Promise<boolean> {
|
||||
try {
|
||||
const query = `EXISTS TABLE ${tableName}`;
|
||||
const result = await executeClickHouseQuery(chConfig, query);
|
||||
return result.trim() === '1';
|
||||
} catch (error) {
|
||||
console.error(`检查表 ${tableName} 失败:`, error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行ClickHouse查询
|
||||
*/
|
||||
async function executeClickHouseQuery(chConfig: ChConfig, query: string): Promise<string> {
|
||||
// 确保URL有效
|
||||
if (!chConfig.clickhouse_url) {
|
||||
throw new Error("无效的ClickHouse URL: 未定义");
|
||||
}
|
||||
|
||||
// 执行HTTP请求
|
||||
try {
|
||||
const response = await fetch(chConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse查询失败 (${response.status}): ${errorText}`);
|
||||
}
|
||||
|
||||
return await response.text();
|
||||
} catch (error) {
|
||||
throw new Error(`执行ClickHouse查询失败: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串,避免SQL注入
|
||||
*/
|
||||
function escapeString(str: string): string {
|
||||
if (!str) return '';
|
||||
return str.replace(/'/g, "''");
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化时间(秒)为可读格式
|
||||
*/
|
||||
function formatTime(seconds: number): string {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
|
||||
if (mins === 0) {
|
||||
return `${secs}秒`;
|
||||
} else {
|
||||
return `${mins}分${secs}秒`;
|
||||
}
|
||||
}
|
||||
709
windmill/sync_shorturl_to_clickhouse.ts
Normal file
709
windmill/sync_shorturl_to_clickhouse.ts
Normal file
@@ -0,0 +1,709 @@
|
||||
// Windmill script to sync shorturl data from PostgreSQL to ClickHouse
|
||||
// 作者: AI Assistant
|
||||
// 创建日期: 2023-10-30
|
||||
// 描述: 此脚本从PostgreSQL数据库获取所有shorturl类型的资源及其关联数据,并同步到ClickHouse
|
||||
|
||||
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
|
||||
import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
|
||||
|
||||
// 资源属性接口
|
||||
interface ResourceAttributes {
|
||||
slug?: string;
|
||||
original_url?: string;
|
||||
originalUrl?: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
expires_at?: string;
|
||||
expiresAt?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// ClickHouse配置接口
|
||||
interface ChConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_url: string;
|
||||
}
|
||||
|
||||
// PostgreSQL配置接口
|
||||
interface PgConfig {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
password: string;
|
||||
dbname?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// 上次同步状态接口
|
||||
interface SyncState {
|
||||
lastSyncTime: string;
|
||||
lastRunTime: string;
|
||||
}
|
||||
|
||||
// 状态变量名称
|
||||
const STATE_VARIABLE_PATH = "f/shorturl_analytics/shorturl_sync_state";
|
||||
|
||||
// Windmill函数定义
|
||||
export async function main(
|
||||
/** PostgreSQL和ClickHouse同步脚本 */
|
||||
params: {
|
||||
/** 同步的资源数量限制,默认500 */
|
||||
limit?: number;
|
||||
/** 是否包含已删除资源 */
|
||||
includeDeleted?: boolean;
|
||||
/** 是否执行实际写入操作 */
|
||||
dryRun?: boolean;
|
||||
/** 是否强制全量同步 */
|
||||
forceFullSync?: boolean;
|
||||
/** 手动指定开始时间(ISO格式)- 会覆盖自动增量设置 */
|
||||
startTime?: string;
|
||||
/** 手动指定结束时间(ISO格式)*/
|
||||
endTime?: string;
|
||||
}
|
||||
) {
|
||||
// 设置默认参数
|
||||
const limit = params.limit || 500;
|
||||
const includeDeleted = params.includeDeleted || false;
|
||||
const dryRun = params.dryRun || false;
|
||||
const forceFullSync = params.forceFullSync || false;
|
||||
|
||||
// 获取当前时间作为本次运行时间
|
||||
const currentRunTime = new Date().toISOString();
|
||||
|
||||
// 初始化同步状态
|
||||
let syncState: SyncState;
|
||||
let startTime: Date | undefined;
|
||||
const endTime: Date | undefined = params.endTime ? new Date(params.endTime) : new Date();
|
||||
|
||||
// 如果强制全量同步或手动指定了开始时间,则使用指定的开始时间
|
||||
if (forceFullSync || params.startTime) {
|
||||
startTime = params.startTime ? new Date(params.startTime) : undefined;
|
||||
console.log(`使用${params.startTime ? '手动指定' : '全量同步'} - 开始时间: ${startTime ? startTime.toISOString() : '无限制'}`);
|
||||
}
|
||||
// 否则尝试获取上次同步时间作为增量同步的开始时间点
|
||||
else {
|
||||
try {
|
||||
// 获取上次同步状态
|
||||
const stateStr = await getVariable(STATE_VARIABLE_PATH);
|
||||
if (stateStr) {
|
||||
syncState = JSON.parse(stateStr);
|
||||
console.log(`获取到上次同步状态: 同步时间=${syncState.lastSyncTime}, 运行时间=${syncState.lastRunTime}`);
|
||||
|
||||
// 使用上次运行时间作为本次的开始时间 (减去1分钟防止边界问题)
|
||||
const lastRunTime = new Date(syncState.lastRunTime);
|
||||
lastRunTime.setMinutes(lastRunTime.getMinutes() - 1);
|
||||
startTime = lastRunTime;
|
||||
} else {
|
||||
console.log("未找到上次同步状态,将执行全量同步");
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
console.log(`获取同步状态出错: ${error instanceof Error ? error.message : String(error)},将执行全量同步`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`开始同步PostgreSQL shorturl数据到ClickHouse`);
|
||||
console.log(`参数: limit=${limit}, includeDeleted=${includeDeleted}, dryRun=${dryRun}`);
|
||||
if (startTime) console.log(`开始时间: ${startTime.toISOString()}`);
|
||||
if (endTime) console.log(`结束时间: ${endTime.toISOString()}`);
|
||||
|
||||
// 获取数据库配置
|
||||
console.log("获取PostgreSQL数据库配置...");
|
||||
const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
|
||||
console.log(`数据库连接配置: host=${pgConfig.host}, port=${pgConfig.port}, database=${pgConfig.dbname || 'postgres'}, user=${pgConfig.user}`);
|
||||
|
||||
let pgPool: Pool | null = null;
|
||||
|
||||
try {
|
||||
console.log("创建PostgreSQL连接池...");
|
||||
|
||||
pgPool = new Pool({
|
||||
hostname: pgConfig.host,
|
||||
port: pgConfig.port,
|
||||
user: pgConfig.user,
|
||||
password: pgConfig.password,
|
||||
database: pgConfig.dbname || 'postgres'
|
||||
}, 3);
|
||||
|
||||
console.log("PostgreSQL连接池创建完成,尝试连接...");
|
||||
|
||||
// 测试连接
|
||||
const client = await pgPool.connect();
|
||||
try {
|
||||
console.log("连接成功,执行测试查询...");
|
||||
const testResult = await client.queryObject(`SELECT 1 AS test`);
|
||||
console.log(`测试查询结果: ${JSON.stringify(testResult.rows)}`);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
|
||||
// 获取所有shorturl类型的资源
|
||||
const shorturls = await fetchShorturlResources(pgPool, {
|
||||
limit,
|
||||
includeDeleted,
|
||||
startTime,
|
||||
endTime,
|
||||
});
|
||||
|
||||
console.log(`获取到 ${shorturls.length} 个shorturl资源`);
|
||||
|
||||
if (shorturls.length === 0) {
|
||||
// 即使没有数据也更新状态
|
||||
await updateSyncState(currentRunTime);
|
||||
return { synced: 0, message: "没有找到需要同步的shorturl资源" };
|
||||
}
|
||||
|
||||
// 为每个资源获取关联数据
|
||||
const enrichedShorturls = await enrichShorturlData(pgPool, shorturls);
|
||||
console.log(`已丰富 ${enrichedShorturls.length} 个shorturl资源的关联数据`);
|
||||
|
||||
// 转换为ClickHouse格式
|
||||
const clickhouseData = formatForClickhouse(enrichedShorturls);
|
||||
|
||||
if (!dryRun) {
|
||||
// 写入ClickHouse
|
||||
const inserted = await insertToClickhouse(clickhouseData);
|
||||
console.log(`成功写入 ${inserted} 条记录到ClickHouse`);
|
||||
|
||||
// 更新同步状态
|
||||
await updateSyncState(currentRunTime);
|
||||
|
||||
return { synced: inserted, message: "同步完成", lastSyncTime: currentRunTime };
|
||||
} else {
|
||||
console.log("Dry run模式 - 不执行实际写入");
|
||||
console.log(`将写入 ${clickhouseData.length} 条记录到ClickHouse`);
|
||||
// 输出示例数据
|
||||
if (clickhouseData.length > 0) {
|
||||
console.log("示例数据:");
|
||||
console.log(JSON.stringify(clickhouseData[0], null, 2));
|
||||
}
|
||||
return { synced: 0, dryRun: true, sampleData: clickhouseData.slice(0, 1) };
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
console.error(`同步过程中发生错误: ${(error as Error).message}`);
|
||||
console.error(`错误类型: ${(error as Error).name}`);
|
||||
if ((error as Error).stack) {
|
||||
console.error(`错误堆栈: ${(error as Error).stack}`);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (pgPool) {
|
||||
await pgPool.end();
|
||||
console.log("PostgreSQL连接池已关闭");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
async function updateSyncState(currentRunTime: string): Promise<void> {
|
||||
try {
|
||||
const syncState: SyncState = {
|
||||
lastSyncTime: new Date().toISOString(), // 记录数据同步完成的时间
|
||||
lastRunTime: currentRunTime // 记录本次运行的时间点
|
||||
};
|
||||
|
||||
console.log(`更新同步状态: ${JSON.stringify(syncState)}`);
|
||||
await setVariable(STATE_VARIABLE_PATH, JSON.stringify(syncState));
|
||||
} catch (error: unknown) {
|
||||
console.error(`更新同步状态失败: ${error instanceof Error ? error.message : String(error)}`);
|
||||
// 不中断主流程,即使状态更新失败
|
||||
}
|
||||
}
|
||||
|
||||
// 从PostgreSQL获取所有shorturl资源
|
||||
async function fetchShorturlResources(
|
||||
pgPool: Pool,
|
||||
options: {
|
||||
limit: number;
|
||||
includeDeleted: boolean;
|
||||
startTime?: Date;
|
||||
endTime?: Date;
|
||||
}
|
||||
) {
|
||||
let query = `
|
||||
SELECT
|
||||
r.id,
|
||||
r.external_id,
|
||||
r.type,
|
||||
r.attributes,
|
||||
r.schema_version,
|
||||
r.creator_id,
|
||||
r.created_at,
|
||||
r.updated_at,
|
||||
r.deleted_at,
|
||||
u.email as creator_email,
|
||||
u.first_name as creator_first_name,
|
||||
u.last_name as creator_last_name
|
||||
FROM
|
||||
limq.resources r
|
||||
LEFT JOIN
|
||||
limq.users u ON r.creator_id = u.id
|
||||
WHERE
|
||||
r.type = 'shorturl'
|
||||
`;
|
||||
|
||||
const params = [];
|
||||
let paramCount = 1;
|
||||
|
||||
if (!options.includeDeleted) {
|
||||
query += ` AND r.deleted_at IS NULL`;
|
||||
}
|
||||
|
||||
// 修改为同时考虑created_at和updated_at,确保捕获自上次同步以来创建或更新的记录
|
||||
if (options.startTime) {
|
||||
query += ` AND (r.created_at >= $${paramCount} OR r.updated_at >= $${paramCount})`;
|
||||
params.push(options.startTime);
|
||||
paramCount++;
|
||||
}
|
||||
|
||||
if (options.endTime) {
|
||||
query += ` AND r.created_at <= $${paramCount}`;
|
||||
params.push(options.endTime);
|
||||
paramCount++;
|
||||
}
|
||||
|
||||
// 优先按更新时间排序,确保最近更新的记录先处理
|
||||
query += ` ORDER BY r.updated_at DESC, r.created_at DESC LIMIT $${paramCount}`;
|
||||
params.push(options.limit);
|
||||
|
||||
const client = await pgPool.connect();
|
||||
try {
|
||||
const result = await client.queryObject(query, params);
|
||||
|
||||
// 添加调试日志 - 显示获取的数据样本
|
||||
if (result.rows.length > 0) {
|
||||
console.log(`获取到 ${result.rows.length} 条shorturl记录`);
|
||||
console.log(`第一条记录ID: ${result.rows[0].id}`);
|
||||
console.log(`attributes类型: ${typeof result.rows[0].attributes}`);
|
||||
console.log(`attributes内容示例: ${JSON.stringify(String(result.rows[0].attributes)).substring(0, 100)}...`);
|
||||
}
|
||||
|
||||
return result.rows;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
// 为每个shorturl资源获取关联数据
|
||||
async function enrichShorturlData(pgPool: Pool, shorturls: Record<string, unknown>[]) {
|
||||
const client = await pgPool.connect();
|
||||
const enriched = [];
|
||||
|
||||
try {
|
||||
for (const shorturl of shorturls) {
|
||||
// 1. 获取项目关联
|
||||
const projectsResult = await client.queryObject(`
|
||||
SELECT
|
||||
pr.resource_id, pr.project_id,
|
||||
p.name as project_name, p.description as project_description,
|
||||
pr.assigned_at
|
||||
FROM
|
||||
limq.project_resources pr
|
||||
JOIN
|
||||
limq.projects p ON pr.project_id = p.id
|
||||
WHERE
|
||||
pr.resource_id = $1
|
||||
`, [shorturl.id]);
|
||||
|
||||
// 2. 获取团队关联(通过项目)
|
||||
const teamIds = projectsResult.rows.map((p: Record<string, unknown>) => p.project_id);
|
||||
const teamsResult = teamIds.length > 0 ? await client.queryObject(`
|
||||
SELECT
|
||||
tp.team_id, tp.project_id,
|
||||
t.name as team_name, t.description as team_description
|
||||
FROM
|
||||
limq.team_projects tp
|
||||
JOIN
|
||||
limq.teams t ON tp.team_id = t.id
|
||||
WHERE
|
||||
tp.project_id = ANY($1::uuid[])
|
||||
`, [teamIds]) : { rows: [] };
|
||||
|
||||
// 3. 获取标签关联
|
||||
const tagsResult = await client.queryObject(`
|
||||
SELECT
|
||||
rt.resource_id, rt.tag_id, rt.created_at,
|
||||
t.name as tag_name, t.type as tag_type
|
||||
FROM
|
||||
limq.resource_tags rt
|
||||
JOIN
|
||||
limq.tags t ON rt.tag_id = t.id
|
||||
WHERE
|
||||
rt.resource_id = $1
|
||||
`, [shorturl.id]);
|
||||
|
||||
// 4. 获取QR码关联
|
||||
const qrCodesResult = await client.queryObject(`
|
||||
SELECT
|
||||
id as qr_id, scan_count, url, template_name, created_at
|
||||
FROM
|
||||
limq.qr_code
|
||||
WHERE
|
||||
resource_id = $1
|
||||
`, [shorturl.id]);
|
||||
|
||||
// 5. 获取渠道关联
|
||||
const channelsResult = await client.queryObject(`
|
||||
SELECT
|
||||
id as channel_id, name as channel_name, path as channel_path,
|
||||
"isUserCreated" as is_user_created
|
||||
FROM
|
||||
limq.channel
|
||||
WHERE
|
||||
"shortUrlId" = $1
|
||||
`, [shorturl.id]);
|
||||
|
||||
// 6. 获取收藏关联
|
||||
const favoritesResult = await client.queryObject(`
|
||||
SELECT
|
||||
f.id as favorite_id, f.user_id, f.created_at,
|
||||
u.first_name, u.last_name
|
||||
FROM
|
||||
limq.favorite f
|
||||
JOIN
|
||||
limq.users u ON f.user_id = u.id
|
||||
WHERE
|
||||
f.favoritable_id = $1 AND f.favoritable_type = 'resource'
|
||||
`, [shorturl.id]);
|
||||
|
||||
// 调试日志
|
||||
console.log(`\n处理资源ID: ${shorturl.id}`);
|
||||
console.log(`attributes类型: ${typeof shorturl.attributes}`);
|
||||
|
||||
// 改进的attributes解析逻辑
|
||||
let attributes: ResourceAttributes = {};
|
||||
try {
|
||||
if (typeof shorturl.attributes === 'string') {
|
||||
// 如果是字符串,尝试解析为JSON
|
||||
console.log(`尝试解析attributes字符串,长度: ${shorturl.attributes.length}`);
|
||||
attributes = JSON.parse(shorturl.attributes);
|
||||
} else if (typeof shorturl.attributes === 'object' && shorturl.attributes !== null) {
|
||||
// 如果已经是对象,直接使用
|
||||
console.log('attributes已经是对象类型');
|
||||
attributes = shorturl.attributes as ResourceAttributes;
|
||||
} else {
|
||||
console.log(`无效的attributes类型: ${typeof shorturl.attributes}`);
|
||||
attributes = {};
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn(`无法解析资源 ${shorturl.id} 的attributes JSON:`, error.message);
|
||||
// 尝试进行更多原始数据分析
|
||||
if (typeof shorturl.attributes === 'string') {
|
||||
console.log(`原始字符串前100字符: ${shorturl.attributes.substring(0, 100)}`);
|
||||
}
|
||||
attributes = {};
|
||||
}
|
||||
|
||||
// 尝试从QR码获取数据
|
||||
let slugFromQr = "";
|
||||
const urlFromQr = "";
|
||||
|
||||
if (qrCodesResult.rows.length > 0 && qrCodesResult.rows[0].url) {
|
||||
const qrUrl = qrCodesResult.rows[0].url as string;
|
||||
console.log(`找到QR码URL: ${qrUrl}`);
|
||||
|
||||
try {
|
||||
const urlParts = qrUrl.split('/');
|
||||
slugFromQr = urlParts[urlParts.length - 1];
|
||||
console.log(`从QR码URL提取的slug: ${slugFromQr}`);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.log('无法从QR码URL提取slug:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 日志输出实际字段值
|
||||
console.log(`提取字段 - name: ${attributes.name || 'N/A'}, slug: ${attributes.slug || 'N/A'}`);
|
||||
console.log(`提取字段 - originalUrl: ${attributes.originalUrl || 'N/A'}, original_url: ${attributes.original_url || 'N/A'}`);
|
||||
|
||||
// 整合所有数据
|
||||
const slug = attributes.slug || attributes.name || slugFromQr || "";
|
||||
const originalUrl = attributes.originalUrl || attributes.original_url || urlFromQr || "";
|
||||
|
||||
console.log(`最终使用的slug: ${slug}`);
|
||||
console.log(`最终使用的originalUrl: ${originalUrl}`);
|
||||
|
||||
enriched.push({
|
||||
...shorturl,
|
||||
attributes,
|
||||
projects: projectsResult.rows,
|
||||
teams: teamsResult.rows,
|
||||
tags: tagsResult.rows,
|
||||
qrCodes: qrCodesResult.rows,
|
||||
channels: channelsResult.rows,
|
||||
favorites: favoritesResult.rows,
|
||||
// 从attributes中提取特定字段 - 使用改进的顺序和QR码备选
|
||||
slug,
|
||||
originalUrl,
|
||||
title: attributes.title || "",
|
||||
description: attributes.description || "",
|
||||
expiresAt: attributes.expires_at || attributes.expiresAt || null
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
|
||||
return enriched;
|
||||
}
|
||||
|
||||
// 将PostgreSQL数据格式化为ClickHouse格式
|
||||
function formatForClickhouse(shorturls: Record<string, unknown>[]) {
|
||||
// 将日期格式化为ClickHouse兼容的DateTime64(3)格式
|
||||
const formatDateTime = (date: Date | string | number | null | undefined): string | null => {
|
||||
if (!date) return null;
|
||||
// 转换为Date对象
|
||||
const dateObj = date instanceof Date ? date : new Date(date);
|
||||
// 返回格式化的字符串: YYYY-MM-DD HH:MM:SS.SSS
|
||||
return dateObj.toISOString().replace('T', ' ').replace('Z', '');
|
||||
};
|
||||
|
||||
console.log(`\n准备格式化 ${shorturls.length} 条记录为ClickHouse格式`);
|
||||
|
||||
return shorturls.map(shorturl => {
|
||||
// 调试日志:输出关键字段
|
||||
console.log(`处理资源: ${shorturl.id}`);
|
||||
console.log(`slug: ${shorturl.slug || 'EMPTY'}`);
|
||||
console.log(`originalUrl: ${shorturl.originalUrl || 'EMPTY'}`);
|
||||
|
||||
// 记录attributes状态
|
||||
const attributesStr = JSON.stringify(shorturl.attributes || {});
|
||||
const attributesPrev = attributesStr.length > 100 ?
|
||||
attributesStr.substring(0, 100) + '...' :
|
||||
attributesStr;
|
||||
console.log(`attributes: ${attributesPrev}`);
|
||||
|
||||
const creatorName = [shorturl.creator_first_name, shorturl.creator_last_name]
|
||||
.filter(Boolean)
|
||||
.join(" ");
|
||||
|
||||
// 格式化项目数据为JSON数组
|
||||
const projects = JSON.stringify((shorturl.projects as Record<string, unknown>[]).map((p) => ({
|
||||
project_id: p.project_id,
|
||||
project_name: p.project_name,
|
||||
project_description: p.project_description,
|
||||
assigned_at: p.assigned_at
|
||||
})));
|
||||
|
||||
// 格式化团队数据为JSON数组
|
||||
const teams = JSON.stringify((shorturl.teams as Record<string, unknown>[]).map((t) => ({
|
||||
team_id: t.team_id,
|
||||
team_name: t.team_name,
|
||||
team_description: t.team_description,
|
||||
via_project_id: t.project_id
|
||||
})));
|
||||
|
||||
// 格式化标签数据为JSON数组
|
||||
const tags = JSON.stringify((shorturl.tags as Record<string, unknown>[]).map((t) => ({
|
||||
tag_id: t.tag_id,
|
||||
tag_name: t.tag_name,
|
||||
tag_type: t.tag_type,
|
||||
created_at: t.created_at
|
||||
})));
|
||||
|
||||
// 格式化QR码数据为JSON数组
|
||||
const qrCodes = JSON.stringify((shorturl.qrCodes as Record<string, unknown>[]).map((q) => ({
|
||||
qr_id: q.qr_id,
|
||||
scan_count: q.scan_count,
|
||||
url: q.url,
|
||||
template_name: q.template_name,
|
||||
created_at: q.created_at
|
||||
})));
|
||||
|
||||
// 格式化渠道数据为JSON数组
|
||||
const channels = JSON.stringify((shorturl.channels as Record<string, unknown>[]).map((c) => ({
|
||||
channel_id: c.channel_id,
|
||||
channel_name: c.channel_name,
|
||||
channel_path: c.channel_path,
|
||||
is_user_created: c.is_user_created
|
||||
})));
|
||||
|
||||
// 格式化收藏数据为JSON数组
|
||||
const favorites = JSON.stringify((shorturl.favorites as Record<string, unknown>[]).map((f) => ({
|
||||
favorite_id: f.favorite_id,
|
||||
user_id: f.user_id,
|
||||
user_name: `${f.first_name || ""} ${f.last_name || ""}`.trim(),
|
||||
created_at: f.created_at
|
||||
})));
|
||||
|
||||
// 统计信息(可通过events表聚合或在其他地方设置)
|
||||
const clickCount = (shorturl.attributes as ResourceAttributes).click_count as number || 0;
|
||||
const uniqueVisitors = 0;
|
||||
|
||||
// 返回ClickHouse格式数据
|
||||
return {
|
||||
id: shorturl.id,
|
||||
external_id: shorturl.external_id || "",
|
||||
type: shorturl.type,
|
||||
slug: shorturl.slug || "",
|
||||
original_url: shorturl.originalUrl || "",
|
||||
title: shorturl.title || "",
|
||||
description: shorturl.description || "",
|
||||
attributes: JSON.stringify(shorturl.attributes || {}),
|
||||
schema_version: shorturl.schema_version || 1,
|
||||
creator_id: shorturl.creator_id || "",
|
||||
creator_email: shorturl.creator_email || "",
|
||||
creator_name: creatorName,
|
||||
created_at: formatDateTime(shorturl.created_at as Date),
|
||||
updated_at: formatDateTime(shorturl.updated_at as Date),
|
||||
deleted_at: formatDateTime(shorturl.deleted_at as Date | null),
|
||||
projects,
|
||||
teams,
|
||||
tags,
|
||||
qr_codes: qrCodes,
|
||||
channels,
|
||||
favorites,
|
||||
expires_at: formatDateTime(shorturl.expiresAt as Date | null),
|
||||
click_count: clickCount,
|
||||
unique_visitors: uniqueVisitors
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// 获取ClickHouse配置
|
||||
async function getClickHouseConfig(): Promise<ChConfig> {
|
||||
try {
|
||||
// 使用getVariable而不是getResource获取ClickHouse配置
|
||||
const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
console.log("原始ClickHouse配置:", typeof chConfigJson);
|
||||
|
||||
// 确保配置不为空
|
||||
if (!chConfigJson) {
|
||||
throw new Error("未找到ClickHouse配置");
|
||||
}
|
||||
|
||||
// 解析JSON字符串为对象
|
||||
let chConfig: ChConfig;
|
||||
if (typeof chConfigJson === 'string') {
|
||||
try {
|
||||
chConfig = JSON.parse(chConfigJson);
|
||||
} catch (parseError) {
|
||||
console.error("解析JSON失败:", parseError);
|
||||
throw new Error("ClickHouse配置不是有效的JSON");
|
||||
}
|
||||
} else {
|
||||
chConfig = chConfigJson as ChConfig;
|
||||
}
|
||||
|
||||
// 验证配置
|
||||
if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
|
||||
chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
|
||||
console.log(`已构建ClickHouse URL: ${chConfig.clickhouse_url}`);
|
||||
}
|
||||
|
||||
if (!chConfig.clickhouse_url) {
|
||||
throw new Error("ClickHouse配置缺少URL");
|
||||
}
|
||||
|
||||
return chConfig;
|
||||
} catch (error) {
|
||||
console.error("获取ClickHouse配置失败:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 写入数据到ClickHouse
|
||||
async function insertToClickhouse(data: Record<string, unknown>[]) {
|
||||
if (data.length === 0) return 0;
|
||||
|
||||
// 获取ClickHouse连接信息
|
||||
const chConfig = await getClickHouseConfig();
|
||||
|
||||
// 确保URL有效
|
||||
if (!chConfig.clickhouse_url) {
|
||||
throw new Error("无效的ClickHouse URL: 未定义");
|
||||
}
|
||||
|
||||
console.log(`准备写入数据到ClickHouse: ${chConfig.clickhouse_url}`);
|
||||
|
||||
// 构建INSERT查询
|
||||
const columns = Object.keys(data[0]).join(", ");
|
||||
|
||||
// 收集所有记录的ID
|
||||
const recordIds = data.map(record => record.id as string);
|
||||
console.log(`需要处理的记录数: ${recordIds.length}`);
|
||||
|
||||
// 先删除可能存在的重复记录
|
||||
try {
|
||||
console.log(`删除可能存在的重复记录...`);
|
||||
|
||||
// 按批次处理删除,避免请求过大
|
||||
const deleteBatchSize = 100;
|
||||
for (let i = 0; i < recordIds.length; i += deleteBatchSize) {
|
||||
const idBatch = recordIds.slice(i, i + deleteBatchSize);
|
||||
const formattedIds = idBatch.map(id => `'${id}'`).join(', ');
|
||||
|
||||
const deleteQuery = `
|
||||
ALTER TABLE shorturl_analytics.shorturl
|
||||
DELETE WHERE id IN (${formattedIds})
|
||||
`;
|
||||
|
||||
const response = await fetch(chConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: deleteQuery,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.warn(`删除记录时出错 (批次 ${i/deleteBatchSize + 1}): ${errorText}`);
|
||||
// 继续执行,不中断流程
|
||||
} else {
|
||||
console.log(`成功删除批次 ${i/deleteBatchSize + 1}/${Math.ceil(recordIds.length/deleteBatchSize)}的潜在重复记录`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`删除重复记录时出错: ${(error as Error).message}`);
|
||||
// 继续执行,不因为删除失败而中断整个过程
|
||||
}
|
||||
|
||||
const query = `
|
||||
INSERT INTO shorturl_analytics.shorturl (${columns})
|
||||
FORMAT JSONEachRow
|
||||
`;
|
||||
|
||||
// 批量插入
|
||||
let inserted = 0;
|
||||
const batchSize = 100;
|
||||
|
||||
for (let i = 0; i < data.length; i += batchSize) {
|
||||
const batch = data.slice(i, i + batchSize);
|
||||
|
||||
// 使用JSONEachRow格式
|
||||
const rows = batch.map(row => JSON.stringify(row)).join('\n');
|
||||
|
||||
// 使用HTTP接口执行查询
|
||||
try {
|
||||
console.log(`正在发送请求到: ${chConfig.clickhouse_url}`);
|
||||
console.log(`认证信息: ${chConfig.clickhouse_user}:***`);
|
||||
|
||||
const response = await fetch(chConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: `${query}\n${rows}`,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse插入失败: ${errorText}`);
|
||||
}
|
||||
|
||||
inserted += batch.length;
|
||||
console.log(`已插入 ${inserted}/${data.length} 条记录`);
|
||||
} catch (error) {
|
||||
console.error(`请求ClickHouse时出错:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return inserted;
|
||||
}
|
||||
660
windmill/sync_shorturl_to_clickhouse_intime.ts
Normal file
660
windmill/sync_shorturl_to_clickhouse_intime.ts
Normal file
@@ -0,0 +1,660 @@
|
||||
// 文件名: sync_resource_relations.ts
|
||||
// 描述: 此脚本用于同步PostgreSQL中资源关联数据到ClickHouse
|
||||
// 作者: AI Assistant
|
||||
// 创建日期: 2023-10-31
|
||||
|
||||
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
|
||||
import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
|
||||
|
||||
// PostgreSQL配置接口
|
||||
interface PgConfig {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
password: string;
|
||||
dbname?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// ClickHouse配置接口
|
||||
interface ChConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_url?: string;
|
||||
}
|
||||
|
||||
// 资源相关接口定义
|
||||
interface TeamData {
|
||||
team_id: string;
|
||||
team_name: string;
|
||||
team_description?: string;
|
||||
project_id?: string;
|
||||
}
|
||||
|
||||
interface ProjectData {
|
||||
project_id: string;
|
||||
project_name: string;
|
||||
project_description?: string;
|
||||
assigned_at?: string;
|
||||
resource_id?: string;
|
||||
}
|
||||
|
||||
interface TagData {
|
||||
tag_id: string;
|
||||
tag_name: string;
|
||||
tag_type?: string;
|
||||
created_at?: string;
|
||||
resource_id?: string;
|
||||
}
|
||||
|
||||
interface FavoriteData {
|
||||
favorite_id: string;
|
||||
user_id: string;
|
||||
first_name?: string;
|
||||
last_name?: string;
|
||||
email?: string;
|
||||
created_at?: string;
|
||||
}
|
||||
|
||||
// 资源关联数据接口
|
||||
interface ResourceRelations {
|
||||
resource_id: string;
|
||||
teams?: TeamData[];
|
||||
projects?: ProjectData[];
|
||||
tags?: TagData[];
|
||||
favorites?: FavoriteData[];
|
||||
external_id?: string;
|
||||
type?: string;
|
||||
attributes?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步PostgreSQL资源关联数据到ClickHouse
|
||||
*/
|
||||
export async function main(
|
||||
params: {
|
||||
/** 要同步的资源ID列表 */
|
||||
resource_ids: string[];
|
||||
/** 是否同步teams数据 */
|
||||
sync_teams?: boolean;
|
||||
/** 是否同步projects数据 */
|
||||
sync_projects?: boolean;
|
||||
/** 是否同步tags数据 */
|
||||
sync_tags?: boolean;
|
||||
/** 是否同步favorites数据 */
|
||||
sync_favorites?: boolean;
|
||||
/** 是否为测试模式(不执行实际更新) */
|
||||
dry_run?: boolean;
|
||||
/** 是否显示详细日志 */
|
||||
verbose?: boolean;
|
||||
}
|
||||
) {
|
||||
// 设置默认参数
|
||||
const resource_ids = params.resource_ids || [];
|
||||
const sync_teams = params.sync_teams !== false;
|
||||
const sync_projects = params.sync_projects !== false;
|
||||
const sync_tags = params.sync_tags !== false;
|
||||
const sync_favorites = params.sync_favorites !== false;
|
||||
const dry_run = params.dry_run || false;
|
||||
const verbose = params.verbose || false;
|
||||
|
||||
if (resource_ids.length === 0) {
|
||||
return { success: false, message: "至少需要提供一个资源ID" };
|
||||
}
|
||||
|
||||
// 初始化日志函数
|
||||
const log = (message: string, isVerbose = false) => {
|
||||
if (!isVerbose || verbose) {
|
||||
console.log(message);
|
||||
}
|
||||
};
|
||||
|
||||
log(`开始同步资源关联数据: ${resource_ids.join(", ")}`);
|
||||
log(`同步选项: teams=${sync_teams}, projects=${sync_projects}, tags=${sync_tags}, favorites=${sync_favorites}`, true);
|
||||
|
||||
let pgPool: Pool | null = null;
|
||||
|
||||
try {
|
||||
// 1. 获取数据库配置
|
||||
log("获取PostgreSQL数据库配置...", true);
|
||||
const pgConfig = await getResource('f/limq/postgresql') as PgConfig;
|
||||
|
||||
// 2. 创建PostgreSQL连接池
|
||||
pgPool = new Pool({
|
||||
hostname: pgConfig.host,
|
||||
port: pgConfig.port,
|
||||
user: pgConfig.user,
|
||||
password: pgConfig.password,
|
||||
database: pgConfig.dbname || 'postgres'
|
||||
}, 3);
|
||||
|
||||
// 3. 获取需要更新的资源完整数据
|
||||
const resourcesData = await getResourcesWithRelations(pgPool, resource_ids, {
|
||||
sync_teams,
|
||||
sync_projects,
|
||||
sync_tags,
|
||||
sync_favorites
|
||||
}, log);
|
||||
|
||||
log(`成功获取 ${resourcesData.length} 个资源的关联数据`);
|
||||
|
||||
if (resourcesData.length === 0) {
|
||||
return { success: true, message: "没有找到需要更新的资源数据", updated: 0 };
|
||||
}
|
||||
|
||||
// 4. 获取ClickHouse配置
|
||||
const chConfig = await getClickHouseConfig();
|
||||
|
||||
// 5. 对每个资源执行更新
|
||||
if (!dry_run) {
|
||||
// 5a. 更新shorturl表数据
|
||||
const shorturlUpdated = await updateClickHouseShorturl(resourcesData, chConfig, log);
|
||||
|
||||
// 5b. 更新events表数据
|
||||
const eventsUpdated = await updateClickHouseEvents(resourcesData, chConfig, log);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: "资源关联数据同步完成",
|
||||
shorturl_updated: shorturlUpdated,
|
||||
events_updated: eventsUpdated,
|
||||
total_updated: shorturlUpdated + eventsUpdated
|
||||
};
|
||||
} else {
|
||||
log("测试模式: 不执行实际更新");
|
||||
if (resourcesData.length > 0) {
|
||||
log("示例数据:");
|
||||
log(JSON.stringify(resourcesData[0], null, 2));
|
||||
}
|
||||
return { success: true, dry_run: true, resources: resourcesData };
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = `同步过程中发生错误: ${(error as Error).message}`;
|
||||
log(errorMessage);
|
||||
if ((error as Error).stack) {
|
||||
log(`错误堆栈: ${(error as Error).stack}`, true);
|
||||
}
|
||||
return { success: false, message: errorMessage };
|
||||
} finally {
|
||||
if (pgPool) {
|
||||
await pgPool.end();
|
||||
log("PostgreSQL连接池已关闭", true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从PostgreSQL获取资源及其关联数据
|
||||
*/
|
||||
async function getResourcesWithRelations(
|
||||
pgPool: Pool,
|
||||
resourceIds: string[],
|
||||
options: {
|
||||
sync_teams: boolean;
|
||||
sync_projects: boolean;
|
||||
sync_tags: boolean;
|
||||
sync_favorites: boolean;
|
||||
},
|
||||
log: (message: string, isVerbose?: boolean) => void
|
||||
): Promise<ResourceRelations[]> {
|
||||
const client = await pgPool.connect();
|
||||
|
||||
try {
|
||||
// 准备资源IDs参数
|
||||
const resourceIdsParam = resourceIds.map(id => `'${id}'`).join(',');
|
||||
|
||||
// 1. 获取基本资源信息
|
||||
log(`获取资源基本信息: ${resourceIdsParam}`, true);
|
||||
const resourcesQuery = `
|
||||
SELECT
|
||||
r.id,
|
||||
r.external_id,
|
||||
r.type,
|
||||
r.attributes,
|
||||
r.schema_version,
|
||||
r.created_at,
|
||||
r.updated_at
|
||||
FROM
|
||||
limq.resources r
|
||||
WHERE
|
||||
r.id IN (${resourceIdsParam})
|
||||
AND r.deleted_at IS NULL
|
||||
`;
|
||||
|
||||
const resourcesResult = await client.queryObject(resourcesQuery);
|
||||
|
||||
if (resourcesResult.rows.length === 0) {
|
||||
log(`未找到有效的资源数据`, true);
|
||||
return [];
|
||||
}
|
||||
|
||||
// 处理每个资源
|
||||
const enrichedResources: ResourceRelations[] = [];
|
||||
|
||||
for (const resource of resourcesResult.rows) {
|
||||
const resourceId = resource.id as string;
|
||||
log(`处理资源ID: ${resourceId}`, true);
|
||||
|
||||
// 初始化关联数据对象
|
||||
const relationData: ResourceRelations = {
|
||||
resource_id: resourceId,
|
||||
external_id: resource.external_id as string,
|
||||
type: resource.type as string,
|
||||
attributes: parseJsonField(resource.attributes)
|
||||
};
|
||||
|
||||
// 2. 获取项目关联
|
||||
if (options.sync_projects) {
|
||||
const projectsQuery = `
|
||||
SELECT
|
||||
pr.resource_id, pr.project_id,
|
||||
p.name as project_name, p.description as project_description,
|
||||
pr.assigned_at
|
||||
FROM
|
||||
limq.project_resources pr
|
||||
JOIN
|
||||
limq.projects p ON pr.project_id = p.id
|
||||
WHERE
|
||||
pr.resource_id = $1
|
||||
AND p.deleted_at IS NULL
|
||||
`;
|
||||
|
||||
const projectsResult = await client.queryObject(projectsQuery, [resourceId]);
|
||||
relationData.projects = projectsResult.rows as ProjectData[];
|
||||
log(`找到 ${projectsResult.rows.length} 个关联项目`, true);
|
||||
}
|
||||
|
||||
// 3. 获取标签关联
|
||||
if (options.sync_tags) {
|
||||
const tagsQuery = `
|
||||
SELECT
|
||||
rt.resource_id, rt.tag_id, rt.created_at,
|
||||
t.name as tag_name, t.type as tag_type
|
||||
FROM
|
||||
limq.resource_tags rt
|
||||
JOIN
|
||||
limq.tags t ON rt.tag_id = t.id
|
||||
WHERE
|
||||
rt.resource_id = $1
|
||||
AND t.deleted_at IS NULL
|
||||
`;
|
||||
|
||||
const tagsResult = await client.queryObject(tagsQuery, [resourceId]);
|
||||
relationData.tags = tagsResult.rows as TagData[];
|
||||
log(`找到 ${tagsResult.rows.length} 个关联标签`, true);
|
||||
}
|
||||
|
||||
// 4. 获取团队关联(通过项目)
|
||||
if (options.sync_teams && relationData.projects && relationData.projects.length > 0) {
|
||||
const projectIds = relationData.projects.map((p: ProjectData) => p.project_id);
|
||||
|
||||
if (projectIds.length > 0) {
|
||||
const teamsQuery = `
|
||||
SELECT
|
||||
tp.team_id, tp.project_id,
|
||||
t.name as team_name, t.description as team_description
|
||||
FROM
|
||||
limq.team_projects tp
|
||||
JOIN
|
||||
limq.teams t ON tp.team_id = t.id
|
||||
WHERE
|
||||
tp.project_id = ANY($1::uuid[])
|
||||
AND t.deleted_at IS NULL
|
||||
`;
|
||||
|
||||
const teamsResult = await client.queryObject(teamsQuery, [projectIds]);
|
||||
relationData.teams = teamsResult.rows as TeamData[];
|
||||
log(`找到 ${teamsResult.rows.length} 个关联团队`, true);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 获取收藏关联
|
||||
if (options.sync_favorites) {
|
||||
const favoritesQuery = `
|
||||
SELECT
|
||||
f.id as favorite_id, f.user_id, f.created_at,
|
||||
u.first_name, u.last_name, u.email
|
||||
FROM
|
||||
limq.favorite f
|
||||
JOIN
|
||||
limq.users u ON f.user_id = u.id
|
||||
WHERE
|
||||
f.favoritable_id = $1
|
||||
AND f.favoritable_type = 'resource'
|
||||
AND f.deleted_at IS NULL
|
||||
`;
|
||||
|
||||
const favoritesResult = await client.queryObject(favoritesQuery, [resourceId]);
|
||||
relationData.favorites = favoritesResult.rows as FavoriteData[];
|
||||
log(`找到 ${favoritesResult.rows.length} 个收藏记录`, true);
|
||||
}
|
||||
|
||||
// 添加到结果集
|
||||
enrichedResources.push(relationData);
|
||||
}
|
||||
|
||||
return enrichedResources;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新ClickHouse中的shorturl表数据
|
||||
*/
|
||||
async function updateClickHouseShorturl(
|
||||
resources: ResourceRelations[],
|
||||
chConfig: ChConfig,
|
||||
log: (message: string, isVerbose?: boolean) => void
|
||||
): Promise<number> {
|
||||
// 只处理类型为shorturl的资源
|
||||
const shorturls = resources.filter(r => r.type === 'shorturl');
|
||||
|
||||
if (shorturls.length === 0) {
|
||||
log('没有找到shorturl类型的资源,跳过shorturl表更新');
|
||||
return 0;
|
||||
}
|
||||
|
||||
log(`准备更新 ${shorturls.length} 个shorturl资源`);
|
||||
|
||||
let updatedCount = 0;
|
||||
|
||||
// 检查ClickHouse中是否存在shorturl表
|
||||
const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.shorturl');
|
||||
|
||||
if (!tableExists) {
|
||||
log('ClickHouse中未找到shorturl表,请先创建表');
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 对每个资源执行更新
|
||||
for (const shorturl of shorturls) {
|
||||
try {
|
||||
// 格式化团队数据
|
||||
const teams = JSON.stringify(shorturl.teams || []);
|
||||
|
||||
// 格式化项目数据
|
||||
const projects = JSON.stringify(shorturl.projects || []);
|
||||
|
||||
// 格式化标签数据
|
||||
const tags = JSON.stringify((shorturl.tags || []).map((t: TagData) => ({
|
||||
tag_id: t.tag_id,
|
||||
tag_name: t.tag_name,
|
||||
tag_type: t.tag_type,
|
||||
created_at: t.created_at
|
||||
})));
|
||||
|
||||
// 格式化收藏数据
|
||||
const favorites = JSON.stringify((shorturl.favorites || []).map((f: FavoriteData) => ({
|
||||
favorite_id: f.favorite_id,
|
||||
user_id: f.user_id,
|
||||
user_name: `${f.first_name || ""} ${f.last_name || ""}`.trim(),
|
||||
created_at: f.created_at
|
||||
})));
|
||||
|
||||
// 尝试更新ClickHouse数据
|
||||
const updateQuery = `
|
||||
ALTER TABLE shorturl_analytics.shorturl
|
||||
UPDATE
|
||||
teams = '${escapeString(teams)}',
|
||||
projects = '${escapeString(projects)}',
|
||||
tags = '${escapeString(tags)}',
|
||||
favorites = '${escapeString(favorites)}'
|
||||
WHERE id = '${shorturl.resource_id}'
|
||||
`;
|
||||
|
||||
await executeClickHouseQuery(chConfig, updateQuery);
|
||||
log(`更新shorturl完成: ${shorturl.resource_id}`, true);
|
||||
updatedCount++;
|
||||
|
||||
} catch (error) {
|
||||
log(`更新shorturl ${shorturl.resource_id} 失败: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return updatedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新ClickHouse中的events表数据
|
||||
*/
|
||||
async function updateClickHouseEvents(
|
||||
resources: ResourceRelations[],
|
||||
chConfig: ChConfig,
|
||||
log: (message: string, isVerbose?: boolean) => void
|
||||
): Promise<number> {
|
||||
// 过滤出有external_id的资源
|
||||
const resourcesWithExternalId = resources.filter(r => r.external_id && r.external_id.trim() !== '');
|
||||
|
||||
if (resourcesWithExternalId.length === 0) {
|
||||
log('没有找到具有external_id的资源,跳过events表更新');
|
||||
return 0;
|
||||
}
|
||||
|
||||
log(`准备更新events表中与 ${resourcesWithExternalId.length} 个外部ID相关的记录`);
|
||||
|
||||
// 检查ClickHouse中是否存在events表
|
||||
const tableExists = await checkClickHouseTable(chConfig, 'shorturl_analytics.events');
|
||||
|
||||
if (!tableExists) {
|
||||
log('ClickHouse中未找到events表,请先创建表');
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 提取所有的external_id
|
||||
const externalIds = resourcesWithExternalId.map(r => r.external_id).filter(Boolean) as string[];
|
||||
|
||||
// 构建资源数据映射(使用external_id作为键)
|
||||
const resourceMapByExternalId = resourcesWithExternalId.reduce((map, resource) => {
|
||||
if (resource.external_id) {
|
||||
map[resource.external_id] = resource;
|
||||
}
|
||||
return map;
|
||||
}, {} as Record<string, ResourceRelations>);
|
||||
|
||||
// 获取ClickHouse中相关资源的事件记录数量
|
||||
let updatedCount = 0;
|
||||
|
||||
try {
|
||||
// 格式化外部ID列表
|
||||
const formattedExternalIds = externalIds.map(id => `'${id}'`).join(', ');
|
||||
|
||||
// 先查询是否有相关事件
|
||||
const countQuery = `
|
||||
SELECT COUNT(*) as count
|
||||
FROM shorturl_analytics.events
|
||||
WHERE event_id IN (${formattedExternalIds})
|
||||
`;
|
||||
|
||||
const countResult = await executeClickHouseQuery(chConfig, countQuery);
|
||||
const eventCount = parseInt(countResult.trim(), 10);
|
||||
|
||||
if (eventCount === 0) {
|
||||
// 尝试另一种查询方式
|
||||
const alternateCountQuery = `
|
||||
SELECT COUNT(*) as count
|
||||
FROM shorturl_analytics.events
|
||||
WHERE link_id IN (${formattedExternalIds})
|
||||
`;
|
||||
|
||||
const alternateCountResult = await executeClickHouseQuery(chConfig, alternateCountQuery);
|
||||
const alternateEventCount = parseInt(alternateCountResult.trim(), 10);
|
||||
|
||||
if (alternateEventCount === 0) {
|
||||
log('没有找到相关事件记录,跳过events表更新');
|
||||
log(`已尝试的匹配字段: event_id,link_id`, true);
|
||||
return 0;
|
||||
} else {
|
||||
log(`找到 ${alternateEventCount} 条以link_id匹配的事件记录需要更新`);
|
||||
}
|
||||
} else {
|
||||
log(`找到 ${eventCount} 条以event_id匹配的事件记录需要更新`);
|
||||
}
|
||||
|
||||
// 批量更新每个资源相关的事件记录
|
||||
for (const externalId of externalIds) {
|
||||
const resource = resourceMapByExternalId[externalId];
|
||||
|
||||
if (!resource) continue;
|
||||
|
||||
// 获取关联数据
|
||||
const tags = resource.tags ? JSON.stringify(resource.tags) : null;
|
||||
|
||||
if (tags) {
|
||||
// 尝试通过event_id更新事件标签
|
||||
const updateTagsQueryByEventId = `
|
||||
ALTER TABLE shorturl_analytics.events
|
||||
UPDATE link_tags = '${escapeString(tags)}'
|
||||
WHERE event_id = '${externalId}'
|
||||
`;
|
||||
|
||||
await executeClickHouseQuery(chConfig, updateTagsQueryByEventId);
|
||||
log(`尝试通过event_id更新事件标签: ${externalId}`, true);
|
||||
|
||||
// 尝试通过link_id更新事件标签
|
||||
const updateTagsQueryByLinkId = `
|
||||
ALTER TABLE shorturl_analytics.events
|
||||
UPDATE link_tags = '${escapeString(tags)}'
|
||||
WHERE link_id = '${externalId}'
|
||||
`;
|
||||
|
||||
await executeClickHouseQuery(chConfig, updateTagsQueryByLinkId);
|
||||
log(`尝试通过link_id更新事件标签: ${externalId}`, true);
|
||||
}
|
||||
|
||||
// 如果资源有resource_id,也尝试使用它来更新
|
||||
if (resource.resource_id) {
|
||||
const updateByResourceId = `
|
||||
ALTER TABLE shorturl_analytics.events
|
||||
UPDATE link_tags = '${escapeString(tags || '[]')}'
|
||||
WHERE link_id = '${resource.resource_id}'
|
||||
`;
|
||||
|
||||
await executeClickHouseQuery(chConfig, updateByResourceId);
|
||||
log(`尝试通过resource_id更新事件标签: ${resource.resource_id}`, true);
|
||||
}
|
||||
|
||||
updatedCount++;
|
||||
}
|
||||
|
||||
log(`已尝试更新 ${updatedCount} 个资源的事件记录`);
|
||||
|
||||
} catch (error) {
|
||||
log(`更新events表失败: ${(error as Error).message}`);
|
||||
}
|
||||
|
||||
return updatedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取ClickHouse配置
|
||||
*/
|
||||
async function getClickHouseConfig(): Promise<ChConfig> {
|
||||
try {
|
||||
const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
|
||||
// 确保配置不为空
|
||||
if (!chConfigJson) {
|
||||
throw new Error("未找到ClickHouse配置");
|
||||
}
|
||||
|
||||
// 解析JSON字符串为对象
|
||||
let chConfig: ChConfig;
|
||||
if (typeof chConfigJson === 'string') {
|
||||
try {
|
||||
chConfig = JSON.parse(chConfigJson);
|
||||
} catch (_) {
|
||||
throw new Error("ClickHouse配置不是有效的JSON");
|
||||
}
|
||||
} else {
|
||||
chConfig = chConfigJson as ChConfig;
|
||||
}
|
||||
|
||||
// 验证并构建URL
|
||||
if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
|
||||
chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
|
||||
}
|
||||
|
||||
if (!chConfig.clickhouse_url) {
|
||||
throw new Error("ClickHouse配置缺少URL");
|
||||
}
|
||||
|
||||
return chConfig;
|
||||
} catch (error) {
|
||||
throw new Error(`获取ClickHouse配置失败: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查ClickHouse中是否存在指定表
|
||||
*/
|
||||
async function checkClickHouseTable(chConfig: ChConfig, tableName: string): Promise<boolean> {
|
||||
try {
|
||||
const query = `EXISTS TABLE ${tableName}`;
|
||||
const result = await executeClickHouseQuery(chConfig, query);
|
||||
return result.trim() === '1';
|
||||
} catch (error) {
|
||||
console.error(`检查表 ${tableName} 失败:`, error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行ClickHouse查询
|
||||
*/
|
||||
async function executeClickHouseQuery(chConfig: ChConfig, query: string): Promise<string> {
|
||||
// 确保URL有效
|
||||
if (!chConfig.clickhouse_url) {
|
||||
throw new Error("无效的ClickHouse URL: 未定义");
|
||||
}
|
||||
|
||||
// 执行HTTP请求
|
||||
try {
|
||||
const response = await fetch(chConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse查询失败 (${response.status}): ${errorText}`);
|
||||
}
|
||||
|
||||
return await response.text();
|
||||
} catch (error) {
|
||||
throw new Error(`执行ClickHouse查询失败: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析JSON字段
|
||||
*/
|
||||
function parseJsonField(field: unknown): Record<string, unknown> {
|
||||
if (!field) return {};
|
||||
|
||||
try {
|
||||
if (typeof field === 'string') {
|
||||
return JSON.parse(field);
|
||||
} else if (typeof field === 'object') {
|
||||
return field as Record<string, unknown>;
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`无法解析JSON字段:`, error);
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串,避免SQL注入
|
||||
*/
|
||||
function escapeString(str: string): string {
|
||||
if (!str) return '';
|
||||
return str.replace(/'/g, "''");
|
||||
}
|
||||
Reference in New Issue
Block a user