Compare commits
26 Commits
53e1611670
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
51e168ee3b | ||
|
|
cf0f35e274 | ||
| 3162836e91 | |||
| d80d5e976b | |||
| 5d5b501a66 | |||
| fe40aad835 | |||
| 92db5ad783 | |||
| b94a91914a | |||
| 8551f5c445 | |||
| dafa7f53ac | |||
| 0203cb4041 | |||
| ced29201da | |||
| a8c94c9621 | |||
| 4736ebe060 | |||
| 6858f2fda5 | |||
| 42f5be4dcb | |||
| 05af4aae70 | |||
| ed1d2e59f6 | |||
| 3cbb76db36 | |||
| ecef81b0ee | |||
| 9cb85a2910 | |||
| 3af015ca44 | |||
| f6f24d3450 | |||
| 4262f789da | |||
| 2e34cd5b4b | |||
| 2cb45781c7 |
31
.env
Normal file
31
.env
Normal file
@@ -0,0 +1,31 @@
|
||||
PORT=3007
|
||||
|
||||
MONGO_URL="mongodb://10.0.1.41:27017"
|
||||
|
||||
# ClickHouse Configuration
|
||||
CLICKHOUSE_HOST=10.0.1.60
|
||||
CLICKHOUSE_PORT=8123
|
||||
CLICKHOUSE_USER=admin
|
||||
CLICKHOUSE_PASSWORD=your_secure_password
|
||||
CLICKHOUSE_DATABASE=shorturl_analytics
|
||||
CLICKHOUSE_URL=http://10.0.1.60:8123
|
||||
|
||||
REDIS_HOST="localhost"
|
||||
REDIS_PORT="6379"
|
||||
REDIS_PASSWORD=""
|
||||
|
||||
# Supabase Configuration
|
||||
SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
|
||||
SUPABASE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc0NDM0NTY0MywiZXhwIjoyMDU5OTIxNjQzfQ.ZenTsEAdGiDu1DCCOT7G8xxvgFXKLl4qhHB-AhSVf6w"
|
||||
SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
|
||||
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
|
||||
|
||||
|
||||
# Next.js Public Environment Variables (accessible in browser)
|
||||
NEXT_PUBLIC_SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
|
||||
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
|
||||
|
||||
NEXT_PUBLIC_LIMQ_API="https://app.upj.to"
|
||||
# Application URL for redirects (replace with your production URL)
|
||||
NEXT_PUBLIC_SITE_URL="https://main.upj.to"
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -31,7 +31,7 @@ yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
# .env
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
42
README-auth-setup.md
Normal file
42
README-auth-setup.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# 身份验证重定向 URL 配置指南
|
||||
|
||||
## 问题：注册后确认邮件链接指向 localhost
|
||||
|
||||
如果您在生产环境中使用此应用，并且发现用户注册后收到的确认邮件中的链接指向 `localhost` 而非您的实际网站域名，请按照以下步骤解决：
|
||||
|
||||
## 解决方案
|
||||
|
||||
### 1. 设置环境变量
|
||||
|
||||
在项目根目录的 `.env.production` 文件中，确保 `NEXT_PUBLIC_SITE_URL` 变量设置为您的实际生产域名：
|
||||
|
||||
```
|
||||
NEXT_PUBLIC_SITE_URL="https://您的真实域名.com"
|
||||
```
|
||||
|
||||
### 2. 在 Supabase 控制台中配置
|
||||
|
||||
登录 [Supabase 控制台](https://app.supabase.com/)，然后：
|
||||
|
||||
1. 选择您的项目
|
||||
2. 导航到 **Authentication** > **URL Configuration**
|
||||
3. 在 **Site URL** 字段中输入您的实际网站 URL
|
||||
4. 在 **Redirect URLs** 部分添加：
|
||||
- `https://您的真实域名.com/auth/callback`
|
||||
|
||||
### 3. 本地开发与生产环境
|
||||
|
||||
- **开发环境**：使用 `.env.development` 文件中的设置，通常为 `http://localhost:3007`
|
||||
- **生产环境**：使用 `.env.production` 文件中的设置，应为您的实际域名
|
||||
|
||||
### 4. 部署后验证
|
||||
|
||||
项目重新部署后：
|
||||
1. 尝试注册一个新账户
|
||||
2. 检查收到的确认邮件，确认链接现在指向您的实际域名而非 localhost
|
||||
|
||||
## 技术说明
|
||||
|
||||
身份验证流程中，应用使用环境变量 `NEXT_PUBLIC_SITE_URL` 构建重定向 URL。如果未设置此变量，它会回退到使用 `window.location.origin`，这在本地开发时会是 `localhost`。
|
||||
|
||||
通过正确设置此变量，您可以确保无论在何处运行应用，邮件中的链接都能正确指向应用的实际位置。
|
||||
50
README-google-auth.md
Normal file
50
README-google-auth.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# 配置 Google 登录功能
|
||||
|
||||
为了启用 Google 登录功能，您需要在 Supabase 和 Google Cloud Platform 进行配置。
|
||||
|
||||
## 步骤 1: 创建 Google OAuth 客户端
|
||||
|
||||
1. 访问 [Google Cloud Console](https://console.cloud.google.com/)
|
||||
2. 创建一个新项目或选择现有项目
|
||||
3. 在左侧菜单中导航到 "API 和服务" > "OAuth 同意屏幕"
|
||||
4. 选择用户类型（外部或内部），然后点击"创建"
|
||||
5. 填写必要的信息（应用名称、用户支持电子邮件等）并保存
|
||||
6. 导航到 "API 和服务" > "凭据"
|
||||
7. 点击"创建凭据" > "OAuth 客户端 ID"
|
||||
8. 应用类型选择 "Web 应用"
|
||||
9. 名称中输入您的应用名称
|
||||
10. 添加以下已获授权的重定向 URI:
|
||||
- `https://mwwvqwevplndzvmqmrxa.supabase.co/auth/v1/callback`
|
||||
11. 点击"创建"
|
||||
12. 复制生成的 "客户端 ID" 和 "客户端密钥"
|
||||
|
||||
## 步骤 2: 在 Supabase 中配置 Google 提供商
|
||||
|
||||
1. 登录 [Supabase 仪表板](https://app.supabase.com)
|
||||
2. 选择您的项目
|
||||
3. 导航到 "身份验证" > "提供商"
|
||||
4. 找到 Google 提供商并启用它
|
||||
5. 粘贴您刚才获取的 "客户端 ID" 和 "客户端密钥"
|
||||
6. 保存配置
|
||||
|
||||
## 步骤 3: 更新重定向 URL（如有需要）
|
||||
|
||||
如果您的应用需要在登录后重定向到特定页面，请确保在 Google Cloud Console 和 Supabase 中配置了正确的重定向 URL。
|
||||
|
||||
在 Supabase 中:
|
||||
1. 导航到 "身份验证" > "URL 配置"
|
||||
2. 添加您的前端 URL 到站点 URL 字段中
|
||||
3. 设置重定向 URL（通常是您的前端 URL）
|
||||
|
||||
## 测试
|
||||
|
||||
1. 在您的应用中，尝试使用 Google 登录
|
||||
2. 验证认证流程，确保可以成功登录并重定向到应用
|
||||
3. 检查 Supabase 中的用户数据，确认新用户已创建
|
||||
|
||||
## 故障排除
|
||||
|
||||
- 确保重定向 URI 完全匹配
|
||||
- 确保 OAuth 同意屏幕已正确配置
|
||||
- 查看 Supabase 和应用程序中的日志以获取详细的错误信息
|
||||
- 如果遇到 CORS 错误，检查您的站点 URL 配置
|
||||
@@ -14,6 +14,7 @@ import { ProjectSelector } from '@/app/components/ui/ProjectSelector';
|
||||
import { TagSelector } from '@/app/components/ui/TagSelector';
|
||||
import { useSearchParams } from 'next/navigation';
|
||||
import { useShortUrlStore } from '@/app/utils/store';
|
||||
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
|
||||
|
||||
// 事件类型定义
|
||||
interface Event {
|
||||
@@ -41,6 +42,7 @@ interface Event {
|
||||
link_slug?: string;
|
||||
link_tags?: string;
|
||||
ip_address?: string;
|
||||
req_full_path?: string;
|
||||
}
|
||||
|
||||
// 格式化日期函数
|
||||
@@ -100,7 +102,7 @@ const extractEventInfo = (event: Event) => {
|
||||
eventTime: event.created_at || event.event_time,
|
||||
linkName: event.link_label || linkAttrs?.name || eventAttrs?.link_name || event.link_slug || '-',
|
||||
originalUrl: event.link_original_url || eventAttrs?.origin_url || '-',
|
||||
fullUrl: eventAttrs?.full_url || '-',
|
||||
fullUrl: event.req_full_path || eventAttrs?.full_url || '-',
|
||||
eventType: event.event_type || '-',
|
||||
visitorId: event.visitor_id?.substring(0, 8) || '-',
|
||||
referrer: eventAttrs?.referrer || '-',
|
||||
@@ -1108,12 +1110,14 @@ function AnalyticsContent() {
|
||||
// Main page component with Suspense
|
||||
export default function AnalyticsPage() {
|
||||
return (
|
||||
<Suspense fallback={
|
||||
<div className="flex items-center justify-center min-h-screen">
|
||||
<div className="animate-spin rounded-full h-12 w-12 border-t-2 border-b-2 border-blue-500" />
|
||||
</div>
|
||||
}>
|
||||
<AnalyticsContent />
|
||||
</Suspense>
|
||||
<ClientRouteGuard>
|
||||
<Suspense fallback={
|
||||
<div className="flex items-center justify-center min-h-screen">
|
||||
<div className="animate-spin rounded-full h-12 w-12 border-t-2 border-b-2 border-blue-500" />
|
||||
</div>
|
||||
}>
|
||||
<AnalyticsContent />
|
||||
</Suspense>
|
||||
</ClientRouteGuard>
|
||||
);
|
||||
}
|
||||
126
app/api/activities/readme.md
Normal file
126
app/api/activities/readme.md
Normal file
@@ -0,0 +1,126 @@
|
||||
# Activities API Documentation
|
||||
|
||||
## Overview
|
||||
The Activities API provides event tracking data for short URLs. It allows retrieving visitor activity information based on various filters such as URL slug, domain, and date ranges.
|
||||
|
||||
## Endpoint
|
||||
```
|
||||
GET /api/activities
|
||||
```
|
||||
|
||||
## Request Parameters
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|------------|---------|----------|-------------|
|
||||
| `slug` | string | No* | The short URL slug to filter events by |
|
||||
| `domain` | string | No* | The domain to filter events by |
|
||||
| `startTime`| string | No* | Start time for date range filter (ISO format) |
|
||||
| `endTime` | string | No* | End time for date range filter (ISO format) |
|
||||
| `page` | integer | No | Page number for pagination (default: 1) |
|
||||
| `pageSize` | integer | No | Number of records per page (default: 50) |
|
||||
| `format` | string | No | Response format, set to 'csv' for CSV output (default: JSON) |
|
||||
|
||||
\* Either `slug`+`domain` combination OR at least one of `startTime`/`endTime` must be provided.
|
||||
|
||||
## Response Formats
|
||||
|
||||
### JSON Format (Default)
|
||||
JSON responses include the following structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": [
|
||||
{
|
||||
"id": "event-id",
|
||||
"type": "event-type",
|
||||
"time": "timestamp",
|
||||
"visitor": {
|
||||
"id": "visitor-id",
|
||||
"ipAddress": "ip-address",
|
||||
"userAgent": "user-agent-string",
|
||||
"referrer": "referrer-url"
|
||||
},
|
||||
"device": {
|
||||
"type": "device-type",
|
||||
"browser": "browser-name",
|
||||
"os": "operating-system"
|
||||
},
|
||||
"location": {
|
||||
"country": "country-code",
|
||||
"city": "city-name"
|
||||
},
|
||||
"link": {
|
||||
"id": "link-id",
|
||||
"slug": "link-slug",
|
||||
"originalUrl": "original-url",
|
||||
"label": "link-label",
|
||||
"tags": ["tag1", "tag2"]
|
||||
},
|
||||
"utm": {
|
||||
"source": "utm-source",
|
||||
"medium": "utm-medium",
|
||||
"campaign": "utm-campaign",
|
||||
"term": "utm-term",
|
||||
"content": "utm-content"
|
||||
}
|
||||
}
|
||||
],
|
||||
"meta": {
|
||||
"total": 100,
|
||||
"page": 1,
|
||||
"pageSize": 50
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
In case of an error:
|
||||
```json
|
||||
{
|
||||
"success": false,
|
||||
"data": null,
|
||||
"error": "Error message description"
|
||||
}
|
||||
```
|
||||
|
||||
### CSV Format
|
||||
When `format=csv` is specified, the response is returned as plain text in CSV format with the following columns:
|
||||
- `time`: Timestamp of the event
|
||||
- `activity`: Type of activity/event
|
||||
- `campaign`: UTM campaign value (defaults to "demo" if not found)
|
||||
- `clientId`: Visitor ID
|
||||
- `originPath`: Original request path or referrer URL
|
||||
|
||||
## Examples
|
||||
|
||||
### Get activities for a specific short URL
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link
|
||||
```
|
||||
|
||||
### Get activities within a date range
|
||||
```
|
||||
GET /api/activities?startTime=2023-06-01T00:00:00Z&endTime=2023-06-30T23:59:59Z
|
||||
```
|
||||
|
||||
### Get events as CSV
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link&format=csv
|
||||
```
|
||||
|
||||
### Pagination example
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link&page=2&pageSize=20
|
||||
```
|
||||
|
||||
## Error Codes
|
||||
|
||||
| Status Code | Description |
|
||||
|-------------|-------------|
|
||||
| 400 | Missing required parameters |
|
||||
| 500 | Server error while processing the request |
|
||||
|
||||
## Notes
|
||||
- For privacy and security reasons, some fields may be omitted or anonymized based on user settings.
|
||||
- The CSV format is optimized for easy import into spreadsheet applications.
|
||||
- When using the CSV format, the response is returned as plain text rather than a downloadable file.
|
||||
250
app/api/activities/route.ts
Normal file
250
app/api/activities/route.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getEvents } from '@/lib/analytics';
|
||||
import { ApiResponse } from '@/lib/types';
|
||||
|
||||
// Extended Event type with required fields
|
||||
interface EventWithFullPath {
|
||||
event_id?: string;
|
||||
event_time?: string;
|
||||
event_type?: string;
|
||||
visitor_id?: string;
|
||||
ip_address?: string;
|
||||
req_full_path?: string;
|
||||
referrer?: string;
|
||||
event_attributes?: string | Record<string, unknown>;
|
||||
link_tags?: string | string[];
|
||||
link_id?: string;
|
||||
link_slug?: string;
|
||||
link_original_url?: string;
|
||||
link_label?: string;
|
||||
device_type?: string;
|
||||
browser?: string;
|
||||
os?: string;
|
||||
country?: string;
|
||||
city?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
|
||||
// Get parameters
|
||||
const slug = searchParams.get('slug');
|
||||
const domain = searchParams.get('domain');
|
||||
const format = searchParams.get('format');
|
||||
|
||||
// Optional date range parameters
|
||||
const startTime = searchParams.get('startTime') || undefined;
|
||||
const endTime = searchParams.get('endTime') || undefined;
|
||||
|
||||
// Check if either slug or domain is provided without the other
|
||||
if ((slug && !domain) || (!slug && domain)) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Both slug and domain parameters must be provided together'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// Ensure either slug+domain or date range is provided
|
||||
if ((!slug && !domain) && (!startTime && !endTime)) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Missing filter parameters: provide either slug+domain or date range'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// Construct the shortUrl from domain and slug if both are provided
|
||||
let shortUrl = undefined;
|
||||
if (slug && domain) {
|
||||
shortUrl = `https://${domain}/${slug}`;
|
||||
|
||||
// Log the request for debugging
|
||||
console.log('Activities API received parameters:', {
|
||||
slug,
|
||||
domain,
|
||||
shortUrl,
|
||||
startTime,
|
||||
endTime
|
||||
});
|
||||
} else {
|
||||
console.log('Activities API using time range filter:', {
|
||||
startTime,
|
||||
endTime
|
||||
});
|
||||
}
|
||||
|
||||
// Set default page size and page
|
||||
const page = parseInt(searchParams.get('page') || '1');
|
||||
const pageSize = parseInt(searchParams.get('pageSize') || '50');
|
||||
|
||||
// Get events for the specified filters
|
||||
const { events, total } = await getEvents({
|
||||
linkSlug: slug || undefined,
|
||||
page,
|
||||
pageSize,
|
||||
startTime,
|
||||
endTime,
|
||||
sortBy: 'event_time',
|
||||
sortOrder: 'desc'
|
||||
});
|
||||
|
||||
// If format=csv, return CSV format data
|
||||
if (format === 'csv') {
|
||||
// CSV header line
|
||||
let csvContent = 'time,activity,campaign,clientId,originPath\n';
|
||||
|
||||
// Helper function to extract utm_campaign from URL
|
||||
const extractUtmCampaign = (url: string | null | undefined): string => {
|
||||
if (!url) return 'demo';
|
||||
|
||||
try {
|
||||
// Try to parse URL and extract utm_campaign parameter
|
||||
const urlObj = new URL(url.startsWith('http') ? url : `https://example.com${url}`);
|
||||
const campaign = urlObj.searchParams.get('utm_campaign');
|
||||
if (campaign) return campaign;
|
||||
|
||||
// If utm_campaign is not found or URL parsing fails, use regex as fallback
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
|
||||
} catch {
|
||||
// If URL parsing fails, try regex directly
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
|
||||
}
|
||||
|
||||
return 'demo'; // Default value
|
||||
};
|
||||
|
||||
// Process each event record
|
||||
events.forEach(event => {
|
||||
// 使用类型断言处理扩展字段
|
||||
const eventWithFullPath = event as unknown as EventWithFullPath;
|
||||
|
||||
// Get the full URL from appropriate field
|
||||
// Try different possible fields that might contain the URL
|
||||
const fullUrl = eventWithFullPath.req_full_path || eventWithFullPath.referrer || '';
|
||||
|
||||
// Extract campaign from URL
|
||||
const campaign = extractUtmCampaign(fullUrl);
|
||||
|
||||
// Format time
|
||||
const time = eventWithFullPath.event_time ?
|
||||
new Date(eventWithFullPath.event_time).toISOString().replace('T', ' ').slice(0, 19) :
|
||||
'';
|
||||
|
||||
// Determine activity (event_type)
|
||||
const activity = eventWithFullPath.event_type || '';
|
||||
|
||||
// 修改：使用link_label替代visitor_id作为clientId
|
||||
const clientId = eventWithFullPath.link_label || 'undefined';
|
||||
|
||||
// Original path - 修正：使用link_original_url作为原始URL来源
|
||||
const originPath = eventWithFullPath.link_original_url || 'undefined';
|
||||
|
||||
// Add to CSV content
|
||||
csvContent += `${time},${activity},${campaign},${clientId},${originPath}\n`;
|
||||
});
|
||||
|
||||
// No need to generate filename since we're not using Content-Disposition header
|
||||
|
||||
// Return CSV response without forcing download
|
||||
return new NextResponse(csvContent, {
|
||||
headers: {
|
||||
'Content-Type': 'text/plain'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Process the events to extract useful information
|
||||
const processedEvents = events.map(event => {
|
||||
// Parse JSON strings to objects safely
|
||||
let eventAttributes: Record<string, unknown> = {};
|
||||
|
||||
try {
|
||||
if (typeof event.event_attributes === 'string') {
|
||||
eventAttributes = JSON.parse(event.event_attributes);
|
||||
} else if (typeof event.event_attributes === 'object') {
|
||||
eventAttributes = event.event_attributes;
|
||||
}
|
||||
} catch {
|
||||
// Keep default empty object if parsing fails
|
||||
}
|
||||
|
||||
// Extract tags
|
||||
let tags: string[] = [];
|
||||
|
||||
try {
|
||||
if (typeof event.link_tags === 'string') {
|
||||
const parsedTags = JSON.parse(event.link_tags);
|
||||
if (Array.isArray(parsedTags)) {
|
||||
tags = parsedTags;
|
||||
}
|
||||
} else if (Array.isArray(event.link_tags)) {
|
||||
tags = event.link_tags;
|
||||
}
|
||||
} catch {
|
||||
// If parsing fails, keep tags as empty array
|
||||
}
|
||||
|
||||
// Return a simplified event object
|
||||
return {
|
||||
id: event.event_id,
|
||||
type: event.event_type,
|
||||
time: event.event_time,
|
||||
visitor: {
|
||||
id: event.visitor_id,
|
||||
ipAddress: event.ip_address,
|
||||
userAgent: eventAttributes.user_agent as string || null,
|
||||
referrer: eventAttributes.referrer as string || null
|
||||
},
|
||||
device: {
|
||||
type: event.device_type,
|
||||
browser: event.browser,
|
||||
os: event.os
|
||||
},
|
||||
location: {
|
||||
country: event.country,
|
||||
city: event.city
|
||||
},
|
||||
link: {
|
||||
id: event.link_id,
|
||||
slug: event.link_slug,
|
||||
originalUrl: event.link_original_url,
|
||||
label: event.link_label,
|
||||
tags
|
||||
},
|
||||
utm: {
|
||||
source: eventAttributes.utm_source as string || null,
|
||||
medium: eventAttributes.utm_medium as string || null,
|
||||
campaign: eventAttributes.utm_campaign as string || null,
|
||||
term: eventAttributes.utm_term as string || null,
|
||||
content: eventAttributes.utm_content as string || null
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Return processed events
|
||||
const response: ApiResponse<typeof processedEvents> = {
|
||||
success: true,
|
||||
data: processedEvents,
|
||||
meta: {
|
||||
total,
|
||||
page,
|
||||
pageSize
|
||||
}
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error retrieving activities:', error);
|
||||
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
data: null,
|
||||
error: error instanceof Error ? error.message : 'An error occurred while retrieving activities'
|
||||
};
|
||||
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
18
app/auth/callback/route.ts
Normal file
18
app/auth/callback/route.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
|
||||
import { cookies } from 'next/headers';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const requestUrl = new URL(request.url);
|
||||
const code = requestUrl.searchParams.get('code');
|
||||
|
||||
if (code) {
|
||||
const cookieStore = cookies();
|
||||
const supabase = createRouteHandlerClient({ cookies: () => cookieStore });
|
||||
await supabase.auth.exchangeCodeForSession(code);
|
||||
}
|
||||
|
||||
// URL to redirect to after sign in process completes
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || 'https://main.upj.to';
|
||||
return NextResponse.redirect(new URL('/analytics', siteUrl));
|
||||
}
|
||||
45
app/components/ClientRouteGuard.tsx
Normal file
45
app/components/ClientRouteGuard.tsx
Normal file
@@ -0,0 +1,45 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
|
||||
// 这个组件会检查 localStorage 中是否有认证令牌，如果没有则重定向到登录页面
|
||||
export default function ClientRouteGuard({ children }: { children: React.ReactNode }) {
|
||||
const router = useRouter();
|
||||
const [isAuthenticated, setIsAuthenticated] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
// 检查 localStorage 中是否有认证令牌
|
||||
const checkAuth = () => {
|
||||
// 查找 Supabase 认证令牌
|
||||
const hasAuthToken = !!localStorage.getItem('sb-mwwvqwevplndzvmqmrxa-auth-token') ||
|
||||
!!localStorage.getItem('sb-auth-token');
|
||||
|
||||
if (!hasAuthToken) {
|
||||
// 如果没有令牌，重定向到登录页面
|
||||
router.push('/login');
|
||||
} else {
|
||||
setIsAuthenticated(true);
|
||||
}
|
||||
setIsLoading(false);
|
||||
};
|
||||
|
||||
checkAuth();
|
||||
}, [router]);
|
||||
|
||||
// 显示加载状态
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen">
|
||||
<div className="text-center">
|
||||
<div className="animate-spin rounded-full h-16 w-16 border-t-2 border-b-2 border-blue-500 mx-auto"></div>
|
||||
<p className="mt-4 text-lg text-gray-700 dark:text-gray-300">加载中...</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// 只有当用户已认证时才显示子组件
|
||||
return isAuthenticated ? <>{children}</> : null;
|
||||
}
|
||||
@@ -44,6 +44,11 @@ export default function Header() {
|
||||
Short Links
|
||||
</Link>
|
||||
</li>
|
||||
<li>
|
||||
<Link href="/create-shorturl" className="text-sm text-gray-700 hover:text-blue-500">
|
||||
Create Short URL
|
||||
</Link>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
)}
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
'use client';
|
||||
|
||||
import Link from 'next/link';
|
||||
|
||||
export default function Navbar() {
|
||||
return (
|
||||
<header className="w-full py-4 border-b border-card-border bg-background">
|
||||
<div className="container flex items-center justify-between px-4 mx-auto">
|
||||
<div className="flex items-center space-x-4">
|
||||
<Link href="/" className="flex items-center space-x-2">
|
||||
<svg
|
||||
className="w-6 h-6 text-accent-blue"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
|
||||
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
|
||||
</svg>
|
||||
<span className="text-xl font-bold text-foreground">ShortURL</span>
|
||||
</Link>
|
||||
<nav className="hidden space-x-4 md:flex">
|
||||
<Link
|
||||
href="/links"
|
||||
className="text-sm text-foreground hover:text-accent-blue transition-colors"
|
||||
>
|
||||
Links
|
||||
</Link>
|
||||
<Link
|
||||
href="/analytics"
|
||||
className="text-sm text-foreground hover:text-accent-blue transition-colors"
|
||||
>
|
||||
Analytics
|
||||
</Link>
|
||||
</nav>
|
||||
</div>
|
||||
<div className="flex items-center space-x-3">
|
||||
<button className="p-2 text-sm text-foreground rounded-md gradient-border">
|
||||
Upgrade
|
||||
</button>
|
||||
<button className="p-2 text-sm text-foreground hover:text-accent-blue">
|
||||
<svg
|
||||
className="w-5 h-5"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<circle cx="12" cy="12" r="10"></circle>
|
||||
<circle cx="12" cy="10" r="3"></circle>
|
||||
<path d="M7 20.662V19a2 2 0 0 1 2-2h6a2 2 0 0 1 2 2v1.662"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { getSupabaseClient } from '../../utils/supabase';
|
||||
import { AuthChangeEvent, Session } from '@supabase/supabase-js';
|
||||
import { Loader2, X, Check } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { limqRequest } from '@/lib/api';
|
||||
|
||||
type Team = Database['limq']['Tables']['teams']['Row'];
|
||||
|
||||
@@ -69,6 +70,14 @@ export function TeamSelector({
|
||||
|
||||
try {
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
// 尝试创建默认团队和项目（如果用户还没有）
|
||||
try {
|
||||
const response = await limqRequest('team/create-default', 'POST');
|
||||
console.log('Default team creation response:', response);
|
||||
} catch (teamError) {
|
||||
console.error('Error creating default team:', teamError);
|
||||
}
|
||||
|
||||
const { data: memberships, error: membershipError } = await supabase
|
||||
.from('team_membership')
|
||||
|
||||
403
app/create-shorturl/page.tsx
Normal file
403
app/create-shorturl/page.tsx
Normal file
@@ -0,0 +1,403 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useAuth } from '@/lib/auth';
|
||||
import { limqRequest } from '@/lib/api';
|
||||
import { TeamSelector } from '@/app/components/ui/TeamSelector';
|
||||
import { ProjectSelector } from '@/app/components/ui/ProjectSelector';
|
||||
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
|
||||
|
||||
interface ShortUrlData {
|
||||
originalUrl: string;
|
||||
customSlug?: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
tags?: string[];
|
||||
teamId: string;
|
||||
projectId: string;
|
||||
domain: string;
|
||||
}
|
||||
|
||||
export default function CreateShortUrlPage() {
|
||||
return (
|
||||
<ClientRouteGuard>
|
||||
<CreateShortUrlForm />
|
||||
</ClientRouteGuard>
|
||||
);
|
||||
}
|
||||
|
||||
function CreateShortUrlForm() {
|
||||
const router = useRouter();
|
||||
const { user } = useAuth();
|
||||
|
||||
const [formData, setFormData] = useState<ShortUrlData>({
|
||||
originalUrl: '',
|
||||
customSlug: '',
|
||||
title: '',
|
||||
description: '',
|
||||
tags: [],
|
||||
teamId: '',
|
||||
projectId: '',
|
||||
domain: 'googleads.link',
|
||||
});
|
||||
|
||||
const [tagInput, setTagInput] = useState('');
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [success, setSuccess] = useState(false);
|
||||
|
||||
// Use useEffect to add user information to form data on load
|
||||
useEffect(() => {
|
||||
if (user) {
|
||||
console.log('Current user:', user.email);
|
||||
// Can add user-related data to the form here
|
||||
}
|
||||
}, [user]);
|
||||
|
||||
const handleChange = (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
|
||||
const { name, value } = e.target;
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
[name]: value
|
||||
}));
|
||||
};
|
||||
|
||||
const handleTagKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === 'Enter' && tagInput.trim()) {
|
||||
e.preventDefault();
|
||||
addTag();
|
||||
}
|
||||
};
|
||||
|
||||
const addTag = () => {
|
||||
if (tagInput.trim() && !formData.tags?.includes(tagInput.trim())) {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
tags: [...(prev.tags || []), tagInput.trim()]
|
||||
}));
|
||||
setTagInput('');
|
||||
}
|
||||
};
|
||||
|
||||
const removeTag = (tagToRemove: string) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
tags: prev.tags?.filter(tag => tag !== tagToRemove)
|
||||
}));
|
||||
};
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
setIsSubmitting(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Validate required fields
|
||||
if (!formData.originalUrl) {
|
||||
throw new Error('Original URL is required');
|
||||
}
|
||||
|
||||
if (!formData.title) {
|
||||
throw new Error('Title is required');
|
||||
}
|
||||
|
||||
if (!formData.teamId) {
|
||||
throw new Error('Team is required');
|
||||
}
|
||||
|
||||
if (!formData.projectId) {
|
||||
throw new Error('Project is required');
|
||||
}
|
||||
|
||||
if (!formData.domain) {
|
||||
throw new Error('Domain is required');
|
||||
}
|
||||
|
||||
// Construct request data according to API requirements
|
||||
const requestData = {
|
||||
type: "shorturl",
|
||||
attributes: {
|
||||
// Can add any additional attributes, but attributes cannot be empty
|
||||
icon: ""
|
||||
},
|
||||
shortUrl: {
|
||||
url: formData.originalUrl,
|
||||
slug: formData.customSlug || undefined,
|
||||
title: formData.title,
|
||||
name: formData.title,
|
||||
description: formData.description || "",
|
||||
domain: formData.domain
|
||||
},
|
||||
teamId: formData.teamId,
|
||||
projectId: formData.projectId,
|
||||
tagIds: formData.tags && formData.tags.length > 0 ? formData.tags : undefined
|
||||
};
|
||||
|
||||
// Call API to create shorturl resource
|
||||
const response = await limqRequest('resource/shorturl', 'POST', requestData as unknown as Record<string, unknown>);
|
||||
|
||||
console.log('Created successfully:', response);
|
||||
setSuccess(true);
|
||||
|
||||
// Redirect to links list page after 2 seconds
|
||||
setTimeout(() => {
|
||||
router.push('/links');
|
||||
}, 2000);
|
||||
|
||||
} catch (err) {
|
||||
console.error('Failed to create short URL:', err);
|
||||
setError(err instanceof Error ? err.message : 'Failed to create short URL, please try again later');
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="container mx-auto px-4 py-8 max-w-3xl">
|
||||
<div className="bg-white rounded-lg shadow-md overflow-hidden">
|
||||
<div className="border-b border-gray-200 bg-blue-50 px-6 py-4">
|
||||
<h1 className="text-xl font-medium text-gray-900">Create Short URL</h1>
|
||||
<p className="mt-1 text-sm text-gray-600">
|
||||
Create a new short URL resource for tracking and analytics
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border-l-4 border-red-500 p-4 m-6">
|
||||
<div className="flex">
|
||||
<div className="flex-shrink-0">
|
||||
<svg className="h-5 w-5 text-red-500" fill="currentColor" viewBox="0 0 20 20">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
<div className="ml-3">
|
||||
<p className="text-sm text-red-700">{error}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{success && (
|
||||
<div className="bg-green-50 border-l-4 border-green-500 p-4 m-6">
|
||||
<div className="flex">
|
||||
<div className="flex-shrink-0">
|
||||
<svg className="h-5 w-5 text-green-500" fill="currentColor" viewBox="0 0 20 20">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
<div className="ml-3">
|
||||
<p className="text-sm text-green-700">
|
||||
Short URL created successfully! Redirecting...
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form onSubmit={handleSubmit} className="p-6 space-y-6">
|
||||
{/* Title */}
|
||||
<div>
|
||||
<label htmlFor="title" className="block text-sm font-medium text-gray-700">
|
||||
Title <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
id="title"
|
||||
name="title"
|
||||
value={formData.title}
|
||||
onChange={handleChange}
|
||||
placeholder="e.g., Product Launch Campaign"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Original URL */}
|
||||
<div>
|
||||
<label htmlFor="originalUrl" className="block text-sm font-medium text-gray-700">
|
||||
Original URL <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="url"
|
||||
id="originalUrl"
|
||||
name="originalUrl"
|
||||
value={formData.originalUrl}
|
||||
onChange={handleChange}
|
||||
placeholder="https://example.com/your-long-url"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Custom Short Link */}
|
||||
<div>
|
||||
<label htmlFor="customSlug" className="block text-sm font-medium text-gray-700">
|
||||
Custom Short Link <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<div className="flex mt-1 rounded-md shadow-sm">
|
||||
<span className="inline-flex items-center px-3 py-2 text-sm text-gray-500 border border-r-0 border-gray-300 rounded-l-md bg-gray-50">
|
||||
{formData.domain}/
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
id="customSlug"
|
||||
name="customSlug"
|
||||
value={formData.customSlug}
|
||||
onChange={handleChange}
|
||||
placeholder="custom-slug"
|
||||
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-none rounded-r-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
</div>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
Leave blank to generate a random short link
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Domain */}
|
||||
<div>
|
||||
<label htmlFor="domain" className="block text-sm font-medium text-gray-700">
|
||||
Domain <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
id="domain"
|
||||
name="domain"
|
||||
value={formData.domain}
|
||||
onChange={handleChange}
|
||||
placeholder="e.g., googleads.link"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Team Selection */}
|
||||
<div>
|
||||
<label htmlFor="teamId" className="block text-sm font-medium text-gray-700">
|
||||
Team <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<div className="mt-1">
|
||||
<TeamSelector
|
||||
value={formData.teamId}
|
||||
onChange={(teamId) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
teamId: teamId as string,
|
||||
// Clear selected project when team changes
|
||||
projectId: ''
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Project Selection */}
|
||||
<div>
|
||||
<label htmlFor="projectId" className="block text-sm font-medium text-gray-700">
|
||||
Project <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<div className="mt-1">
|
||||
<ProjectSelector
|
||||
teamId={formData.teamId}
|
||||
value={formData.projectId}
|
||||
onChange={(projectId) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
projectId: projectId as string
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Description */}
|
||||
<div>
|
||||
<label htmlFor="description" className="block text-sm font-medium text-gray-700">
|
||||
Description <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<textarea
|
||||
id="description"
|
||||
name="description"
|
||||
value={formData.description}
|
||||
onChange={handleChange}
|
||||
rows={3}
|
||||
placeholder="A brief description of this link"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Tags */}
|
||||
<div>
|
||||
<label htmlFor="tagInput" className="block text-sm font-medium text-gray-700">
|
||||
Tags <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<div className="flex mt-1 rounded-md shadow-sm">
|
||||
<input
|
||||
type="text"
|
||||
id="tagInput"
|
||||
value={tagInput}
|
||||
onChange={(e) => setTagInput(e.target.value)}
|
||||
onKeyDown={handleTagKeyDown}
|
||||
placeholder="Add a tag and press Enter"
|
||||
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-l-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={addTag}
|
||||
className="inline-flex items-center px-3 py-2 text-sm font-medium text-white border border-transparent rounded-r-md shadow-sm bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
Add
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{formData.tags && formData.tags.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2 mt-2">
|
||||
{formData.tags.map(tag => (
|
||||
<span key={tag} className="inline-flex items-center px-2 py-0.5 text-xs font-medium bg-blue-100 rounded-full text-blue-800">
|
||||
{tag}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeTag(tag)}
|
||||
className="flex-shrink-0 ml-1 text-blue-500 rounded-full hover:text-blue-700 focus:outline-none"
|
||||
>
|
||||
<span className="sr-only">Remove tag {tag}</span>
|
||||
<svg className="w-3 h-3" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Submit Button */}
|
||||
<div className="flex justify-end pt-4 border-t border-gray-200">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => router.back()}
|
||||
className="px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 mr-3"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isSubmitting}
|
||||
className="inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
{isSubmitting ? (
|
||||
<>
|
||||
<svg className="w-5 h-5 mr-2 -ml-1 animate-spin" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
||||
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
Processing...
|
||||
</>
|
||||
) : 'Create Short URL'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { Loader2, ExternalLink, Search } from 'lucide-react';
|
||||
import { TeamSelector } from '@/app/components/ui/TeamSelector';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useShortUrlStore, ShortUrlData } from '@/app/utils/store';
|
||||
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
|
||||
|
||||
// Define attribute type to avoid using 'any'
|
||||
interface LinkAttributes {
|
||||
@@ -102,6 +103,14 @@ const convertClickHouseToShortLink = (data: Record<string, unknown>): ShortLink
|
||||
};
|
||||
|
||||
export default function LinksPage() {
|
||||
return (
|
||||
<ClientRouteGuard>
|
||||
<LinksPageContent />
|
||||
</ClientRouteGuard>
|
||||
);
|
||||
}
|
||||
|
||||
function LinksPageContent() {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [links, setLinks] = useState<ShortLink[]>([]);
|
||||
|
||||
@@ -1,13 +1,27 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useState, useEffect, Suspense } from 'react';
|
||||
import { useRouter, useSearchParams } from 'next/navigation';
|
||||
import Link from 'next/link';
|
||||
import { useAuth } from '@/lib/auth';
|
||||
|
||||
// Separate component for message handling to isolate useSearchParams
|
||||
function MessageHandler({ setMessage }: { setMessage: (message: { type: string, content: string }) => void }) {
|
||||
const searchParams = useSearchParams();
|
||||
|
||||
useEffect(() => {
|
||||
const messageParam = searchParams.get('message');
|
||||
if (messageParam) {
|
||||
setMessage({ type: 'info', content: messageParam });
|
||||
}
|
||||
}, [searchParams, setMessage]);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export default function LoginPage() {
|
||||
const router = useRouter();
|
||||
const { signIn, signInWithGitHub, signInWithGoogle, user } = useAuth();
|
||||
const { signIn, signInWithGoogle, user } = useAuth();
|
||||
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
@@ -53,28 +67,6 @@ export default function LoginPage() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleGitHubSignIn = async () => {
|
||||
try {
|
||||
setIsLoading(true);
|
||||
setMessage({ type: '', content: '' });
|
||||
|
||||
const { error } = await signInWithGitHub();
|
||||
|
||||
if (error) {
|
||||
throw new Error(error.message);
|
||||
}
|
||||
|
||||
// 登录成功后，会通过 useEffect 重定向
|
||||
} catch (error) {
|
||||
console.error('GitHub login error:', error);
|
||||
setMessage({
|
||||
type: 'error',
|
||||
content: error instanceof Error ? error.message : 'Failed to sign in with GitHub'
|
||||
});
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleGoogleSignIn = async () => {
|
||||
try {
|
||||
setIsLoading(true);
|
||||
@@ -86,7 +78,7 @@ export default function LoginPage() {
|
||||
throw new Error(error.message);
|
||||
}
|
||||
|
||||
// 登录成功后，会通过 useEffect 重定向
|
||||
// Google OAuth will handle the redirect
|
||||
} catch (error) {
|
||||
console.error('Google login error:', error);
|
||||
setMessage({
|
||||
@@ -99,6 +91,11 @@ export default function LoginPage() {
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen bg-gray-100">
|
||||
{/* Wrap the component using useSearchParams in Suspense */}
|
||||
<Suspense fallback={null}>
|
||||
<MessageHandler setMessage={setMessage} />
|
||||
</Suspense>
|
||||
|
||||
<div className="w-full max-w-md p-8 space-y-8 bg-white rounded-lg shadow-md">
|
||||
<div className="text-center">
|
||||
<h1 className="text-2xl font-bold text-gray-900">Login</h1>
|
||||
@@ -112,16 +109,45 @@ export default function LoginPage() {
|
||||
|
||||
{/* Message display */}
|
||||
{message.content && (
|
||||
<div className={`p-4 mb-4 text-sm ${
|
||||
<div className={`p-4 mb-4 text-sm rounded-lg ${
|
||||
message.type === 'error'
|
||||
? 'text-red-700 bg-red-100 rounded-lg'
|
||||
: 'text-blue-700 bg-blue-100 rounded-lg'
|
||||
? 'text-red-700 bg-red-100 border border-red-200'
|
||||
: 'text-blue-700 bg-blue-50 border border-blue-200'
|
||||
}`}>
|
||||
{message.type === 'error' ? (
|
||||
<span className="font-medium">Error: </span>
|
||||
) : (
|
||||
<span className="font-medium">Notice: </span>
|
||||
)}
|
||||
{message.content}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form onSubmit={handleEmailSignIn} className="mt-8 space-y-6">
|
||||
{/* Google Sign In Button */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleGoogleSignIn}
|
||||
disabled={isLoading}
|
||||
className="w-full flex items-center justify-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
<svg className="h-5 w-5 mr-2" viewBox="0 0 24 24" width="24" height="24">
|
||||
<g transform="matrix(1, 0, 0, 1, 27.009001, -39.238998)">
|
||||
<path fill="#4285F4" d="M -3.264 51.509 C -3.264 50.719 -3.334 49.969 -3.454 49.239 L -14.754 49.239 L -14.754 53.749 L -8.284 53.749 C -8.574 55.229 -9.424 56.479 -10.684 57.329 L -10.684 60.329 L -6.824 60.329 C -4.564 58.239 -3.264 55.159 -3.264 51.509 Z"/>
|
||||
<path fill="#34A853" d="M -14.754 63.239 C -11.514 63.239 -8.804 62.159 -6.824 60.329 L -10.684 57.329 C -11.764 58.049 -13.134 58.489 -14.754 58.489 C -17.884 58.489 -20.534 56.379 -21.484 53.529 L -25.464 53.529 L -25.464 56.619 C -23.494 60.539 -19.444 63.239 -14.754 63.239 Z"/>
|
||||
<path fill="#FBBC05" d="M -21.484 53.529 C -21.734 52.809 -21.864 52.039 -21.864 51.239 C -21.864 50.439 -21.724 49.669 -21.484 48.949 L -21.484 45.859 L -25.464 45.859 C -26.284 47.479 -26.754 49.299 -26.754 51.239 C -26.754 53.179 -26.284 54.999 -25.464 56.619 L -21.484 53.529 Z"/>
|
||||
<path fill="#EA4335" d="M -14.754 43.989 C -12.984 43.989 -11.404 44.599 -10.154 45.789 L -6.734 42.369 C -8.804 40.429 -11.514 39.239 -14.754 39.239 C -19.444 39.239 -23.494 41.939 -25.464 45.859 L -21.484 48.949 C -20.534 46.099 -17.884 43.989 -14.754 43.989 Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
{isLoading ? 'Signing in...' : 'Sign in with Google'}
|
||||
</button>
|
||||
|
||||
<div className="mt-6 flex items-center justify-center">
|
||||
<div className="border-t border-gray-300 flex-grow mr-3"></div>
|
||||
<span className="text-sm text-gray-500">or</span>
|
||||
<div className="border-t border-gray-300 flex-grow ml-3"></div>
|
||||
</div>
|
||||
|
||||
<form onSubmit={handleEmailSignIn} className="mt-6 space-y-6">
|
||||
<div>
|
||||
<label htmlFor="email" className="block text-sm font-medium text-gray-700">
|
||||
Email address
|
||||
@@ -164,52 +190,12 @@ export default function LoginPage() {
|
||||
disabled={isLoading}
|
||||
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
{isLoading ? 'Signing in...' : 'Sign in'}
|
||||
{isLoading ? 'Signing in...' : 'Sign in with Email'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div className="mt-6">
|
||||
<div className="relative">
|
||||
<div className="absolute inset-0 flex items-center">
|
||||
<div className="w-full border-t border-gray-300"></div>
|
||||
</div>
|
||||
<div className="relative flex justify-center text-sm">
|
||||
<span className="px-2 bg-white text-gray-500">Or</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-6 grid grid-cols-2 gap-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleGitHubSignIn}
|
||||
disabled={isLoading}
|
||||
className="flex justify-center items-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
<svg className="h-5 w-5 mr-2" fill="currentColor" viewBox="0 0 24 24">
|
||||
<path fillRule="evenodd" d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z" clipRule="evenodd" />
|
||||
</svg>
|
||||
GitHub
|
||||
</button>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleGoogleSignIn}
|
||||
disabled={isLoading}
|
||||
className="flex justify-center items-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
<svg className="h-5 w-5 mr-2" viewBox="0 0 24 24">
|
||||
<path d="M12.545 12.151L12.545 12.151L12.545 12.151C12.545 9.85553 14.0905 7.98375 16.088 7.98375C17.0865 7.98375 17.938 8.43025 18.5592 9.0514L21.3404 6.27019C19.7172 4.75612 18.0026 4 16.088 4C12.5405 4 9.5 6.67528 9.5 10.2505C9.5 12.0582 10.1533 13.4581 10.8634 14.4685C12.1453 16.3618 14.4737 18.501 16.088 18.501C19.9265 18.501 22 16.0057 22 12.4071C22 11.4245 21.9318 10.9113 21.7953 10.2505H16.088V12.151H12.545Z" fill="#4285F4" />
|
||||
<path d="M5.90607 10.2197C5.40834 11.1993 5.12343 12.2959 5.12343 13.4564C5.12343 14.6646 5.41958 15.782 5.92853 16.7831L5.92786 16.7818C6.91998 18.6136 8.81431 19.8018 11.0008 19.8018C12.5581 19.8018 13.8262 19.318 14.7997 18.5825L14.7976 18.5845C15.6806 17.9139 16.401 16.9218 16.6662 15.7257L16.6657 15.7276C16.7331 15.3933 16.7688 15.0493 16.7688 14.6895H11.0008C10.3375 14.6895 9.80078 14.1523 9.80078 13.4882V10.2197H5.90607Z" fill="#34A853" />
|
||||
<path d="M5.12207 6.25024C4 7.86024 3.33789 9.81535 3.33789 11.9339C3.33789 12.9995 3.55215 14.0269 3.94853 14.9805L5.90673 10.2197H9.80143V6.25024H5.12207Z" fill="#FBBC05" />
|
||||
<path d="M11.001 3.57764C12.4571 3.57764 13.778 4.11181 14.8023 5.06959L14.8028 5.0692L17.2711 2.60092L17.271 2.60082C15.5041 0.97625 13.3649 0 11.001 0C8.81453 0 6.91994 1.18824 5.92853 3.02125L9.80224 6.25031V6.25031H11.001V3.57764Z" fill="#EA4335" />
|
||||
</svg>
|
||||
Google
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="text-sm text-gray-600">
|
||||
<p className="text-sm mt-6 text-gray-600">
|
||||
Don't have an account?{' '}
|
||||
<Link href="/register" className="font-medium text-blue-600 hover:text-blue-500">
|
||||
Register
|
||||
|
||||
@@ -12,44 +12,44 @@ export default function RegisterPage() {
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const { signUp, signInWithGoogle } = useAuth();
|
||||
|
||||
// 处理注册表单提交
|
||||
// Handle registration form submission
|
||||
const handleSubmit = async (e: FormEvent) => {
|
||||
e.preventDefault();
|
||||
setError(null);
|
||||
|
||||
// 验证密码
|
||||
// Validate passwords
|
||||
if (password !== confirmPassword) {
|
||||
setError('两次输入的密码不一致');
|
||||
setError('Passwords do not match');
|
||||
return;
|
||||
}
|
||||
|
||||
// 密码强度验证
|
||||
// Password strength validation
|
||||
if (password.length < 6) {
|
||||
setError('密码长度至少为6个字符');
|
||||
setError('Password must be at least 6 characters');
|
||||
return;
|
||||
}
|
||||
|
||||
setIsLoading(true);
|
||||
try {
|
||||
await signUp(email, password);
|
||||
// 注册成功后会跳转到登录页面，提示用户验证邮箱
|
||||
// After successful registration, redirect to login page with email verification prompt
|
||||
} catch (error) {
|
||||
console.error('Registration error:', error);
|
||||
setError('注册失败，请稍后再试或使用其他邮箱');
|
||||
setError('Registration failed. Please try again later or use a different email');
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
// 处理Google注册/登录
|
||||
// Handle Google registration/login
|
||||
const handleGoogleSignIn = async () => {
|
||||
setError(null);
|
||||
try {
|
||||
await signInWithGoogle();
|
||||
// 登录流程会重定向到Google，然后回到应用
|
||||
// Login flow will redirect to Google and then back to the application
|
||||
} catch (error) {
|
||||
console.error('Google sign in error:', error);
|
||||
setError('Google登录失败，请稍后再试');
|
||||
setError('Google login failed. Please try again later');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -57,13 +57,13 @@ export default function RegisterPage() {
|
||||
<div className="flex items-center justify-center min-h-screen bg-gray-100 dark:bg-gray-900">
|
||||
<div className="w-full max-w-md p-8 space-y-8 bg-white dark:bg-gray-800 rounded-lg shadow-md">
|
||||
<div className="text-center">
|
||||
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">注册</h1>
|
||||
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">Register</h1>
|
||||
<p className="mt-2 text-sm text-gray-600 dark:text-gray-400">
|
||||
创建您的帐户以访问分析仪表板
|
||||
Create your account to access the analytics dashboard
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* 错误提示 */}
|
||||
{/* Error message */}
|
||||
{error && (
|
||||
<div className="p-4 mb-4 text-sm text-red-700 bg-red-100 dark:bg-red-900 dark:text-red-200 rounded-lg">
|
||||
{error}
|
||||
@@ -74,7 +74,7 @@ export default function RegisterPage() {
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<label htmlFor="email" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
邮箱地址
|
||||
Email Address
|
||||
</label>
|
||||
<input
|
||||
id="email"
|
||||
@@ -90,7 +90,7 @@ export default function RegisterPage() {
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="password" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
密码
|
||||
Password
|
||||
</label>
|
||||
<input
|
||||
id="password"
|
||||
@@ -106,7 +106,7 @@ export default function RegisterPage() {
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="confirmPassword" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
确认密码
|
||||
Confirm Password
|
||||
</label>
|
||||
<input
|
||||
id="confirmPassword"
|
||||
@@ -128,7 +128,7 @@ export default function RegisterPage() {
|
||||
disabled={isLoading}
|
||||
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
{isLoading ? '注册中...' : '注册'}
|
||||
{isLoading ? 'Registering...' : 'Register'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@@ -137,7 +137,7 @@ export default function RegisterPage() {
|
||||
<div className="w-full border-t border-gray-300 dark:border-gray-600"></div>
|
||||
</div>
|
||||
<div className="relative flex justify-center text-sm">
|
||||
<span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">或</span>
|
||||
<span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">or</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -173,19 +173,19 @@ export default function RegisterPage() {
|
||||
/>
|
||||
</g>
|
||||
</svg>
|
||||
使用Google账号注册
|
||||
Sign up with Google
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div className="mt-6 text-center">
|
||||
<p className="text-sm text-gray-600 dark:text-gray-400">
|
||||
已有账号？{' '}
|
||||
Already have an account?{' '}
|
||||
<Link
|
||||
href="/login"
|
||||
className="font-medium text-blue-600 hover:text-blue-500 dark:text-blue-400 dark:hover:text-blue-300"
|
||||
>
|
||||
登录
|
||||
Log in
|
||||
</Link>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
50
lib/api.ts
Normal file
50
lib/api.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import supabase from './supabase';
|
||||
|
||||
// Define response type for API
|
||||
export interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
// Common function for authenticated API requests to LIMQ
|
||||
export async function limqRequest<T = unknown>(
|
||||
endpoint: string,
|
||||
method: 'GET' | 'POST' | 'PUT' | 'DELETE' = 'GET',
|
||||
data?: Record<string, unknown>
|
||||
): Promise<ApiResponse<T>> {
|
||||
// Get current session
|
||||
const { data: { session } } = await supabase.auth.getSession();
|
||||
|
||||
if (!session) {
|
||||
throw new Error('No active session. User must be authenticated.');
|
||||
}
|
||||
|
||||
const baseUrl = process.env.NEXT_PUBLIC_LIMQ_API;
|
||||
const url = `${baseUrl}${endpoint.startsWith('/') ? endpoint : '/' + endpoint}`;
|
||||
|
||||
const options: RequestInit = {
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${session.access_token}`
|
||||
},
|
||||
mode: 'cors'
|
||||
};
|
||||
|
||||
if (data && (method === 'POST' || method === 'PUT')) {
|
||||
options.body = JSON.stringify(data);
|
||||
}
|
||||
|
||||
const response = await fetch(url, options);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => null);
|
||||
throw new Error(
|
||||
errorData?.error || `Request failed with status ${response.status}`
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
63
lib/auth.tsx
63
lib/auth.tsx
@@ -4,6 +4,7 @@ import React, { createContext, useContext, useEffect, useState } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { Session, User } from '@supabase/supabase-js';
|
||||
import supabase from './supabase';
|
||||
import { limqRequest } from './api';
|
||||
|
||||
// 定义用户类型
|
||||
export type AuthUser = User | null;
|
||||
@@ -13,21 +14,16 @@ export type AuthContextType = {
|
||||
user: AuthUser;
|
||||
session: Session | null;
|
||||
isLoading: boolean;
|
||||
signIn: (email: string, password: string) => Promise<{ error?: any }>;
|
||||
signInWithGoogle: () => Promise<{ error?: any }>;
|
||||
signInWithGitHub: () => Promise<{ error?: any }>;
|
||||
signIn: (email: string, password: string) => Promise<{ error?: unknown }>;
|
||||
signInWithGoogle: () => Promise<{ error?: unknown }>;
|
||||
signInWithGitHub: () => Promise<{ error?: unknown }>;
|
||||
signUp: (email: string, password: string) => Promise<void>;
|
||||
signOut: () => Promise<void>;
|
||||
autoRegisterTestUser: () => Promise<void>; // 添加自动注册测试用户函数
|
||||
};
|
||||
|
||||
// 创建验证上下文
|
||||
const AuthContext = createContext<AuthContextType | undefined>(undefined);
|
||||
|
||||
// 测试账户常量 - 使用已验证的账户
|
||||
const TEST_EMAIL = 'vitalitymailg@gmail.com';
|
||||
const TEST_PASSWORD = 'password123';
|
||||
|
||||
// 验证提供者组件
|
||||
export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
|
||||
const [user, setUser] = useState<AuthUser>(null);
|
||||
@@ -90,7 +86,8 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
|
||||
setSession(data.session);
|
||||
setUser(data.user);
|
||||
router.push('/dashboard');
|
||||
|
||||
router.push('/analytics');
|
||||
return {};
|
||||
} catch (error) {
|
||||
console.error('登录过程出错:', error);
|
||||
@@ -104,11 +101,14 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
const signInWithGoogle = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL，如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase登录Google
|
||||
const { error } = await supabase.auth.signInWithOAuth({
|
||||
provider: 'google',
|
||||
options: {
|
||||
redirectTo: `${window.location.origin}/auth/callback`,
|
||||
redirectTo: `${siteUrl}/auth/callback`,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -130,11 +130,14 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
const signInWithGitHub = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL，如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase登录GitHub
|
||||
const { error } = await supabase.auth.signInWithOAuth({
|
||||
provider: 'github',
|
||||
options: {
|
||||
redirectTo: `${window.location.origin}/auth/callback`,
|
||||
redirectTo: `${siteUrl}/auth/callback`,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -156,10 +159,16 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
const signUp = async (email: string, password: string) => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL，如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase注册
|
||||
const { error } = await supabase.auth.signUp({
|
||||
email,
|
||||
password,
|
||||
options: {
|
||||
emailRedirectTo: `${siteUrl}/auth/callback`,
|
||||
}
|
||||
});
|
||||
|
||||
if (error) {
|
||||
@@ -168,7 +177,7 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
}
|
||||
|
||||
// 注册成功后跳转到登录页面并显示确认消息
|
||||
router.push('/login?message=注册成功，请查看邮箱确认账户');
|
||||
router.push('/login?message=Registration successful! Please check your email to verify your account before logging in.');
|
||||
} catch (error) {
|
||||
console.error('注册过程出错:', error);
|
||||
throw error;
|
||||
@@ -198,35 +207,6 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
}
|
||||
};
|
||||
|
||||
// 自动注册测试用户函数
|
||||
const autoRegisterTestUser = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
console.log('正在使用测试账户登录:', TEST_EMAIL);
|
||||
|
||||
// 使用测试账户直接登录
|
||||
const { data, error } = await supabase.auth.signInWithPassword({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('测试账户登录失败:', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
console.log('测试账户登录成功!');
|
||||
setSession(data.session);
|
||||
setUser(data.user);
|
||||
router.push('/dashboard');
|
||||
} catch (error) {
|
||||
console.error('测试账户登录出错:', error);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const contextValue: AuthContextType = {
|
||||
user,
|
||||
session,
|
||||
@@ -236,7 +216,6 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
signInWithGitHub,
|
||||
signUp,
|
||||
signOut,
|
||||
autoRegisterTestUser,
|
||||
};
|
||||
|
||||
return (
|
||||
|
||||
22
middleware.ts
Normal file
22
middleware.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { createMiddlewareClient } from '@supabase/auth-helpers-nextjs';
|
||||
import { NextResponse } from 'next/server';
|
||||
import type { NextRequest } from 'next/server';
|
||||
|
||||
export async function middleware(req: NextRequest) {
|
||||
const res = NextResponse.next();
|
||||
|
||||
// Create a Supabase client configured to use cookies
|
||||
const supabase = createMiddlewareClient({ req, res });
|
||||
|
||||
// Refresh session if expired - required for Server Components
|
||||
await supabase.auth.getSession();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// Specify the paths where this middleware should run
|
||||
export const config = {
|
||||
matcher: [
|
||||
'/((?!_next/static|_next/image|favicon.ico).*)',
|
||||
],
|
||||
};
|
||||
@@ -3,7 +3,7 @@
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"dev": "next dev -p 3007",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
|
||||
2
windmill/scripts/.gitignore
vendored
Normal file
2
windmill/scripts/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
/node_modules
|
||||
/package-lock.json
|
||||
19
windmill/scripts/package.json
Normal file
19
windmill/scripts/package.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "scripts",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"date-fns": "^4.1.0",
|
||||
"dotenv": "^16.5.0",
|
||||
"fs-extra": "^11.3.0",
|
||||
"mongodb": "^6.16.0",
|
||||
"node-fetch": "^2.7.0"
|
||||
}
|
||||
}
|
||||
714
windmill/scripts/sync_mongo_to_clickhouse.js
Normal file
714
windmill/scripts/sync_mongo_to_clickhouse.js
Normal file
@@ -0,0 +1,714 @@
|
||||
// 从MongoDB的trace表同步数据到ClickHouse的events表
|
||||
//
|
||||
// 支持以下同步模式：
|
||||
// 1. 增量同步：基于上次同步状态，只同步新数据（默认模式）
|
||||
// 2. 自定义时间范围同步：通过指定开始时间和结束时间，同步特定时间范围内的数据
|
||||
// - 可以通过时间戳参数(startTime/endTime)指定范围
|
||||
// - 也可以通过日期字符串参数(startDate/endDate)指定范围，支持ISO格式或yyyy-MM-dd格式
|
||||
|
||||
const { MongoClient, ObjectId } = require('mongodb');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
// 同步状态键名和保存路径
|
||||
const SYNC_STATE_FILE = path.join(__dirname, 'mongo_sync_state.json');
|
||||
|
||||
// 直接使用配置值
|
||||
const mongoConfig = {
|
||||
url: "mongodb://10.0.1.41:27017",
|
||||
db: "main" // 注意：请替换为您的实际数据库名称
|
||||
};
|
||||
|
||||
const clickhouseConfig = {
|
||||
clickhouse_host: "10.0.1.60",
|
||||
clickhouse_port: "8123",
|
||||
clickhouse_user: "admin",
|
||||
clickhouse_password: "your_secure_password",
|
||||
clickhouse_database: "shorturl_analytics",
|
||||
clickhouse_url: "http://10.0.1.60:8123"
|
||||
};
|
||||
|
||||
// 封装本地读取变量函数
|
||||
async function getVariable(key) {
|
||||
try {
|
||||
if (key === 'f/shorturl_analytics/mongodb') {
|
||||
return mongoConfig;
|
||||
} else if (key === 'f/shorturl_analytics/clickhouse') {
|
||||
return clickhouseConfig;
|
||||
} else if (key === 'f/shorturl_analytics/mongo_sync_state') {
|
||||
if (fs.existsSync(SYNC_STATE_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(SYNC_STATE_FILE, 'utf8'));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error(`获取变量失败: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 封装本地保存变量函数
|
||||
async function setVariable(key, value) {
|
||||
try {
|
||||
if (key === 'f/shorturl_analytics/mongo_sync_state') {
|
||||
fs.writeFileSync(SYNC_STATE_FILE, JSON.stringify(value, null, 2));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`保存变量失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 日期字符串转时间戳工具函数（接受ISO字符串或yyyy-MM-dd格式）
|
||||
function dateToTimestamp(dateStr) {
|
||||
try {
|
||||
// 尝试直接解析完整的ISO日期字符串
|
||||
const date = new Date(dateStr);
|
||||
|
||||
// 检查是否为有效日期
|
||||
if (isNaN(date.getTime())) {
|
||||
// 尝试解析yyyy-MM-dd格式，默认设置为当天的00:00:00
|
||||
const parts = dateStr.split('-');
|
||||
if (parts.length === 3) {
|
||||
const year = parseInt(parts[0], 10);
|
||||
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
|
||||
const day = parseInt(parts[2], 10);
|
||||
|
||||
const dateObj = new Date(year, month, day, 0, 0, 0);
|
||||
return dateObj.getTime();
|
||||
}
|
||||
throw new Error(`无法解析日期字符串: ${dateStr}`);
|
||||
}
|
||||
|
||||
return date.getTime();
|
||||
} catch (err) {
|
||||
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从URL中提取UTM参数的函数，增强版
|
||||
function extractUtmParams(url, debug = false) {
|
||||
const defaultUtmParams = {
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
utm_term: "",
|
||||
utm_content: ""
|
||||
};
|
||||
|
||||
if (!url) return defaultUtmParams;
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 原始URL: ${url}`);
|
||||
}
|
||||
|
||||
// 准备一个解析后的参数对象
|
||||
const params = { ...defaultUtmParams };
|
||||
|
||||
// 尝试多种方法提取UTM参数
|
||||
|
||||
// 方法1: 使用URL对象解析
|
||||
try {
|
||||
// 先处理URL，确保是完整的URL格式
|
||||
let normalizedUrl = url;
|
||||
if (!url.match(/^https?:\/\//i)) {
|
||||
normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
|
||||
}
|
||||
|
||||
const urlObj = new URL(normalizedUrl);
|
||||
|
||||
// 读取URL参数
|
||||
if (urlObj.searchParams.has('utm_source'))
|
||||
params.utm_source = urlObj.searchParams.get('utm_source') || "";
|
||||
if (urlObj.searchParams.has('utm_medium'))
|
||||
params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
|
||||
if (urlObj.searchParams.has('utm_campaign'))
|
||||
params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
|
||||
if (urlObj.searchParams.has('utm_term'))
|
||||
params.utm_term = urlObj.searchParams.get('utm_term') || "";
|
||||
if (urlObj.searchParams.has('utm_content'))
|
||||
params.utm_content = urlObj.searchParams.get('utm_content') || "";
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
// 如果至少找到一个UTM参数，则返回
|
||||
if (params.utm_source || params.utm_medium || params.utm_campaign ||
|
||||
params.utm_term || params.utm_content) {
|
||||
return params;
|
||||
}
|
||||
} catch (err) {
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析失败，尝试正则表达式`);
|
||||
}
|
||||
}
|
||||
|
||||
// 方法2: 使用正则表达式提取参数
|
||||
// 使用正则表达式（最安全的方法，适用于任何格式）
|
||||
const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
|
||||
if (sourceMatch && sourceMatch[1]) {
|
||||
try {
|
||||
params.utm_source = decodeURIComponent(sourceMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_source = sourceMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
|
||||
if (mediumMatch && mediumMatch[1]) {
|
||||
try {
|
||||
params.utm_medium = decodeURIComponent(mediumMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_medium = mediumMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) {
|
||||
try {
|
||||
params.utm_campaign = decodeURIComponent(campaignMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_campaign = campaignMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
|
||||
if (termMatch && termMatch[1]) {
|
||||
try {
|
||||
params.utm_term = decodeURIComponent(termMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_term = termMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
|
||||
if (contentMatch && contentMatch[1]) {
|
||||
try {
|
||||
params.utm_content = decodeURIComponent(contentMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_content = contentMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
// 解析命令行参数
|
||||
function parseCommandLineArgs() {
|
||||
const args = {};
|
||||
process.argv.slice(2).forEach(arg => {
|
||||
if (arg.startsWith('--')) {
|
||||
const [key, value] = arg.substring(2).split('=');
|
||||
args[key] = value || true;
|
||||
}
|
||||
});
|
||||
return args;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseCommandLineArgs();
|
||||
|
||||
// 参数设置
|
||||
const batch_size = parseInt(args['batch-size'] || '1000');
|
||||
const max_records = parseInt(args['max-records'] || '9999999');
|
||||
const timeout_minutes = parseInt(args['timeout'] || '60');
|
||||
const skip_clickhouse_check = args['skip-clickhouse-check'] === 'true';
|
||||
const force_insert = args['force-insert'] !== 'false';
|
||||
const database_override = args['database'] || 'shorturl_analytics';
|
||||
const reset_sync_state = args['reset-sync-state'] === 'true';
|
||||
const debug_utm = args['debug-utm'] === 'true';
|
||||
const start_time = args['start-time'] ? parseInt(args['start-time']) : undefined;
|
||||
const end_time = args['end-time'] ? parseInt(args['end-time']) : undefined;
|
||||
const use_custom_time_range = args['use-custom-time-range'] === 'true';
|
||||
const start_date = args['start-date'];
|
||||
const end_date = args['end-date'];
|
||||
|
||||
const logWithTimestamp = (message) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
|
||||
let customStartTime = start_time;
|
||||
let customEndTime = end_time;
|
||||
let useCustomTimeRange = use_custom_time_range;
|
||||
|
||||
// 处理日期字符串参数，转换为时间戳
|
||||
if (start_date) {
|
||||
try {
|
||||
customStartTime = dateToTimestamp(start_date);
|
||||
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${customStartTime}`);
|
||||
useCustomTimeRange = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (end_date) {
|
||||
try {
|
||||
customEndTime = dateToTimestamp(end_date);
|
||||
// 如果是日期格式，设置为当天结束时间 (23:59:59.999)
|
||||
if (end_date.split('-').length === 3 && end_date.length <= 10) {
|
||||
customEndTime += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${customEndTime}`);
|
||||
} else {
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${customEndTime}`);
|
||||
}
|
||||
useCustomTimeRange = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
|
||||
}
|
||||
if (force_insert) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
|
||||
}
|
||||
if (reset_sync_state) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用重置同步状态，将从头开始同步数据");
|
||||
}
|
||||
if (debug_utm) {
|
||||
logWithTimestamp("已启用UTM参数调试日志");
|
||||
}
|
||||
if (useCustomTimeRange) {
|
||||
if (customStartTime) {
|
||||
logWithTimestamp(`已启用自定义时间范围：开始时间 ${new Date(customStartTime).toISOString()}`);
|
||||
}
|
||||
if (customEndTime) {
|
||||
logWithTimestamp(`已启用自定义时间范围：结束时间 ${new Date(customEndTime).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
// 检查是否超时
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`运行时间超过${timeout_minutes}分钟，暂停执行`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// 获取上次同步状态
|
||||
let lastSyncState = null;
|
||||
if (!reset_sync_state) {
|
||||
try {
|
||||
const rawSyncState = await getVariable("f/shorturl_analytics/mongo_sync_state");
|
||||
if (rawSyncState) {
|
||||
lastSyncState = rawSyncState;
|
||||
}
|
||||
} catch (error) {
|
||||
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
|
||||
}
|
||||
}
|
||||
|
||||
if (lastSyncState) {
|
||||
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
|
||||
if (lastSyncState.last_sync_id) {
|
||||
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("没有找到上次同步状态，将从头开始同步");
|
||||
}
|
||||
|
||||
// 连接MongoDB
|
||||
const client = new MongoClient(mongoConfig.url);
|
||||
try {
|
||||
await client.connect();
|
||||
console.log("MongoDB连接成功");
|
||||
|
||||
const db = client.db(mongoConfig.db);
|
||||
const traceCollection = db.collection("trace");
|
||||
const shortCollection = db.collection("short");
|
||||
|
||||
// 构建查询条件
|
||||
const query = {
|
||||
type: 1 // 只同步type为1的记录
|
||||
};
|
||||
|
||||
// 根据时间范围参数构建查询条件
|
||||
if (useCustomTimeRange) {
|
||||
// 使用自定义时间范围
|
||||
const timeQuery = {};
|
||||
|
||||
if (customStartTime) {
|
||||
timeQuery.$gte = customStartTime;
|
||||
logWithTimestamp(`将只同步createTime >= ${customStartTime} (${new Date(customStartTime).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
if (customEndTime) {
|
||||
timeQuery.$lte = customEndTime;
|
||||
logWithTimestamp(`将只同步createTime <= ${customEndTime} (${new Date(customEndTime).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 只有当至少指定了一个时间限制时才添加时间查询条件
|
||||
if (Object.keys(timeQuery).length > 0) {
|
||||
query.createTime = timeQuery;
|
||||
}
|
||||
}
|
||||
// 如果不使用自定义时间范围，且有上次同步状态，则只获取更新的记录
|
||||
else if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 使用上次同步时间作为过滤条件
|
||||
query.createTime = { $gt: lastSyncState.last_sync_time };
|
||||
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 计算总记录数
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`找到 ${totalRecords} 条新记录需要同步`);
|
||||
|
||||
// 限制此次处理的记录数量
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`本次将处理 ${recordsToProcess} 条记录`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("没有新记录需要同步，任务完成");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "没有新记录需要同步"
|
||||
};
|
||||
}
|
||||
|
||||
// 检查ClickHouse连接状态
|
||||
const checkClickHouseConnection = async () => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，不测试连接");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("测试ClickHouse连接...");
|
||||
const clickhouseUrl = clickhouseConfig.clickhouse_url;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`,
|
||||
},
|
||||
body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse连接测试成功");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ ClickHouse连接测试失败，请启用skip_clickhouse_check=true参数来跳过连接检查");
|
||||
throw new Error("ClickHouse连接失败，无法继续同步");
|
||||
}
|
||||
|
||||
// 处理记录的函数
|
||||
const processRecords = async (records) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
|
||||
|
||||
// 强制使用所有记录，不检查重复
|
||||
const newRecords = records;
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
|
||||
|
||||
// 获取链接信息
|
||||
const slugIds = newRecords.map(record => new ObjectId(record.slugId));
|
||||
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
|
||||
const shortLinks = await shortCollection.find({
|
||||
_id: { $in: slugIds }
|
||||
}).toArray();
|
||||
|
||||
// 创建映射用于快速查找
|
||||
const shortLinksMap = new Map(shortLinks.map((link) => [link._id.toString(), link]));
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息，${newRecords.length - shortLinks.length} 条数据将使用占位符`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const eventTime = new Date(record.createTime);
|
||||
|
||||
// 获取对应的短链接信息
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString());
|
||||
|
||||
// 提取URL中的UTM参数
|
||||
if (debug_utm && record.url) {
|
||||
logWithTimestamp(`======== UTM参数调试 ========`);
|
||||
logWithTimestamp(`记录ID: ${record._id.toString()}`);
|
||||
logWithTimestamp(`原始URL: ${record.url}`);
|
||||
}
|
||||
|
||||
const utmParams = extractUtmParams(record.url || "", debug_utm);
|
||||
|
||||
if (debug_utm) {
|
||||
logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
|
||||
logWithTimestamp(`===========================`);
|
||||
}
|
||||
|
||||
// 保存提取的UTM参数和URL到event_attributes
|
||||
const eventAttributes = {
|
||||
mongo_id: record._id.toString(),
|
||||
url: record.url || "",
|
||||
...(record.url ? { raw_url: record.url } : {})
|
||||
};
|
||||
|
||||
// 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
|
||||
return {
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
event_type: record.type === 1 ? "visit" : "custom",
|
||||
event_attributes: JSON.stringify(eventAttributes),
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
|
||||
link_label: record.label || "",
|
||||
link_title: shortLink?.title || "unknown_title", // 使用占位符
|
||||
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
|
||||
link_created_at: shortLink?.createTime
|
||||
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
link_expires_at: shortLink?.expiresAt
|
||||
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: null,
|
||||
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
|
||||
user_id: shortLink?.user || "unknown_user", // 使用占位符
|
||||
user_name: "unknown_user", // 使用占位符
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
team_id: shortLink?.teamId || "unknown_team", // 使用占位符
|
||||
team_name: "unknown_team", // 使用占位符
|
||||
team_attributes: "{}",
|
||||
project_id: shortLink?.projectId || "unknown_project", // 使用占位符
|
||||
project_name: "unknown_project", // 使用占位符
|
||||
project_attributes: "{}",
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: record._id.toString() + "-" + record.createTime,
|
||||
ip_address: record.ip || "0.0.0.0", // 使用占位符
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "unknown", // 使用占位符
|
||||
os: record.platformOS || "unknown", // 使用占位符
|
||||
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
|
||||
referrer: record.url || "",
|
||||
utm_source: utmParams.utm_source || "",
|
||||
utm_medium: utmParams.utm_medium || "",
|
||||
utm_campaign: utmParams.utm_campaign || "",
|
||||
utm_term: utmParams.utm_term || "",
|
||||
utm_content: utmParams.utm_content || "",
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
conversion_type: "visit",
|
||||
conversion_value: 0,
|
||||
req_full_path: record.url || ""
|
||||
};
|
||||
});
|
||||
|
||||
// 生成ClickHouse插入SQL
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
|
||||
(event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title,
|
||||
link_original_url, link_attributes, link_created_at, link_expires_at, link_tags,
|
||||
user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes,
|
||||
project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes,
|
||||
visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent,
|
||||
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
|
||||
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
|
||||
VALUES ${clickhouseData.map(record => {
|
||||
// 确保所有字符串值都是字符串类型，并安全处理替换
|
||||
const safeReplace = (val) => {
|
||||
// 确保值是字符串，如果是null或undefined则使用空字符串
|
||||
const str = val === null || val === undefined ? "" : String(val);
|
||||
// 安全替换单引号
|
||||
return str.replace(/'/g, "''");
|
||||
};
|
||||
|
||||
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
|
||||
'${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}',
|
||||
'${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}',
|
||||
${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}',
|
||||
'${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}',
|
||||
'${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}',
|
||||
'${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}',
|
||||
'${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}',
|
||||
'${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}',
|
||||
'${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}',
|
||||
'${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}',
|
||||
'${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}',
|
||||
'${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}',
|
||||
'${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan},
|
||||
'${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
|
||||
}).join(", ")}
|
||||
`;
|
||||
|
||||
if (insertSQL.length === 0) {
|
||||
console.log("没有新记录需要插入");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 发送请求到ClickHouse
|
||||
const clickhouseUrl = clickhouseConfig.clickhouse_url;
|
||||
try {
|
||||
logWithTimestamp("发送插入请求到ClickHouse...");
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`
|
||||
},
|
||||
body: insertSQL,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`向ClickHouse插入数据失败: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// 批量处理记录
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
let lastSyncTime = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录，因超时暂停执行`);
|
||||
break;
|
||||
}
|
||||
|
||||
// 每批次都输出进度
|
||||
logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
|
||||
|
||||
// 查询MongoDB数据
|
||||
const records = await traceCollection.find(query)
|
||||
.sort({ createTime: 1 })
|
||||
.skip(page * batch_size)
|
||||
.limit(batch_size)
|
||||
.toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("没有找到更多数据，同步结束");
|
||||
break;
|
||||
}
|
||||
|
||||
// 找到数据，开始处理
|
||||
logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
|
||||
// 输出当前批次的部分数据信息
|
||||
if (records.length > 0) {
|
||||
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
|
||||
if (records.length > 1) {
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
|
||||
}
|
||||
|
||||
// 如果开启了调试，输出一些URL样本
|
||||
if (debug_utm) {
|
||||
const sampleSize = Math.min(5, records.length);
|
||||
logWithTimestamp(`URL样本 (前${sampleSize}条):`);
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
if (records[i].url) {
|
||||
logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
// 更新最后处理的记录时间和ID
|
||||
if (records.length > 0) {
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
|
||||
}
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
if (processedRecords > 0 && lastSyncTime > 0) {
|
||||
// 只在非自定义时间范围模式下更新同步状态
|
||||
if (!useCustomTimeRange) {
|
||||
// 创建新的同步状态，简化对象结构
|
||||
const newSyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords
|
||||
};
|
||||
|
||||
try {
|
||||
// 保存同步状态
|
||||
await setVariable("f/shorturl_analytics/mongo_sync_state", newSyncState);
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
logWithTimestamp(`更新同步状态失败: ${err.message}，将继续执行`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("使用自定义时间范围模式，不更新全局同步状态");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
|
||||
message: useCustomTimeRange ? "自定义时间范围数据同步完成" : "数据同步完成",
|
||||
custom_time_range_used: useCustomTimeRange
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("同步过程中发生错误:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err.message,
|
||||
stack: err.stack
|
||||
};
|
||||
} finally {
|
||||
// 关闭MongoDB连接
|
||||
await client.close();
|
||||
console.log("MongoDB连接已关闭");
|
||||
}
|
||||
}
|
||||
|
||||
// 执行主函数
|
||||
main().then(result => {
|
||||
console.log("任务执行结果:", result);
|
||||
process.exit(result.success ? 0 : 1);
|
||||
}).catch(err => {
|
||||
console.error("执行出错:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,4 +1,12 @@
|
||||
// 从MongoDB的trace表同步数据到ClickHouse的events表
|
||||
//
|
||||
// 支持以下同步模式：
|
||||
// 1. 增量同步：基于上次同步状态，只同步新数据（默认模式）
|
||||
// 2. 自定义时间范围同步：通过指定开始时间和结束时间，同步特定时间范围内的数据
|
||||
// - 可以通过时间戳参数(start_time/end_time)指定范围
|
||||
// - 也可以通过日期字符串参数(start_date/end_date)指定范围，支持ISO格式或yyyy-MM-dd格式
|
||||
//
|
||||
// 使用自定义时间范围时，将不会更新同步状态，避免干扰增量同步进度
|
||||
import { getVariable, setVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
@@ -33,23 +41,194 @@ interface TraceRecord {
|
||||
createTime: number;
|
||||
}
|
||||
|
||||
// 添加 ShortRecord 接口定义
|
||||
interface ShortRecord {
|
||||
_id: ObjectId;
|
||||
slug: string; // 短链接的slug部分
|
||||
origin: string; // 原始URL
|
||||
domain?: string; // 域名
|
||||
createTime: number; // 创建时间戳
|
||||
user?: string; // 创建用户
|
||||
title?: string; // 标题
|
||||
description?: string; // 描述
|
||||
tags?: string[]; // 标签
|
||||
active?: boolean; // 是否活跃
|
||||
expiresAt?: number; // 过期时间戳
|
||||
teamId?: string; // 团队ID
|
||||
projectId?: string; // 项目ID
|
||||
}
|
||||
|
||||
interface SyncState {
|
||||
last_sync_time: number;
|
||||
records_synced: number;
|
||||
last_sync_id?: string;
|
||||
}
|
||||
|
||||
// 定义UTM参数接口
|
||||
interface UtmParams {
|
||||
utm_source: string;
|
||||
utm_medium: string;
|
||||
utm_campaign: string;
|
||||
utm_term: string;
|
||||
utm_content: string;
|
||||
}
|
||||
|
||||
// 同步状态键名
|
||||
const SYNC_STATE_KEY = "f/shorturl_analytics/mongo_sync_state";
|
||||
|
||||
// 日期字符串转时间戳工具函数（接受ISO字符串或yyyy-MM-dd格式）
|
||||
function dateToTimestamp(dateStr: string): number {
|
||||
try {
|
||||
// 尝试直接解析完整的ISO日期字符串
|
||||
const date = new Date(dateStr);
|
||||
|
||||
// 检查是否为有效日期
|
||||
if (isNaN(date.getTime())) {
|
||||
// 尝试解析yyyy-MM-dd格式，默认设置为当天的00:00:00
|
||||
const parts = dateStr.split('-');
|
||||
if (parts.length === 3) {
|
||||
const year = parseInt(parts[0], 10);
|
||||
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
|
||||
const day = parseInt(parts[2], 10);
|
||||
|
||||
const dateObj = new Date(year, month, day, 0, 0, 0);
|
||||
return dateObj.getTime();
|
||||
}
|
||||
throw new Error(`无法解析日期字符串: ${dateStr}`);
|
||||
}
|
||||
|
||||
return date.getTime();
|
||||
} catch (err) {
|
||||
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从URL中提取UTM参数的函数，增强版
|
||||
function extractUtmParams(url: string, debug = false): UtmParams {
|
||||
const defaultUtmParams: UtmParams = {
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
utm_term: "",
|
||||
utm_content: ""
|
||||
};
|
||||
|
||||
if (!url) return defaultUtmParams;
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 原始URL: ${url}`);
|
||||
}
|
||||
|
||||
// 准备一个解析后的参数对象
|
||||
const params: UtmParams = { ...defaultUtmParams };
|
||||
|
||||
// 尝试多种方法提取UTM参数
|
||||
|
||||
// 方法1: 使用URL对象解析
|
||||
try {
|
||||
// 先处理URL，确保是完整的URL格式
|
||||
let normalizedUrl = url;
|
||||
if (!url.match(/^https?:\/\//i)) {
|
||||
normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
|
||||
}
|
||||
|
||||
const urlObj = new URL(normalizedUrl);
|
||||
|
||||
// 读取URL参数
|
||||
if (urlObj.searchParams.has('utm_source'))
|
||||
params.utm_source = urlObj.searchParams.get('utm_source') || "";
|
||||
if (urlObj.searchParams.has('utm_medium'))
|
||||
params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
|
||||
if (urlObj.searchParams.has('utm_campaign'))
|
||||
params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
|
||||
if (urlObj.searchParams.has('utm_term'))
|
||||
params.utm_term = urlObj.searchParams.get('utm_term') || "";
|
||||
if (urlObj.searchParams.has('utm_content'))
|
||||
params.utm_content = urlObj.searchParams.get('utm_content') || "";
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
// 如果至少找到一个UTM参数，则返回
|
||||
if (params.utm_source || params.utm_medium || params.utm_campaign ||
|
||||
params.utm_term || params.utm_content) {
|
||||
return params;
|
||||
}
|
||||
} catch (_err) {
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析失败，尝试正则表达式`);
|
||||
}
|
||||
}
|
||||
|
||||
// 方法2: 使用正则表达式提取参数
|
||||
// 使用正则表达式（最安全的方法，适用于任何格式）
|
||||
const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
|
||||
if (sourceMatch && sourceMatch[1]) {
|
||||
try {
|
||||
params.utm_source = decodeURIComponent(sourceMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_source = sourceMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
|
||||
if (mediumMatch && mediumMatch[1]) {
|
||||
try {
|
||||
params.utm_medium = decodeURIComponent(mediumMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_medium = mediumMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) {
|
||||
try {
|
||||
params.utm_campaign = decodeURIComponent(campaignMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_campaign = campaignMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
|
||||
if (termMatch && termMatch[1]) {
|
||||
try {
|
||||
params.utm_term = decodeURIComponent(termMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_term = termMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
|
||||
if (contentMatch && contentMatch[1]) {
|
||||
try {
|
||||
params.utm_content = decodeURIComponent(contentMatch[1]);
|
||||
} catch (_) {
|
||||
params.utm_content = contentMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
export async function main(
|
||||
batch_size = 1000,
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false,
|
||||
force_insert = true,
|
||||
database_override = "shorturl_analytics", // 添加数据库名称参数，默认为shorturl_analytics
|
||||
reset_sync_state = false // 添加参数用于重置同步状态
|
||||
reset_sync_state = false, // 添加参数用于重置同步状态
|
||||
debug_utm = false, // 添加参数控制UTM调试日志输出
|
||||
start_time?: number, // 添加参数指定同步的开始时间戳，可选
|
||||
end_time?: number, // 添加参数指定同步的结束时间戳，可选
|
||||
use_custom_time_range = false, // 添加参数控制是否使用自定义时间范围
|
||||
start_date?: string, // 添加开始日期字符串参数（ISO格式或yyyy-MM-dd格式）
|
||||
end_date?: string // 添加结束日期字符串参数（ISO格式或yyyy-MM-dd格式）
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
@@ -58,6 +237,34 @@ export async function main(
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
|
||||
// 处理日期字符串参数，转换为时间戳
|
||||
if (start_date) {
|
||||
try {
|
||||
start_time = dateToTimestamp(start_date);
|
||||
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${start_time}`);
|
||||
use_custom_time_range = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (end_date) {
|
||||
try {
|
||||
end_time = dateToTimestamp(end_date);
|
||||
// 如果是日期格式，设置为当天结束时间 (23:59:59.999)
|
||||
if (end_date.split('-').length === 3 && end_date.length <= 10) {
|
||||
end_time += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${end_time}`);
|
||||
} else {
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${end_time}`);
|
||||
}
|
||||
use_custom_time_range = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
|
||||
}
|
||||
@@ -67,6 +274,17 @@ export async function main(
|
||||
if (reset_sync_state) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用重置同步状态，将从头开始同步数据");
|
||||
}
|
||||
if (debug_utm) {
|
||||
logWithTimestamp("已启用UTM参数调试日志");
|
||||
}
|
||||
if (use_custom_time_range) {
|
||||
if (start_time) {
|
||||
logWithTimestamp(`已启用自定义时间范围：开始时间 ${new Date(start_time).toISOString()}`);
|
||||
}
|
||||
if (end_time) {
|
||||
logWithTimestamp(`已启用自定义时间范围：结束时间 ${new Date(end_time).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
@@ -181,14 +399,36 @@ export async function main(
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const traceCollection = db.collection<TraceRecord>("trace");
|
||||
// 添加对short集合的引用
|
||||
const shortCollection = db.collection<ShortRecord>("short");
|
||||
|
||||
// 构建查询条件，根据上次同步状态获取新记录
|
||||
const query: Record<string, unknown> = {
|
||||
type: 1 // 只同步type为1的记录
|
||||
// 删除了 type: 1 的条件，将同步所有数据
|
||||
};
|
||||
|
||||
// 如果有上次同步状态，则只获取更新的记录
|
||||
if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 根据时间范围参数构建查询条件
|
||||
if (use_custom_time_range) {
|
||||
// 使用自定义时间范围
|
||||
const timeQuery: Record<string, number> = {};
|
||||
|
||||
if (start_time) {
|
||||
timeQuery.$gte = start_time;
|
||||
logWithTimestamp(`将只同步createTime >= ${start_time} (${new Date(start_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
if (end_time) {
|
||||
timeQuery.$lte = end_time;
|
||||
logWithTimestamp(`将只同步createTime <= ${end_time} (${new Date(end_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 只有当至少指定了一个时间限制时才添加时间查询条件
|
||||
if (Object.keys(timeQuery).length > 0) {
|
||||
query.createTime = timeQuery;
|
||||
}
|
||||
}
|
||||
// 如果不使用自定义时间范围，且有上次同步状态，则只获取更新的记录
|
||||
else if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 使用上次同步时间作为过滤条件
|
||||
query.createTime = { $gt: lastSyncState.last_sync_time };
|
||||
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
|
||||
@@ -247,104 +487,6 @@ export async function main(
|
||||
}
|
||||
};
|
||||
|
||||
// 检查记录是否已经存在于ClickHouse中
|
||||
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
// 如果跳过ClickHouse检查或强制插入，则直接返回所有记录
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`已跳过ClickHouse重复检查，准备处理所有 ${records.length} 条记录`);
|
||||
return records;
|
||||
}
|
||||
|
||||
logWithTimestamp(`正在检查 ${records.length} 条记录是否已存在于ClickHouse中...`);
|
||||
|
||||
try {
|
||||
// 验证数据库名称
|
||||
if (!clickhouseConfig.clickhouse_database || clickhouseConfig.clickhouse_database === "undefined") {
|
||||
throw new Error("数据库名称未定义或无效，请检查配置");
|
||||
}
|
||||
|
||||
// 提取所有记录的ID
|
||||
const recordIds = records.map(record => record.slugId.toString()); // 使用slugId作为link_id查询
|
||||
logWithTimestamp(`待检查的记录ID: ${recordIds.join(', ')}`);
|
||||
|
||||
// 构建查询SQL，检查记录是否已存在，确保添加FORMAT JSON来获取正确的JSON格式响应
|
||||
const query = `
|
||||
SELECT link_id, visitor_id
|
||||
FROM ${clickhouseConfig.clickhouse_database}.events
|
||||
WHERE link_id IN ('${recordIds.join("','")}')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
logWithTimestamp(`执行ClickHouse查询: ${query.replace(/\n\s*/g, ' ')}`);
|
||||
|
||||
// 发送请求到ClickHouse，添加10秒超时
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse查询错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
// 获取响应文本以便记录
|
||||
const responseText = await response.text();
|
||||
logWithTimestamp(`ClickHouse查询响应: ${responseText.slice(0, 200)}${responseText.length > 200 ? '...' : ''}`);
|
||||
|
||||
if (!responseText.trim()) {
|
||||
logWithTimestamp("ClickHouse返回空响应，假定没有记录存在");
|
||||
return records; // 如果响应为空，假设没有记录
|
||||
}
|
||||
|
||||
// 解析结果
|
||||
let result;
|
||||
try {
|
||||
result = JSON.parse(responseText);
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse响应不是有效的JSON: ${responseText}`);
|
||||
throw new Error(`解析ClickHouse响应失败: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// 确保result有正确的结构
|
||||
if (!result.data) {
|
||||
logWithTimestamp(`ClickHouse响应缺少data字段: ${JSON.stringify(result)}`);
|
||||
return records; // 如果没有data字段，假设没有记录
|
||||
}
|
||||
|
||||
// 提取已存在的记录ID
|
||||
const existingIds = new Set(result.data.map((row: { link_id: string }) => row.link_id));
|
||||
|
||||
logWithTimestamp(`检测到 ${existingIds.size} 条记录已存在于ClickHouse中`);
|
||||
if (existingIds.size > 0) {
|
||||
logWithTimestamp(`已存在的记录ID: ${Array.from(existingIds).join(', ')}`);
|
||||
}
|
||||
|
||||
// 过滤出不存在的记录
|
||||
const newRecords = records.filter(record => !existingIds.has(record.slugId.toString())); // 使用slugId匹配link_id
|
||||
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
|
||||
|
||||
return newRecords;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`ClickHouse查询出错: ${error.message}`);
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，将继续处理所有记录");
|
||||
return records;
|
||||
} else {
|
||||
throw error; // 如果没有启用跳过检查，则抛出错误
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
@@ -358,74 +500,97 @@ export async function main(
|
||||
|
||||
logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
|
||||
|
||||
// 检查记录是否已存在
|
||||
let newRecords;
|
||||
try {
|
||||
newRecords = await checkExistingRecords(records);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
|
||||
if (!skip_clickhouse_check && !force_insert) {
|
||||
throw error;
|
||||
}
|
||||
// 如果跳过检查或强制插入，则使用所有记录
|
||||
logWithTimestamp("将使用所有记录进行处理");
|
||||
newRecords = records;
|
||||
}
|
||||
// 强制使用所有记录，不检查重复
|
||||
const newRecords = records;
|
||||
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("所有记录都已存在，跳过处理");
|
||||
return 0;
|
||||
}
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
|
||||
// 获取链接信息 - 新增代码
|
||||
const slugIds = newRecords.map(record => record.slugId);
|
||||
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
|
||||
const shortLinks = await shortCollection.find({
|
||||
_id: { $in: slugIds }
|
||||
}).toArray();
|
||||
|
||||
// 创建映射用于快速查找 - 新增代码
|
||||
const shortLinksMap = new Map(shortLinks.map((link: ShortRecord) => [link._id.toString(), link]));
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息，${newRecords.length - shortLinks.length} 条数据将使用占位符`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const eventTime = new Date(record.createTime);
|
||||
|
||||
// 获取对应的短链接信息 - 新增代码
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString()) as ShortRecord | undefined;
|
||||
|
||||
// 提取URL中的UTM参数 - 增加调试日志
|
||||
if (debug_utm && record.url) {
|
||||
logWithTimestamp(`======== UTM参数调试 ========`);
|
||||
logWithTimestamp(`记录ID: ${record._id.toString()}`);
|
||||
logWithTimestamp(`原始URL: ${record.url}`);
|
||||
}
|
||||
|
||||
const utmParams = extractUtmParams(record.url || "", debug_utm);
|
||||
|
||||
if (debug_utm) {
|
||||
logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
|
||||
logWithTimestamp(`===========================`);
|
||||
}
|
||||
|
||||
// 保存提取的UTM参数和URL到event_attributes
|
||||
const eventAttributes = {
|
||||
mongo_id: record._id.toString(),
|
||||
url: record.url || "",
|
||||
...(record.url ? { raw_url: record.url } : {})
|
||||
};
|
||||
|
||||
// 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
|
||||
return {
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
event_type: record.type === 1 ? "visit" : "custom",
|
||||
event_attributes: `{"mongo_id":"${record._id.toString()}"}`,
|
||||
event_type: "click", // 将所有event_type都设置为click
|
||||
event_attributes: JSON.stringify(eventAttributes),
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: "", // 这些字段可能需要从其他表获取
|
||||
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
|
||||
link_label: record.label || "",
|
||||
link_title: "",
|
||||
link_original_url: "",
|
||||
link_attributes: "{}",
|
||||
link_created_at: eventTime.toISOString().replace('T', ' ').replace('Z', ''), // 暂用访问时间代替，可能需要从其他表获取
|
||||
link_expires_at: null,
|
||||
link_tags: "[]",
|
||||
user_id: "",
|
||||
user_name: "",
|
||||
link_title: shortLink?.title || "unknown_title", // 使用占位符
|
||||
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
|
||||
link_created_at: shortLink?.createTime
|
||||
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
link_expires_at: shortLink?.expiresAt
|
||||
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: null,
|
||||
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
|
||||
user_id: "3680f452-e404-4339-a3d2-2a8e1ff92102", // 使用占位符
|
||||
user_name: "unknown_user", // 使用占位符
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
team_id: "",
|
||||
team_name: "",
|
||||
team_id: "e02251eb-eb98-47c8-b5dd-4f6e4fdb1f49", // 使用占位符
|
||||
team_name: "", // 使用占位符
|
||||
team_attributes: "{}",
|
||||
project_id: "",
|
||||
project_name: "",
|
||||
project_id: "34cdb8b9-8b8e-4033-876a-0632002ef1f9", // 使用占位符
|
||||
project_name: "", // 使用占位符
|
||||
project_attributes: "{}",
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
visitor_id: record._id.toString(), // 使用MongoDB ID作为访客ID
|
||||
session_id: record._id.toString() + "-" + record.createTime, // 创建一个唯一会话ID
|
||||
ip_address: record.ip,
|
||||
country: "", // 这些字段在MongoDB中不存在，使用默认值
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: record._id.toString() + "-" + record.createTime,
|
||||
ip_address: record.ip || "0.0.0.0", // 使用占位符
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "",
|
||||
os: record.platformOS || "",
|
||||
user_agent: record.browser + " " + record.browserVersion,
|
||||
browser: record.browser || "unknown", // 使用占位符
|
||||
os: record.platformOS || "unknown", // 使用占位符
|
||||
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
|
||||
referrer: record.url || "",
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
utm_term: "",
|
||||
utm_content: "",
|
||||
utm_source: utmParams.utm_source || "",
|
||||
utm_medium: utmParams.utm_medium || "",
|
||||
utm_campaign: utmParams.utm_campaign || "",
|
||||
utm_term: utmParams.utm_term || "",
|
||||
utm_content: utmParams.utm_content || "",
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
@@ -446,12 +611,23 @@ export async function main(
|
||||
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
|
||||
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
|
||||
VALUES ${clickhouseData.map(record => {
|
||||
// 确保所有字符串值都是字符串类型，并安全处理替换
|
||||
// 增强版安全替换函数，处理所有特殊字符
|
||||
const safeReplace = (val: unknown): string => {
|
||||
// 确保值是字符串，如果是null或undefined则使用空字符串
|
||||
const str = val === null || val === undefined ? "" : String(val);
|
||||
// 安全替换单引号
|
||||
return str.replace(/'/g, "''");
|
||||
|
||||
// 转义所有可能导致SQL注入或格式错误的字符
|
||||
// 1. 先替换所有反斜杠
|
||||
// 2. 再替换单引号
|
||||
// 3. 替换所有控制字符和特殊字符
|
||||
return str
|
||||
.replace(/\\/g, "\\\\") // 转义反斜杠
|
||||
.replace(/'/g, "\\'") // 转义单引号
|
||||
.replace(/\r/g, "\\r") // 转义回车
|
||||
.replace(/\n/g, "\\n") // 转义换行
|
||||
.replace(/\t/g, "\\t") // 转义制表符
|
||||
.replace(/\0/g, "") // 移除空字符
|
||||
.replace(/[\x00-\x1F\x7F-\x9F]/g, ""); // 移除所有控制字符
|
||||
};
|
||||
|
||||
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
|
||||
@@ -509,7 +685,6 @@ export async function main(
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
let lastSyncTime = 0;
|
||||
let lastSyncId = "";
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
@@ -545,6 +720,17 @@ export async function main(
|
||||
if (records.length > 1) {
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
|
||||
}
|
||||
|
||||
// 如果开启了调试，输出一些URL样本
|
||||
if (debug_utm) {
|
||||
const sampleSize = Math.min(5, records.length);
|
||||
logWithTimestamp(`URL样本 (前${sampleSize}条):`);
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
if (records[i].url) {
|
||||
logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
@@ -555,7 +741,6 @@ export async function main(
|
||||
if (records.length > 0) {
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
|
||||
lastSyncId = lastRecord._id.toString();
|
||||
}
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
@@ -563,20 +748,25 @@ export async function main(
|
||||
|
||||
// 更新同步状态
|
||||
if (processedRecords > 0 && lastSyncTime > 0) {
|
||||
// 创建新的同步状态
|
||||
const newSyncState: SyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + totalBatchRecords,
|
||||
last_sync_id: lastSyncId
|
||||
};
|
||||
|
||||
try {
|
||||
// 保存同步状态
|
||||
await setVariable(SYNC_STATE_KEY, newSyncState);
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`更新同步状态失败: ${error.message}`);
|
||||
// 只在非自定义时间范围模式下更新同步状态
|
||||
if (!use_custom_time_range) {
|
||||
// 创建新的同步状态，简化对象结构
|
||||
const newSyncState: SyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords, // 使用处理的总记录数，而不是实际插入数
|
||||
};
|
||||
|
||||
try {
|
||||
// 保存同步状态
|
||||
await setVariable(SYNC_STATE_KEY, newSyncState);
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`更新同步状态失败: ${error.message}，将继续执行`);
|
||||
// 不抛出错误，继续执行
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("使用自定义时间范围模式，不更新全局同步状态");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -585,7 +775,8 @@ export async function main(
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
|
||||
message: "数据同步完成"
|
||||
message: use_custom_time_range ? "自定义时间范围数据同步完成" : "数据同步完成",
|
||||
custom_time_range_used: use_custom_time_range
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("同步过程中发生错误:", err);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// 描述: 此脚本从PostgreSQL数据库获取所有shorturl类型的资源及其关联数据，并同步到ClickHouse
|
||||
|
||||
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
|
||||
import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
|
||||
import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
|
||||
|
||||
// 资源属性接口
|
||||
interface ResourceAttributes {
|
||||
@@ -37,6 +37,15 @@ interface PgConfig {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// 上次同步状态接口
|
||||
interface SyncState {
|
||||
lastSyncTime: string;
|
||||
lastRunTime: string;
|
||||
}
|
||||
|
||||
// 状态变量名称
|
||||
const STATE_VARIABLE_PATH = "f/shorturl_analytics/shorturl_sync_state";
|
||||
|
||||
// Windmill函数定义
|
||||
export async function main(
|
||||
/** PostgreSQL和ClickHouse同步脚本 */
|
||||
@@ -47,9 +56,11 @@ export async function main(
|
||||
includeDeleted?: boolean;
|
||||
/** 是否执行实际写入操作 */
|
||||
dryRun?: boolean;
|
||||
/** 开始时间（ISO格式）*/
|
||||
/** 是否强制全量同步 */
|
||||
forceFullSync?: boolean;
|
||||
/** 手动指定开始时间（ISO格式）- 会覆盖自动增量设置 */
|
||||
startTime?: string;
|
||||
/** 结束时间（ISO格式）*/
|
||||
/** 手动指定结束时间（ISO格式）*/
|
||||
endTime?: string;
|
||||
}
|
||||
) {
|
||||
@@ -57,8 +68,41 @@ export async function main(
|
||||
const limit = params.limit || 500;
|
||||
const includeDeleted = params.includeDeleted || false;
|
||||
const dryRun = params.dryRun || false;
|
||||
const startTime = params.startTime ? new Date(params.startTime) : undefined;
|
||||
const endTime = params.endTime ? new Date(params.endTime) : undefined;
|
||||
const forceFullSync = params.forceFullSync || false;
|
||||
|
||||
// 获取当前时间作为本次运行时间
|
||||
const currentRunTime = new Date().toISOString();
|
||||
|
||||
// 初始化同步状态
|
||||
let syncState: SyncState;
|
||||
let startTime: Date | undefined;
|
||||
const endTime: Date | undefined = params.endTime ? new Date(params.endTime) : new Date();
|
||||
|
||||
// 如果强制全量同步或手动指定了开始时间，则使用指定的开始时间
|
||||
if (forceFullSync || params.startTime) {
|
||||
startTime = params.startTime ? new Date(params.startTime) : undefined;
|
||||
console.log(`使用${params.startTime ? '手动指定' : '全量同步'} - 开始时间: ${startTime ? startTime.toISOString() : '无限制'}`);
|
||||
}
|
||||
// 否则尝试获取上次同步时间作为增量同步的开始时间点
|
||||
else {
|
||||
try {
|
||||
// 获取上次同步状态
|
||||
const stateStr = await getVariable(STATE_VARIABLE_PATH);
|
||||
if (stateStr) {
|
||||
syncState = JSON.parse(stateStr);
|
||||
console.log(`获取到上次同步状态: 同步时间=${syncState.lastSyncTime}, 运行时间=${syncState.lastRunTime}`);
|
||||
|
||||
// 使用上次运行时间作为本次的开始时间 (减去1分钟防止边界问题)
|
||||
const lastRunTime = new Date(syncState.lastRunTime);
|
||||
lastRunTime.setMinutes(lastRunTime.getMinutes() - 1);
|
||||
startTime = lastRunTime;
|
||||
} else {
|
||||
console.log("未找到上次同步状态，将执行全量同步");
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
console.log(`获取同步状态出错: ${error instanceof Error ? error.message : String(error)}，将执行全量同步`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`开始同步PostgreSQL shorturl数据到ClickHouse`);
|
||||
console.log(`参数: limit=${limit}, includeDeleted=${includeDeleted}, dryRun=${dryRun}`);
|
||||
@@ -67,7 +111,7 @@ export async function main(
|
||||
|
||||
// 获取数据库配置
|
||||
console.log("获取PostgreSQL数据库配置...");
|
||||
const pgConfig = await getResource('f/limq/postgresql') as PgConfig;
|
||||
const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
|
||||
console.log(`数据库连接配置: host=${pgConfig.host}, port=${pgConfig.port}, database=${pgConfig.dbname || 'postgres'}, user=${pgConfig.user}`);
|
||||
|
||||
let pgPool: Pool | null = null;
|
||||
@@ -106,6 +150,8 @@ export async function main(
|
||||
console.log(`获取到 ${shorturls.length} 个shorturl资源`);
|
||||
|
||||
if (shorturls.length === 0) {
|
||||
// 即使没有数据也更新状态
|
||||
await updateSyncState(currentRunTime);
|
||||
return { synced: 0, message: "没有找到需要同步的shorturl资源" };
|
||||
}
|
||||
|
||||
@@ -120,7 +166,11 @@ export async function main(
|
||||
// 写入ClickHouse
|
||||
const inserted = await insertToClickhouse(clickhouseData);
|
||||
console.log(`成功写入 ${inserted} 条记录到ClickHouse`);
|
||||
return { synced: inserted, message: "同步完成" };
|
||||
|
||||
// 更新同步状态
|
||||
await updateSyncState(currentRunTime);
|
||||
|
||||
return { synced: inserted, message: "同步完成", lastSyncTime: currentRunTime };
|
||||
} else {
|
||||
console.log("Dry run模式 - 不执行实际写入");
|
||||
console.log(`将写入 ${clickhouseData.length} 条记录到ClickHouse`);
|
||||
@@ -146,6 +196,22 @@ export async function main(
|
||||
}
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
async function updateSyncState(currentRunTime: string): Promise<void> {
|
||||
try {
|
||||
const syncState: SyncState = {
|
||||
lastSyncTime: new Date().toISOString(), // 记录数据同步完成的时间
|
||||
lastRunTime: currentRunTime // 记录本次运行的时间点
|
||||
};
|
||||
|
||||
console.log(`更新同步状态: ${JSON.stringify(syncState)}`);
|
||||
await setVariable(STATE_VARIABLE_PATH, JSON.stringify(syncState));
|
||||
} catch (error: unknown) {
|
||||
console.error(`更新同步状态失败: ${error instanceof Error ? error.message : String(error)}`);
|
||||
// 不中断主流程，即使状态更新失败
|
||||
}
|
||||
}
|
||||
|
||||
// 从PostgreSQL获取所有shorturl资源
|
||||
async function fetchShorturlResources(
|
||||
pgPool: Pool,
|
||||
@@ -185,8 +251,9 @@ async function fetchShorturlResources(
|
||||
query += ` AND r.deleted_at IS NULL`;
|
||||
}
|
||||
|
||||
// 修改为同时考虑created_at和updated_at，确保捕获自上次同步以来创建或更新的记录
|
||||
if (options.startTime) {
|
||||
query += ` AND r.created_at >= $${paramCount}`;
|
||||
query += ` AND (r.created_at >= $${paramCount} OR r.updated_at >= $${paramCount})`;
|
||||
params.push(options.startTime);
|
||||
paramCount++;
|
||||
}
|
||||
@@ -197,7 +264,8 @@ async function fetchShorturlResources(
|
||||
paramCount++;
|
||||
}
|
||||
|
||||
query += ` ORDER BY r.created_at DESC LIMIT $${paramCount}`;
|
||||
// 优先按更新时间排序，确保最近更新的记录先处理
|
||||
query += ` ORDER BY r.updated_at DESC, r.created_at DESC LIMIT $${paramCount}`;
|
||||
params.push(options.limit);
|
||||
|
||||
const client = await pgPool.connect();
|
||||
|
||||
Reference in New Issue
Block a user