Compare commits
18 Commits
3cbb76db36
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
51e168ee3b | ||
|
|
cf0f35e274 | ||
| 3162836e91 | |||
| d80d5e976b | |||
| 5d5b501a66 | |||
| fe40aad835 | |||
| 92db5ad783 | |||
| b94a91914a | |||
| 8551f5c445 | |||
| dafa7f53ac | |||
| 0203cb4041 | |||
| ced29201da | |||
| a8c94c9621 | |||
| 4736ebe060 | |||
| 6858f2fda5 | |||
| 42f5be4dcb | |||
| 05af4aae70 | |||
| ed1d2e59f6 |
31
.env
Normal file
31
.env
Normal file
@@ -0,0 +1,31 @@
|
||||
PORT=3007
|
||||
|
||||
MONGO_URL="mongodb://10.0.1.41:27017"
|
||||
|
||||
# ClickHouse Configuration
|
||||
CLICKHOUSE_HOST=10.0.1.60
|
||||
CLICKHOUSE_PORT=8123
|
||||
CLICKHOUSE_USER=admin
|
||||
CLICKHOUSE_PASSWORD=your_secure_password
|
||||
CLICKHOUSE_DATABASE=shorturl_analytics
|
||||
CLICKHOUSE_URL=http://10.0.1.60:8123
|
||||
|
||||
REDIS_HOST="localhost"
|
||||
REDIS_PORT="6379"
|
||||
REDIS_PASSWORD=""
|
||||
|
||||
# Supabase Configuration
|
||||
SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
|
||||
SUPABASE_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc0NDM0NTY0MywiZXhwIjoyMDU5OTIxNjQzfQ.ZenTsEAdGiDu1DCCOT7G8xxvgFXKLl4qhHB-AhSVf6w"
|
||||
SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
|
||||
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
|
||||
|
||||
|
||||
# Next.js Public Environment Variables (accessible in browser)
|
||||
NEXT_PUBLIC_SUPABASE_URL="https://mwwvqwevplndzvmqmrxa.supabase.co"
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im13d3Zxd2V2cGxuZHp2bXFtcnhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzNDU2NDMsImV4cCI6MjA1OTkyMTY0M30.EI7OY0Aq3zYj6fRG_IUn4IlUZ89b0LOg0jb0nMLLKWU"
|
||||
DATABASE_URL="postgresql://postgres.mwwvqwevplndzvmqmrxa:eYYdarJsRL*Z6&p9gD@aws-0-ap-northeast-1.pooler.supabase.com:5432/postgres"
|
||||
|
||||
NEXT_PUBLIC_LIMQ_API="https://app.upj.to"
|
||||
# Application URL for redirects (replace with your production URL)
|
||||
NEXT_PUBLIC_SITE_URL="https://main.upj.to"
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -31,7 +31,7 @@ yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env
|
||||
# .env
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
42
README-auth-setup.md
Normal file
42
README-auth-setup.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# 身份验证重定向 URL 配置指南
|
||||
|
||||
## 问题：注册后确认邮件链接指向 localhost
|
||||
|
||||
如果您在生产环境中使用此应用，并且发现用户注册后收到的确认邮件中的链接指向 `localhost` 而非您的实际网站域名，请按照以下步骤解决：
|
||||
|
||||
## 解决方案
|
||||
|
||||
### 1. 设置环境变量
|
||||
|
||||
在项目根目录的 `.env.production` 文件中，确保 `NEXT_PUBLIC_SITE_URL` 变量设置为您的实际生产域名：
|
||||
|
||||
```
|
||||
NEXT_PUBLIC_SITE_URL="https://您的真实域名.com"
|
||||
```
|
||||
|
||||
### 2. 在 Supabase 控制台中配置
|
||||
|
||||
登录 [Supabase 控制台](https://app.supabase.com/)，然后：
|
||||
|
||||
1. 选择您的项目
|
||||
2. 导航到 **Authentication** > **URL Configuration**
|
||||
3. 在 **Site URL** 字段中输入您的实际网站 URL
|
||||
4. 在 **Redirect URLs** 部分添加：
|
||||
- `https://您的真实域名.com/auth/callback`
|
||||
|
||||
### 3. 本地开发与生产环境
|
||||
|
||||
- **开发环境**：使用 `.env.development` 文件中的设置，通常为 `http://localhost:3007`
|
||||
- **生产环境**：使用 `.env.production` 文件中的设置，应为您的实际域名
|
||||
|
||||
### 4. 部署后验证
|
||||
|
||||
项目重新部署后：
|
||||
1. 尝试注册一个新账户
|
||||
2. 检查收到的确认邮件，确认链接现在指向您的实际域名而非 localhost
|
||||
|
||||
## 技术说明
|
||||
|
||||
身份验证流程中，应用使用环境变量 `NEXT_PUBLIC_SITE_URL` 构建重定向 URL。如果未设置此变量，它会回退到使用 `window.location.origin`，这在本地开发时会是 `localhost`。
|
||||
|
||||
通过正确设置此变量，您可以确保无论在何处运行应用，邮件中的链接都能正确指向应用的实际位置。
|
||||
50
README-google-auth.md
Normal file
50
README-google-auth.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# 配置 Google 登录功能
|
||||
|
||||
为了启用 Google 登录功能，您需要在 Supabase 和 Google Cloud Platform 进行配置。
|
||||
|
||||
## 步骤 1: 创建 Google OAuth 客户端
|
||||
|
||||
1. 访问 [Google Cloud Console](https://console.cloud.google.com/)
|
||||
2. 创建一个新项目或选择现有项目
|
||||
3. 在左侧菜单中导航到 "API 和服务" > "OAuth 同意屏幕"
|
||||
4. 选择用户类型（外部或内部），然后点击"创建"
|
||||
5. 填写必要的信息（应用名称、用户支持电子邮件等）并保存
|
||||
6. 导航到 "API 和服务" > "凭据"
|
||||
7. 点击"创建凭据" > "OAuth 客户端 ID"
|
||||
8. 应用类型选择 "Web 应用"
|
||||
9. 名称中输入您的应用名称
|
||||
10. 添加以下已获授权的重定向 URI:
|
||||
- `https://mwwvqwevplndzvmqmrxa.supabase.co/auth/v1/callback`
|
||||
11. 点击"创建"
|
||||
12. 复制生成的 "客户端 ID" 和 "客户端密钥"
|
||||
|
||||
## 步骤 2: 在 Supabase 中配置 Google 提供商
|
||||
|
||||
1. 登录 [Supabase 仪表板](https://app.supabase.com)
|
||||
2. 选择您的项目
|
||||
3. 导航到 "身份验证" > "提供商"
|
||||
4. 找到 Google 提供商并启用它
|
||||
5. 粘贴您刚才获取的 "客户端 ID" 和 "客户端密钥"
|
||||
6. 保存配置
|
||||
|
||||
## 步骤 3: 更新重定向 URL（如有需要）
|
||||
|
||||
如果您的应用需要在登录后重定向到特定页面，请确保在 Google Cloud Console 和 Supabase 中配置了正确的重定向 URL。
|
||||
|
||||
在 Supabase 中:
|
||||
1. 导航到 "身份验证" > "URL 配置"
|
||||
2. 添加您的前端 URL 到站点 URL 字段中
|
||||
3. 设置重定向 URL（通常是您的前端 URL）
|
||||
|
||||
## 测试
|
||||
|
||||
1. 在您的应用中，尝试使用 Google 登录
|
||||
2. 验证认证流程，确保可以成功登录并重定向到应用
|
||||
3. 检查 Supabase 中的用户数据，确认新用户已创建
|
||||
|
||||
## 故障排除
|
||||
|
||||
- 确保重定向 URI 完全匹配
|
||||
- 确保 OAuth 同意屏幕已正确配置
|
||||
- 查看 Supabase 和应用程序中的日志以获取详细的错误信息
|
||||
- 如果遇到 CORS 错误，检查您的站点 URL 配置
|
||||
@@ -14,6 +14,7 @@ import { ProjectSelector } from '@/app/components/ui/ProjectSelector';
|
||||
import { TagSelector } from '@/app/components/ui/TagSelector';
|
||||
import { useSearchParams } from 'next/navigation';
|
||||
import { useShortUrlStore } from '@/app/utils/store';
|
||||
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
|
||||
|
||||
// 事件类型定义
|
||||
interface Event {
|
||||
@@ -1109,6 +1110,7 @@ function AnalyticsContent() {
|
||||
// Main page component with Suspense
|
||||
export default function AnalyticsPage() {
|
||||
return (
|
||||
<ClientRouteGuard>
|
||||
<Suspense fallback={
|
||||
<div className="flex items-center justify-center min-h-screen">
|
||||
<div className="animate-spin rounded-full h-12 w-12 border-t-2 border-b-2 border-blue-500" />
|
||||
@@ -1116,5 +1118,6 @@ export default function AnalyticsPage() {
|
||||
}>
|
||||
<AnalyticsContent />
|
||||
</Suspense>
|
||||
</ClientRouteGuard>
|
||||
);
|
||||
}
|
||||
126
app/api/activities/readme.md
Normal file
126
app/api/activities/readme.md
Normal file
@@ -0,0 +1,126 @@
|
||||
# Activities API Documentation
|
||||
|
||||
## Overview
|
||||
The Activities API provides event tracking data for short URLs. It allows retrieving visitor activity information based on various filters such as URL slug, domain, and date ranges.
|
||||
|
||||
## Endpoint
|
||||
```
|
||||
GET /api/activities
|
||||
```
|
||||
|
||||
## Request Parameters
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|------------|---------|----------|-------------|
|
||||
| `slug` | string | No* | The short URL slug to filter events by |
|
||||
| `domain` | string | No* | The domain to filter events by |
|
||||
| `startTime`| string | No* | Start time for date range filter (ISO format) |
|
||||
| `endTime` | string | No* | End time for date range filter (ISO format) |
|
||||
| `page` | integer | No | Page number for pagination (default: 1) |
|
||||
| `pageSize` | integer | No | Number of records per page (default: 50) |
|
||||
| `format` | string | No | Response format, set to 'csv' for CSV output (default: JSON) |
|
||||
|
||||
\* Either `slug`+`domain` combination OR at least one of `startTime`/`endTime` must be provided.
|
||||
|
||||
## Response Formats
|
||||
|
||||
### JSON Format (Default)
|
||||
JSON responses include the following structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": [
|
||||
{
|
||||
"id": "event-id",
|
||||
"type": "event-type",
|
||||
"time": "timestamp",
|
||||
"visitor": {
|
||||
"id": "visitor-id",
|
||||
"ipAddress": "ip-address",
|
||||
"userAgent": "user-agent-string",
|
||||
"referrer": "referrer-url"
|
||||
},
|
||||
"device": {
|
||||
"type": "device-type",
|
||||
"browser": "browser-name",
|
||||
"os": "operating-system"
|
||||
},
|
||||
"location": {
|
||||
"country": "country-code",
|
||||
"city": "city-name"
|
||||
},
|
||||
"link": {
|
||||
"id": "link-id",
|
||||
"slug": "link-slug",
|
||||
"originalUrl": "original-url",
|
||||
"label": "link-label",
|
||||
"tags": ["tag1", "tag2"]
|
||||
},
|
||||
"utm": {
|
||||
"source": "utm-source",
|
||||
"medium": "utm-medium",
|
||||
"campaign": "utm-campaign",
|
||||
"term": "utm-term",
|
||||
"content": "utm-content"
|
||||
}
|
||||
}
|
||||
],
|
||||
"meta": {
|
||||
"total": 100,
|
||||
"page": 1,
|
||||
"pageSize": 50
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
In case of an error:
|
||||
```json
|
||||
{
|
||||
"success": false,
|
||||
"data": null,
|
||||
"error": "Error message description"
|
||||
}
|
||||
```
|
||||
|
||||
### CSV Format
|
||||
When `format=csv` is specified, the response is returned as plain text in CSV format with the following columns:
|
||||
- `time`: Timestamp of the event
|
||||
- `activity`: Type of activity/event
|
||||
- `campaign`: UTM campaign value (defaults to "demo" if not found)
|
||||
- `clientId`: Visitor ID
|
||||
- `originPath`: Original request path or referrer URL
|
||||
|
||||
## Examples
|
||||
|
||||
### Get activities for a specific short URL
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link
|
||||
```
|
||||
|
||||
### Get activities within a date range
|
||||
```
|
||||
GET /api/activities?startTime=2023-06-01T00:00:00Z&endTime=2023-06-30T23:59:59Z
|
||||
```
|
||||
|
||||
### Get events as CSV
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link&format=csv
|
||||
```
|
||||
|
||||
### Pagination example
|
||||
```
|
||||
GET /api/activities?slug=promo123&domain=googleads.link&page=2&pageSize=20
|
||||
```
|
||||
|
||||
## Error Codes
|
||||
|
||||
| Status Code | Description |
|
||||
|-------------|-------------|
|
||||
| 400 | Missing required parameters |
|
||||
| 500 | Server error while processing the request |
|
||||
|
||||
## Notes
|
||||
- For privacy and security reasons, some fields may be omitted or anonymized based on user settings.
|
||||
- The CSV format is optimized for easy import into spreadsheet applications.
|
||||
- When using the CSV format, the response is returned as plain text rather than a downloadable file.
|
||||
@@ -2,8 +2,8 @@ import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getEvents } from '@/lib/analytics';
|
||||
import { ApiResponse } from '@/lib/types';
|
||||
|
||||
// 扩展Event类型以包含所需字段
|
||||
interface EventWithFullPath extends Record<string, any> {
|
||||
// Extended Event type with required fields
|
||||
interface EventWithFullPath {
|
||||
event_id?: string;
|
||||
event_time?: string;
|
||||
event_type?: string;
|
||||
@@ -11,7 +11,18 @@ interface EventWithFullPath extends Record<string, any> {
|
||||
ip_address?: string;
|
||||
req_full_path?: string;
|
||||
referrer?: string;
|
||||
// 其他可能的字段
|
||||
event_attributes?: string | Record<string, unknown>;
|
||||
link_tags?: string | string[];
|
||||
link_id?: string;
|
||||
link_slug?: string;
|
||||
link_original_url?: string;
|
||||
link_label?: string;
|
||||
device_type?: string;
|
||||
browser?: string;
|
||||
os?: string;
|
||||
country?: string;
|
||||
city?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
@@ -27,12 +38,19 @@ export async function GET(request: NextRequest) {
|
||||
const startTime = searchParams.get('startTime') || undefined;
|
||||
const endTime = searchParams.get('endTime') || undefined;
|
||||
|
||||
// 修改验证逻辑，允许只使用时间范围
|
||||
// 现在只需要确保有足够的过滤条件
|
||||
// Check if either slug or domain is provided without the other
|
||||
if ((slug && !domain) || (!slug && domain)) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Both slug and domain parameters must be provided together'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// Ensure either slug+domain or date range is provided
|
||||
if ((!slug && !domain) && (!startTime && !endTime)) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Missing filter parameters: provide either slug/domain or date range'
|
||||
error: 'Missing filter parameters: provide either slug+domain or date range'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
@@ -89,7 +107,7 @@ export async function GET(request: NextRequest) {
|
||||
// If utm_campaign is not found or URL parsing fails, use regex as fallback
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
|
||||
} catch (_) {
|
||||
} catch {
|
||||
// If URL parsing fails, try regex directly
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) return campaignMatch[1];
|
||||
@@ -118,26 +136,22 @@ export async function GET(request: NextRequest) {
|
||||
// Determine activity (event_type)
|
||||
const activity = eventWithFullPath.event_type || '';
|
||||
|
||||
// Client ID (possibly part of visitor_id)
|
||||
const clientId = eventWithFullPath.visitor_id?.split('-')[0] || 'undefined';
|
||||
// 修改：使用link_label替代visitor_id作为clientId
|
||||
const clientId = eventWithFullPath.link_label || 'undefined';
|
||||
|
||||
// Original path (use full URL field)
|
||||
const originPath = fullUrl || 'undefined';
|
||||
// Original path - 修正：使用link_original_url作为原始URL来源
|
||||
const originPath = eventWithFullPath.link_original_url || 'undefined';
|
||||
|
||||
// Add to CSV content
|
||||
csvContent += `${time},${activity},${campaign},${clientId},${originPath}\n`;
|
||||
});
|
||||
|
||||
// Generate filename based on available parameters
|
||||
const filename = slug
|
||||
? `activities-${slug}.csv`
|
||||
: `activities-${new Date().toISOString().slice(0,10)}.csv`;
|
||||
// No need to generate filename since we're not using Content-Disposition header
|
||||
|
||||
// Return CSV response
|
||||
// Return CSV response without forcing download
|
||||
return new NextResponse(csvContent, {
|
||||
headers: {
|
||||
'Content-Type': 'text/csv',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`
|
||||
'Content-Type': 'text/plain'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
18
app/auth/callback/route.ts
Normal file
18
app/auth/callback/route.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { createRouteHandlerClient } from '@supabase/auth-helpers-nextjs';
|
||||
import { cookies } from 'next/headers';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const requestUrl = new URL(request.url);
|
||||
const code = requestUrl.searchParams.get('code');
|
||||
|
||||
if (code) {
|
||||
const cookieStore = cookies();
|
||||
const supabase = createRouteHandlerClient({ cookies: () => cookieStore });
|
||||
await supabase.auth.exchangeCodeForSession(code);
|
||||
}
|
||||
|
||||
// URL to redirect to after sign in process completes
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || 'https://main.upj.to';
|
||||
return NextResponse.redirect(new URL('/analytics', siteUrl));
|
||||
}
|
||||
45
app/components/ClientRouteGuard.tsx
Normal file
45
app/components/ClientRouteGuard.tsx
Normal file
@@ -0,0 +1,45 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
|
||||
// 这个组件会检查 localStorage 中是否有认证令牌，如果没有则重定向到登录页面
|
||||
export default function ClientRouteGuard({ children }: { children: React.ReactNode }) {
|
||||
const router = useRouter();
|
||||
const [isAuthenticated, setIsAuthenticated] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
// 检查 localStorage 中是否有认证令牌
|
||||
const checkAuth = () => {
|
||||
// 查找 Supabase 认证令牌
|
||||
const hasAuthToken = !!localStorage.getItem('sb-mwwvqwevplndzvmqmrxa-auth-token') ||
|
||||
!!localStorage.getItem('sb-auth-token');
|
||||
|
||||
if (!hasAuthToken) {
|
||||
// 如果没有令牌，重定向到登录页面
|
||||
router.push('/login');
|
||||
} else {
|
||||
setIsAuthenticated(true);
|
||||
}
|
||||
setIsLoading(false);
|
||||
};
|
||||
|
||||
checkAuth();
|
||||
}, [router]);
|
||||
|
||||
// 显示加载状态
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen">
|
||||
<div className="text-center">
|
||||
<div className="animate-spin rounded-full h-16 w-16 border-t-2 border-b-2 border-blue-500 mx-auto"></div>
|
||||
<p className="mt-4 text-lg text-gray-700 dark:text-gray-300">加载中...</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// 只有当用户已认证时才显示子组件
|
||||
return isAuthenticated ? <>{children}</> : null;
|
||||
}
|
||||
@@ -44,6 +44,11 @@ export default function Header() {
|
||||
Short Links
|
||||
</Link>
|
||||
</li>
|
||||
<li>
|
||||
<Link href="/create-shorturl" className="text-sm text-gray-700 hover:text-blue-500">
|
||||
Create Short URL
|
||||
</Link>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
)}
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
'use client';
|
||||
|
||||
import Link from 'next/link';
|
||||
|
||||
export default function Navbar() {
|
||||
return (
|
||||
<header className="w-full py-4 border-b border-card-border bg-background">
|
||||
<div className="container flex items-center justify-between px-4 mx-auto">
|
||||
<div className="flex items-center space-x-4">
|
||||
<Link href="/" className="flex items-center space-x-2">
|
||||
<svg
|
||||
className="w-6 h-6 text-accent-blue"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path>
|
||||
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path>
|
||||
</svg>
|
||||
<span className="text-xl font-bold text-foreground">ShortURL</span>
|
||||
</Link>
|
||||
<nav className="hidden space-x-4 md:flex">
|
||||
<Link
|
||||
href="/links"
|
||||
className="text-sm text-foreground hover:text-accent-blue transition-colors"
|
||||
>
|
||||
Links
|
||||
</Link>
|
||||
<Link
|
||||
href="/analytics"
|
||||
className="text-sm text-foreground hover:text-accent-blue transition-colors"
|
||||
>
|
||||
Analytics
|
||||
</Link>
|
||||
</nav>
|
||||
</div>
|
||||
<div className="flex items-center space-x-3">
|
||||
<button className="p-2 text-sm text-foreground rounded-md gradient-border">
|
||||
Upgrade
|
||||
</button>
|
||||
<button className="p-2 text-sm text-foreground hover:text-accent-blue">
|
||||
<svg
|
||||
className="w-5 h-5"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<circle cx="12" cy="12" r="10"></circle>
|
||||
<circle cx="12" cy="10" r="3"></circle>
|
||||
<path d="M7 20.662V19a2 2 0 0 1 2-2h6a2 2 0 0 1 2 2v1.662"></path>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { getSupabaseClient } from '../../utils/supabase';
|
||||
import { AuthChangeEvent, Session } from '@supabase/supabase-js';
|
||||
import { Loader2, X, Check } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { limqRequest } from '@/lib/api';
|
||||
|
||||
type Team = Database['limq']['Tables']['teams']['Row'];
|
||||
|
||||
@@ -70,6 +71,14 @@ export function TeamSelector({
|
||||
try {
|
||||
const supabase = getSupabaseClient();
|
||||
|
||||
// 尝试创建默认团队和项目（如果用户还没有）
|
||||
try {
|
||||
const response = await limqRequest('team/create-default', 'POST');
|
||||
console.log('Default team creation response:', response);
|
||||
} catch (teamError) {
|
||||
console.error('Error creating default team:', teamError);
|
||||
}
|
||||
|
||||
const { data: memberships, error: membershipError } = await supabase
|
||||
.from('team_membership')
|
||||
.select('team_id')
|
||||
|
||||
403
app/create-shorturl/page.tsx
Normal file
403
app/create-shorturl/page.tsx
Normal file
@@ -0,0 +1,403 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useAuth } from '@/lib/auth';
|
||||
import { limqRequest } from '@/lib/api';
|
||||
import { TeamSelector } from '@/app/components/ui/TeamSelector';
|
||||
import { ProjectSelector } from '@/app/components/ui/ProjectSelector';
|
||||
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
|
||||
|
||||
interface ShortUrlData {
|
||||
originalUrl: string;
|
||||
customSlug?: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
tags?: string[];
|
||||
teamId: string;
|
||||
projectId: string;
|
||||
domain: string;
|
||||
}
|
||||
|
||||
export default function CreateShortUrlPage() {
|
||||
return (
|
||||
<ClientRouteGuard>
|
||||
<CreateShortUrlForm />
|
||||
</ClientRouteGuard>
|
||||
);
|
||||
}
|
||||
|
||||
function CreateShortUrlForm() {
|
||||
const router = useRouter();
|
||||
const { user } = useAuth();
|
||||
|
||||
const [formData, setFormData] = useState<ShortUrlData>({
|
||||
originalUrl: '',
|
||||
customSlug: '',
|
||||
title: '',
|
||||
description: '',
|
||||
tags: [],
|
||||
teamId: '',
|
||||
projectId: '',
|
||||
domain: 'googleads.link',
|
||||
});
|
||||
|
||||
const [tagInput, setTagInput] = useState('');
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [success, setSuccess] = useState(false);
|
||||
|
||||
// Use useEffect to add user information to form data on load
|
||||
useEffect(() => {
|
||||
if (user) {
|
||||
console.log('Current user:', user.email);
|
||||
// Can add user-related data to the form here
|
||||
}
|
||||
}, [user]);
|
||||
|
||||
const handleChange = (e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
|
||||
const { name, value } = e.target;
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
[name]: value
|
||||
}));
|
||||
};
|
||||
|
||||
const handleTagKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === 'Enter' && tagInput.trim()) {
|
||||
e.preventDefault();
|
||||
addTag();
|
||||
}
|
||||
};
|
||||
|
||||
const addTag = () => {
|
||||
if (tagInput.trim() && !formData.tags?.includes(tagInput.trim())) {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
tags: [...(prev.tags || []), tagInput.trim()]
|
||||
}));
|
||||
setTagInput('');
|
||||
}
|
||||
};
|
||||
|
||||
const removeTag = (tagToRemove: string) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
tags: prev.tags?.filter(tag => tag !== tagToRemove)
|
||||
}));
|
||||
};
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
setIsSubmitting(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Validate required fields
|
||||
if (!formData.originalUrl) {
|
||||
throw new Error('Original URL is required');
|
||||
}
|
||||
|
||||
if (!formData.title) {
|
||||
throw new Error('Title is required');
|
||||
}
|
||||
|
||||
if (!formData.teamId) {
|
||||
throw new Error('Team is required');
|
||||
}
|
||||
|
||||
if (!formData.projectId) {
|
||||
throw new Error('Project is required');
|
||||
}
|
||||
|
||||
if (!formData.domain) {
|
||||
throw new Error('Domain is required');
|
||||
}
|
||||
|
||||
// Construct request data according to API requirements
|
||||
const requestData = {
|
||||
type: "shorturl",
|
||||
attributes: {
|
||||
// Can add any additional attributes, but attributes cannot be empty
|
||||
icon: ""
|
||||
},
|
||||
shortUrl: {
|
||||
url: formData.originalUrl,
|
||||
slug: formData.customSlug || undefined,
|
||||
title: formData.title,
|
||||
name: formData.title,
|
||||
description: formData.description || "",
|
||||
domain: formData.domain
|
||||
},
|
||||
teamId: formData.teamId,
|
||||
projectId: formData.projectId,
|
||||
tagIds: formData.tags && formData.tags.length > 0 ? formData.tags : undefined
|
||||
};
|
||||
|
||||
// Call API to create shorturl resource
|
||||
const response = await limqRequest('resource/shorturl', 'POST', requestData as unknown as Record<string, unknown>);
|
||||
|
||||
console.log('Created successfully:', response);
|
||||
setSuccess(true);
|
||||
|
||||
// Redirect to links list page after 2 seconds
|
||||
setTimeout(() => {
|
||||
router.push('/links');
|
||||
}, 2000);
|
||||
|
||||
} catch (err) {
|
||||
console.error('Failed to create short URL:', err);
|
||||
setError(err instanceof Error ? err.message : 'Failed to create short URL, please try again later');
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="container mx-auto px-4 py-8 max-w-3xl">
|
||||
<div className="bg-white rounded-lg shadow-md overflow-hidden">
|
||||
<div className="border-b border-gray-200 bg-blue-50 px-6 py-4">
|
||||
<h1 className="text-xl font-medium text-gray-900">Create Short URL</h1>
|
||||
<p className="mt-1 text-sm text-gray-600">
|
||||
Create a new short URL resource for tracking and analytics
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border-l-4 border-red-500 p-4 m-6">
|
||||
<div className="flex">
|
||||
<div className="flex-shrink-0">
|
||||
<svg className="h-5 w-5 text-red-500" fill="currentColor" viewBox="0 0 20 20">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
<div className="ml-3">
|
||||
<p className="text-sm text-red-700">{error}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{success && (
|
||||
<div className="bg-green-50 border-l-4 border-green-500 p-4 m-6">
|
||||
<div className="flex">
|
||||
<div className="flex-shrink-0">
|
||||
<svg className="h-5 w-5 text-green-500" fill="currentColor" viewBox="0 0 20 20">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
<div className="ml-3">
|
||||
<p className="text-sm text-green-700">
|
||||
Short URL created successfully! Redirecting...
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form onSubmit={handleSubmit} className="p-6 space-y-6">
|
||||
{/* Title */}
|
||||
<div>
|
||||
<label htmlFor="title" className="block text-sm font-medium text-gray-700">
|
||||
Title <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
id="title"
|
||||
name="title"
|
||||
value={formData.title}
|
||||
onChange={handleChange}
|
||||
placeholder="e.g., Product Launch Campaign"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Original URL */}
|
||||
<div>
|
||||
<label htmlFor="originalUrl" className="block text-sm font-medium text-gray-700">
|
||||
Original URL <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="url"
|
||||
id="originalUrl"
|
||||
name="originalUrl"
|
||||
value={formData.originalUrl}
|
||||
onChange={handleChange}
|
||||
placeholder="https://example.com/your-long-url"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Custom Short Link */}
|
||||
<div>
|
||||
<label htmlFor="customSlug" className="block text-sm font-medium text-gray-700">
|
||||
Custom Short Link <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<div className="flex mt-1 rounded-md shadow-sm">
|
||||
<span className="inline-flex items-center px-3 py-2 text-sm text-gray-500 border border-r-0 border-gray-300 rounded-l-md bg-gray-50">
|
||||
{formData.domain}/
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
id="customSlug"
|
||||
name="customSlug"
|
||||
value={formData.customSlug}
|
||||
onChange={handleChange}
|
||||
placeholder="custom-slug"
|
||||
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-none rounded-r-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
</div>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
Leave blank to generate a random short link
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Domain */}
|
||||
<div>
|
||||
<label htmlFor="domain" className="block text-sm font-medium text-gray-700">
|
||||
Domain <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
id="domain"
|
||||
name="domain"
|
||||
value={formData.domain}
|
||||
onChange={handleChange}
|
||||
placeholder="e.g., googleads.link"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Team Selection */}
|
||||
<div>
|
||||
<label htmlFor="teamId" className="block text-sm font-medium text-gray-700">
|
||||
Team <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<div className="mt-1">
|
||||
<TeamSelector
|
||||
value={formData.teamId}
|
||||
onChange={(teamId) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
teamId: teamId as string,
|
||||
// Clear selected project when team changes
|
||||
projectId: ''
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Project Selection */}
|
||||
<div>
|
||||
<label htmlFor="projectId" className="block text-sm font-medium text-gray-700">
|
||||
Project <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<div className="mt-1">
|
||||
<ProjectSelector
|
||||
teamId={formData.teamId}
|
||||
value={formData.projectId}
|
||||
onChange={(projectId) => {
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
projectId: projectId as string
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Description */}
|
||||
<div>
|
||||
<label htmlFor="description" className="block text-sm font-medium text-gray-700">
|
||||
Description <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<textarea
|
||||
id="description"
|
||||
name="description"
|
||||
value={formData.description}
|
||||
onChange={handleChange}
|
||||
rows={3}
|
||||
placeholder="A brief description of this link"
|
||||
className="mt-1 block w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Tags */}
|
||||
<div>
|
||||
<label htmlFor="tagInput" className="block text-sm font-medium text-gray-700">
|
||||
Tags <span className="text-gray-500">(Optional)</span>
|
||||
</label>
|
||||
<div className="flex mt-1 rounded-md shadow-sm">
|
||||
<input
|
||||
type="text"
|
||||
id="tagInput"
|
||||
value={tagInput}
|
||||
onChange={(e) => setTagInput(e.target.value)}
|
||||
onKeyDown={handleTagKeyDown}
|
||||
placeholder="Add a tag and press Enter"
|
||||
className="flex-1 block w-full min-w-0 px-3 py-2 border border-gray-300 rounded-l-md focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={addTag}
|
||||
className="inline-flex items-center px-3 py-2 text-sm font-medium text-white border border-transparent rounded-r-md shadow-sm bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
Add
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{formData.tags && formData.tags.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2 mt-2">
|
||||
{formData.tags.map(tag => (
|
||||
<span key={tag} className="inline-flex items-center px-2 py-0.5 text-xs font-medium bg-blue-100 rounded-full text-blue-800">
|
||||
{tag}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeTag(tag)}
|
||||
className="flex-shrink-0 ml-1 text-blue-500 rounded-full hover:text-blue-700 focus:outline-none"
|
||||
>
|
||||
<span className="sr-only">Remove tag {tag}</span>
|
||||
<svg className="w-3 h-3" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Submit Button */}
|
||||
<div className="flex justify-end pt-4 border-t border-gray-200">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => router.back()}
|
||||
className="px-4 py-2 text-sm font-medium text-gray-700 bg-white border border-gray-300 rounded-md shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 mr-3"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isSubmitting}
|
||||
className="inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-blue-600 border border-transparent rounded-md shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
{isSubmitting ? (
|
||||
<>
|
||||
<svg className="w-5 h-5 mr-2 -ml-1 animate-spin" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
||||
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
Processing...
|
||||
</>
|
||||
) : 'Create Short URL'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { Loader2, ExternalLink, Search } from 'lucide-react';
|
||||
import { TeamSelector } from '@/app/components/ui/TeamSelector';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useShortUrlStore, ShortUrlData } from '@/app/utils/store';
|
||||
import ClientRouteGuard from '@/app/components/ClientRouteGuard';
|
||||
|
||||
// Define attribute type to avoid using 'any'
|
||||
interface LinkAttributes {
|
||||
@@ -102,6 +103,14 @@ const convertClickHouseToShortLink = (data: Record<string, unknown>): ShortLink
|
||||
};
|
||||
|
||||
export default function LinksPage() {
|
||||
return (
|
||||
<ClientRouteGuard>
|
||||
<LinksPageContent />
|
||||
</ClientRouteGuard>
|
||||
);
|
||||
}
|
||||
|
||||
function LinksPageContent() {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [links, setLinks] = useState<ShortLink[]>([]);
|
||||
|
||||
@@ -21,7 +21,7 @@ function MessageHandler({ setMessage }: { setMessage: (message: { type: string,
|
||||
|
||||
export default function LoginPage() {
|
||||
const router = useRouter();
|
||||
const { signIn, user } = useAuth();
|
||||
const { signIn, signInWithGoogle, user } = useAuth();
|
||||
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
@@ -67,6 +67,28 @@ export default function LoginPage() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleGoogleSignIn = async () => {
|
||||
try {
|
||||
setIsLoading(true);
|
||||
setMessage({ type: '', content: '' });
|
||||
|
||||
const { error } = await signInWithGoogle();
|
||||
|
||||
if (error) {
|
||||
throw new Error(error.message);
|
||||
}
|
||||
|
||||
// Google OAuth will handle the redirect
|
||||
} catch (error) {
|
||||
console.error('Google login error:', error);
|
||||
setMessage({
|
||||
type: 'error',
|
||||
content: error instanceof Error ? error.message : 'Failed to sign in with Google'
|
||||
});
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-center min-h-screen bg-gray-100">
|
||||
{/* Wrap the component using useSearchParams in Suspense */}
|
||||
@@ -101,7 +123,31 @@ export default function LoginPage() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form onSubmit={handleEmailSignIn} className="mt-8 space-y-6">
|
||||
{/* Google Sign In Button */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleGoogleSignIn}
|
||||
disabled={isLoading}
|
||||
className="w-full flex items-center justify-center py-2 px-4 border border-gray-300 rounded-md shadow-sm text-sm font-medium text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
<svg className="h-5 w-5 mr-2" viewBox="0 0 24 24" width="24" height="24">
|
||||
<g transform="matrix(1, 0, 0, 1, 27.009001, -39.238998)">
|
||||
<path fill="#4285F4" d="M -3.264 51.509 C -3.264 50.719 -3.334 49.969 -3.454 49.239 L -14.754 49.239 L -14.754 53.749 L -8.284 53.749 C -8.574 55.229 -9.424 56.479 -10.684 57.329 L -10.684 60.329 L -6.824 60.329 C -4.564 58.239 -3.264 55.159 -3.264 51.509 Z"/>
|
||||
<path fill="#34A853" d="M -14.754 63.239 C -11.514 63.239 -8.804 62.159 -6.824 60.329 L -10.684 57.329 C -11.764 58.049 -13.134 58.489 -14.754 58.489 C -17.884 58.489 -20.534 56.379 -21.484 53.529 L -25.464 53.529 L -25.464 56.619 C -23.494 60.539 -19.444 63.239 -14.754 63.239 Z"/>
|
||||
<path fill="#FBBC05" d="M -21.484 53.529 C -21.734 52.809 -21.864 52.039 -21.864 51.239 C -21.864 50.439 -21.724 49.669 -21.484 48.949 L -21.484 45.859 L -25.464 45.859 C -26.284 47.479 -26.754 49.299 -26.754 51.239 C -26.754 53.179 -26.284 54.999 -25.464 56.619 L -21.484 53.529 Z"/>
|
||||
<path fill="#EA4335" d="M -14.754 43.989 C -12.984 43.989 -11.404 44.599 -10.154 45.789 L -6.734 42.369 C -8.804 40.429 -11.514 39.239 -14.754 39.239 C -19.444 39.239 -23.494 41.939 -25.464 45.859 L -21.484 48.949 C -20.534 46.099 -17.884 43.989 -14.754 43.989 Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
{isLoading ? 'Signing in...' : 'Sign in with Google'}
|
||||
</button>
|
||||
|
||||
<div className="mt-6 flex items-center justify-center">
|
||||
<div className="border-t border-gray-300 flex-grow mr-3"></div>
|
||||
<span className="text-sm text-gray-500">or</span>
|
||||
<div className="border-t border-gray-300 flex-grow ml-3"></div>
|
||||
</div>
|
||||
|
||||
<form onSubmit={handleEmailSignIn} className="mt-6 space-y-6">
|
||||
<div>
|
||||
<label htmlFor="email" className="block text-sm font-medium text-gray-700">
|
||||
Email address
|
||||
@@ -144,7 +190,7 @@ export default function LoginPage() {
|
||||
disabled={isLoading}
|
||||
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
{isLoading ? 'Signing in...' : 'Sign in'}
|
||||
{isLoading ? 'Signing in...' : 'Sign in with Email'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
@@ -12,44 +12,44 @@ export default function RegisterPage() {
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const { signUp, signInWithGoogle } = useAuth();
|
||||
|
||||
// 处理注册表单提交
|
||||
// Handle registration form submission
|
||||
const handleSubmit = async (e: FormEvent) => {
|
||||
e.preventDefault();
|
||||
setError(null);
|
||||
|
||||
// 验证密码
|
||||
// Validate passwords
|
||||
if (password !== confirmPassword) {
|
||||
setError('两次输入的密码不一致');
|
||||
setError('Passwords do not match');
|
||||
return;
|
||||
}
|
||||
|
||||
// 密码强度验证
|
||||
// Password strength validation
|
||||
if (password.length < 6) {
|
||||
setError('密码长度至少为6个字符');
|
||||
setError('Password must be at least 6 characters');
|
||||
return;
|
||||
}
|
||||
|
||||
setIsLoading(true);
|
||||
try {
|
||||
await signUp(email, password);
|
||||
// 注册成功后会跳转到登录页面，提示用户验证邮箱
|
||||
// After successful registration, redirect to login page with email verification prompt
|
||||
} catch (error) {
|
||||
console.error('Registration error:', error);
|
||||
setError('注册失败，请稍后再试或使用其他邮箱');
|
||||
setError('Registration failed. Please try again later or use a different email');
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
// 处理Google注册/登录
|
||||
// Handle Google registration/login
|
||||
const handleGoogleSignIn = async () => {
|
||||
setError(null);
|
||||
try {
|
||||
await signInWithGoogle();
|
||||
// 登录流程会重定向到Google，然后回到应用
|
||||
// Login flow will redirect to Google and then back to the application
|
||||
} catch (error) {
|
||||
console.error('Google sign in error:', error);
|
||||
setError('Google登录失败，请稍后再试');
|
||||
setError('Google login failed. Please try again later');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -57,13 +57,13 @@ export default function RegisterPage() {
|
||||
<div className="flex items-center justify-center min-h-screen bg-gray-100 dark:bg-gray-900">
|
||||
<div className="w-full max-w-md p-8 space-y-8 bg-white dark:bg-gray-800 rounded-lg shadow-md">
|
||||
<div className="text-center">
|
||||
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">注册</h1>
|
||||
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">Register</h1>
|
||||
<p className="mt-2 text-sm text-gray-600 dark:text-gray-400">
|
||||
创建您的帐户以访问分析仪表板
|
||||
Create your account to access the analytics dashboard
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* 错误提示 */}
|
||||
{/* Error message */}
|
||||
{error && (
|
||||
<div className="p-4 mb-4 text-sm text-red-700 bg-red-100 dark:bg-red-900 dark:text-red-200 rounded-lg">
|
||||
{error}
|
||||
@@ -74,7 +74,7 @@ export default function RegisterPage() {
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<label htmlFor="email" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
邮箱地址
|
||||
Email Address
|
||||
</label>
|
||||
<input
|
||||
id="email"
|
||||
@@ -90,7 +90,7 @@ export default function RegisterPage() {
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="password" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
密码
|
||||
Password
|
||||
</label>
|
||||
<input
|
||||
id="password"
|
||||
@@ -106,7 +106,7 @@ export default function RegisterPage() {
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="confirmPassword" className="block text-sm font-medium text-gray-700 dark:text-gray-300">
|
||||
确认密码
|
||||
Confirm Password
|
||||
</label>
|
||||
<input
|
||||
id="confirmPassword"
|
||||
@@ -128,7 +128,7 @@ export default function RegisterPage() {
|
||||
disabled={isLoading}
|
||||
className="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||
>
|
||||
{isLoading ? '注册中...' : '注册'}
|
||||
{isLoading ? 'Registering...' : 'Register'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@@ -137,7 +137,7 @@ export default function RegisterPage() {
|
||||
<div className="w-full border-t border-gray-300 dark:border-gray-600"></div>
|
||||
</div>
|
||||
<div className="relative flex justify-center text-sm">
|
||||
<span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">或</span>
|
||||
<span className="px-2 bg-white dark:bg-gray-800 text-gray-500 dark:text-gray-400">or</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -173,19 +173,19 @@ export default function RegisterPage() {
|
||||
/>
|
||||
</g>
|
||||
</svg>
|
||||
使用Google账号注册
|
||||
Sign up with Google
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div className="mt-6 text-center">
|
||||
<p className="text-sm text-gray-600 dark:text-gray-400">
|
||||
已有账号？{' '}
|
||||
Already have an account?{' '}
|
||||
<Link
|
||||
href="/login"
|
||||
className="font-medium text-blue-600 hover:text-blue-500 dark:text-blue-400 dark:hover:text-blue-300"
|
||||
>
|
||||
登录
|
||||
Log in
|
||||
</Link>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
50
lib/api.ts
Normal file
50
lib/api.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import supabase from './supabase';
|
||||
|
||||
// Define response type for API
|
||||
export interface ApiResponse<T = unknown> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
// Common function for authenticated API requests to LIMQ
|
||||
export async function limqRequest<T = unknown>(
|
||||
endpoint: string,
|
||||
method: 'GET' | 'POST' | 'PUT' | 'DELETE' = 'GET',
|
||||
data?: Record<string, unknown>
|
||||
): Promise<ApiResponse<T>> {
|
||||
// Get current session
|
||||
const { data: { session } } = await supabase.auth.getSession();
|
||||
|
||||
if (!session) {
|
||||
throw new Error('No active session. User must be authenticated.');
|
||||
}
|
||||
|
||||
const baseUrl = process.env.NEXT_PUBLIC_LIMQ_API;
|
||||
const url = `${baseUrl}${endpoint.startsWith('/') ? endpoint : '/' + endpoint}`;
|
||||
|
||||
const options: RequestInit = {
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${session.access_token}`
|
||||
},
|
||||
mode: 'cors'
|
||||
};
|
||||
|
||||
if (data && (method === 'POST' || method === 'PUT')) {
|
||||
options.body = JSON.stringify(data);
|
||||
}
|
||||
|
||||
const response = await fetch(url, options);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => null);
|
||||
throw new Error(
|
||||
errorData?.error || `Request failed with status ${response.status}`
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
60
lib/auth.tsx
60
lib/auth.tsx
@@ -4,6 +4,7 @@ import React, { createContext, useContext, useEffect, useState } from 'react';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { Session, User } from '@supabase/supabase-js';
|
||||
import supabase from './supabase';
|
||||
import { limqRequest } from './api';
|
||||
|
||||
// 定义用户类型
|
||||
export type AuthUser = User | null;
|
||||
@@ -13,21 +14,16 @@ export type AuthContextType = {
|
||||
user: AuthUser;
|
||||
session: Session | null;
|
||||
isLoading: boolean;
|
||||
signIn: (email: string, password: string) => Promise<{ error?: any }>;
|
||||
signInWithGoogle: () => Promise<{ error?: any }>;
|
||||
signInWithGitHub: () => Promise<{ error?: any }>;
|
||||
signIn: (email: string, password: string) => Promise<{ error?: unknown }>;
|
||||
signInWithGoogle: () => Promise<{ error?: unknown }>;
|
||||
signInWithGitHub: () => Promise<{ error?: unknown }>;
|
||||
signUp: (email: string, password: string) => Promise<void>;
|
||||
signOut: () => Promise<void>;
|
||||
autoRegisterTestUser: () => Promise<void>; // 添加自动注册测试用户函数
|
||||
};
|
||||
|
||||
// 创建验证上下文
|
||||
const AuthContext = createContext<AuthContextType | undefined>(undefined);
|
||||
|
||||
// 测试账户常量 - 使用已验证的账户
|
||||
const TEST_EMAIL = 'vitalitymailg@gmail.com';
|
||||
const TEST_PASSWORD = 'password123';
|
||||
|
||||
// 验证提供者组件
|
||||
export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
|
||||
const [user, setUser] = useState<AuthUser>(null);
|
||||
@@ -90,7 +86,8 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
|
||||
setSession(data.session);
|
||||
setUser(data.user);
|
||||
router.push('/dashboard');
|
||||
|
||||
router.push('/analytics');
|
||||
return {};
|
||||
} catch (error) {
|
||||
console.error('登录过程出错:', error);
|
||||
@@ -104,11 +101,14 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
const signInWithGoogle = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL，如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase登录Google
|
||||
const { error } = await supabase.auth.signInWithOAuth({
|
||||
provider: 'google',
|
||||
options: {
|
||||
redirectTo: `${window.location.origin}/auth/callback`,
|
||||
redirectTo: `${siteUrl}/auth/callback`,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -130,11 +130,14 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
const signInWithGitHub = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL，如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase登录GitHub
|
||||
const { error } = await supabase.auth.signInWithOAuth({
|
||||
provider: 'github',
|
||||
options: {
|
||||
redirectTo: `${window.location.origin}/auth/callback`,
|
||||
redirectTo: `${siteUrl}/auth/callback`,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -156,12 +159,15 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
const signUp = async (email: string, password: string) => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
// 获取网站 URL，如果环境变量不存在则使用当前来源
|
||||
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || window.location.origin;
|
||||
|
||||
// 尝试通过Supabase注册
|
||||
const { error } = await supabase.auth.signUp({
|
||||
email,
|
||||
password,
|
||||
options: {
|
||||
emailRedirectTo: `${window.location.origin}/auth/callback`,
|
||||
emailRedirectTo: `${siteUrl}/auth/callback`,
|
||||
}
|
||||
});
|
||||
|
||||
@@ -201,35 +207,6 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
}
|
||||
};
|
||||
|
||||
// 自动注册测试用户函数
|
||||
const autoRegisterTestUser = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
console.log('正在使用测试账户登录:', TEST_EMAIL);
|
||||
|
||||
// 使用测试账户直接登录
|
||||
const { data, error } = await supabase.auth.signInWithPassword({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('测试账户登录失败:', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
console.log('测试账户登录成功!');
|
||||
setSession(data.session);
|
||||
setUser(data.user);
|
||||
router.push('/dashboard');
|
||||
} catch (error) {
|
||||
console.error('测试账户登录出错:', error);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const contextValue: AuthContextType = {
|
||||
user,
|
||||
session,
|
||||
@@ -239,7 +216,6 @@ export const AuthProvider: React.FC<{ children: React.ReactNode }> = ({ children
|
||||
signInWithGitHub,
|
||||
signUp,
|
||||
signOut,
|
||||
autoRegisterTestUser,
|
||||
};
|
||||
|
||||
return (
|
||||
|
||||
22
middleware.ts
Normal file
22
middleware.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { createMiddlewareClient } from '@supabase/auth-helpers-nextjs';
|
||||
import { NextResponse } from 'next/server';
|
||||
import type { NextRequest } from 'next/server';
|
||||
|
||||
export async function middleware(req: NextRequest) {
|
||||
const res = NextResponse.next();
|
||||
|
||||
// Create a Supabase client configured to use cookies
|
||||
const supabase = createMiddlewareClient({ req, res });
|
||||
|
||||
// Refresh session if expired - required for Server Components
|
||||
await supabase.auth.getSession();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// Specify the paths where this middleware should run
|
||||
export const config = {
|
||||
matcher: [
|
||||
'/((?!_next/static|_next/image|favicon.ico).*)',
|
||||
],
|
||||
};
|
||||
2
windmill/scripts/.gitignore
vendored
Normal file
2
windmill/scripts/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
/node_modules
|
||||
/package-lock.json
|
||||
19
windmill/scripts/package.json
Normal file
19
windmill/scripts/package.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "scripts",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"date-fns": "^4.1.0",
|
||||
"dotenv": "^16.5.0",
|
||||
"fs-extra": "^11.3.0",
|
||||
"mongodb": "^6.16.0",
|
||||
"node-fetch": "^2.7.0"
|
||||
}
|
||||
}
|
||||
714
windmill/scripts/sync_mongo_to_clickhouse.js
Normal file
714
windmill/scripts/sync_mongo_to_clickhouse.js
Normal file
@@ -0,0 +1,714 @@
|
||||
// 从MongoDB的trace表同步数据到ClickHouse的events表
|
||||
//
|
||||
// 支持以下同步模式：
|
||||
// 1. 增量同步：基于上次同步状态，只同步新数据（默认模式）
|
||||
// 2. 自定义时间范围同步：通过指定开始时间和结束时间，同步特定时间范围内的数据
|
||||
// - 可以通过时间戳参数(startTime/endTime)指定范围
|
||||
// - 也可以通过日期字符串参数(startDate/endDate)指定范围，支持ISO格式或yyyy-MM-dd格式
|
||||
|
||||
const { MongoClient, ObjectId } = require('mongodb');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
// 同步状态键名和保存路径
|
||||
const SYNC_STATE_FILE = path.join(__dirname, 'mongo_sync_state.json');
|
||||
|
||||
// 直接使用配置值
|
||||
const mongoConfig = {
|
||||
url: "mongodb://10.0.1.41:27017",
|
||||
db: "main" // 注意：请替换为您的实际数据库名称
|
||||
};
|
||||
|
||||
const clickhouseConfig = {
|
||||
clickhouse_host: "10.0.1.60",
|
||||
clickhouse_port: "8123",
|
||||
clickhouse_user: "admin",
|
||||
clickhouse_password: "your_secure_password",
|
||||
clickhouse_database: "shorturl_analytics",
|
||||
clickhouse_url: "http://10.0.1.60:8123"
|
||||
};
|
||||
|
||||
// 封装本地读取变量函数
|
||||
async function getVariable(key) {
|
||||
try {
|
||||
if (key === 'f/shorturl_analytics/mongodb') {
|
||||
return mongoConfig;
|
||||
} else if (key === 'f/shorturl_analytics/clickhouse') {
|
||||
return clickhouseConfig;
|
||||
} else if (key === 'f/shorturl_analytics/mongo_sync_state') {
|
||||
if (fs.existsSync(SYNC_STATE_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(SYNC_STATE_FILE, 'utf8'));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error(`获取变量失败: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 封装本地保存变量函数
|
||||
async function setVariable(key, value) {
|
||||
try {
|
||||
if (key === 'f/shorturl_analytics/mongo_sync_state') {
|
||||
fs.writeFileSync(SYNC_STATE_FILE, JSON.stringify(value, null, 2));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`保存变量失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 日期字符串转时间戳工具函数（接受ISO字符串或yyyy-MM-dd格式）
|
||||
function dateToTimestamp(dateStr) {
|
||||
try {
|
||||
// 尝试直接解析完整的ISO日期字符串
|
||||
const date = new Date(dateStr);
|
||||
|
||||
// 检查是否为有效日期
|
||||
if (isNaN(date.getTime())) {
|
||||
// 尝试解析yyyy-MM-dd格式，默认设置为当天的00:00:00
|
||||
const parts = dateStr.split('-');
|
||||
if (parts.length === 3) {
|
||||
const year = parseInt(parts[0], 10);
|
||||
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
|
||||
const day = parseInt(parts[2], 10);
|
||||
|
||||
const dateObj = new Date(year, month, day, 0, 0, 0);
|
||||
return dateObj.getTime();
|
||||
}
|
||||
throw new Error(`无法解析日期字符串: ${dateStr}`);
|
||||
}
|
||||
|
||||
return date.getTime();
|
||||
} catch (err) {
|
||||
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从URL中提取UTM参数的函数，增强版
|
||||
function extractUtmParams(url, debug = false) {
|
||||
const defaultUtmParams = {
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
utm_term: "",
|
||||
utm_content: ""
|
||||
};
|
||||
|
||||
if (!url) return defaultUtmParams;
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 原始URL: ${url}`);
|
||||
}
|
||||
|
||||
// 准备一个解析后的参数对象
|
||||
const params = { ...defaultUtmParams };
|
||||
|
||||
// 尝试多种方法提取UTM参数
|
||||
|
||||
// 方法1: 使用URL对象解析
|
||||
try {
|
||||
// 先处理URL，确保是完整的URL格式
|
||||
let normalizedUrl = url;
|
||||
if (!url.match(/^https?:\/\//i)) {
|
||||
normalizedUrl = `https://example.com${url.startsWith('/') ? '' : '/'}${url}`;
|
||||
}
|
||||
|
||||
const urlObj = new URL(normalizedUrl);
|
||||
|
||||
// 读取URL参数
|
||||
if (urlObj.searchParams.has('utm_source'))
|
||||
params.utm_source = urlObj.searchParams.get('utm_source') || "";
|
||||
if (urlObj.searchParams.has('utm_medium'))
|
||||
params.utm_medium = urlObj.searchParams.get('utm_medium') || "";
|
||||
if (urlObj.searchParams.has('utm_campaign'))
|
||||
params.utm_campaign = urlObj.searchParams.get('utm_campaign') || "";
|
||||
if (urlObj.searchParams.has('utm_term'))
|
||||
params.utm_term = urlObj.searchParams.get('utm_term') || "";
|
||||
if (urlObj.searchParams.has('utm_content'))
|
||||
params.utm_content = urlObj.searchParams.get('utm_content') || "";
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
// 如果至少找到一个UTM参数，则返回
|
||||
if (params.utm_source || params.utm_medium || params.utm_campaign ||
|
||||
params.utm_term || params.utm_content) {
|
||||
return params;
|
||||
}
|
||||
} catch (err) {
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] URL对象解析失败，尝试正则表达式`);
|
||||
}
|
||||
}
|
||||
|
||||
// 方法2: 使用正则表达式提取参数
|
||||
// 使用正则表达式（最安全的方法，适用于任何格式）
|
||||
const sourceMatch = url.match(/[?&]utm_source=([^&#]+)/i);
|
||||
if (sourceMatch && sourceMatch[1]) {
|
||||
try {
|
||||
params.utm_source = decodeURIComponent(sourceMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_source = sourceMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const mediumMatch = url.match(/[?&]utm_medium=([^&#]+)/i);
|
||||
if (mediumMatch && mediumMatch[1]) {
|
||||
try {
|
||||
params.utm_medium = decodeURIComponent(mediumMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_medium = mediumMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const campaignMatch = url.match(/[?&]utm_campaign=([^&#]+)/i);
|
||||
if (campaignMatch && campaignMatch[1]) {
|
||||
try {
|
||||
params.utm_campaign = decodeURIComponent(campaignMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_campaign = campaignMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const termMatch = url.match(/[?&]utm_term=([^&#]+)/i);
|
||||
if (termMatch && termMatch[1]) {
|
||||
try {
|
||||
params.utm_term = decodeURIComponent(termMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_term = termMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
const contentMatch = url.match(/[?&]utm_content=([^&#]+)/i);
|
||||
if (contentMatch && contentMatch[1]) {
|
||||
try {
|
||||
params.utm_content = decodeURIComponent(contentMatch[1]);
|
||||
} catch (err) {
|
||||
params.utm_content = contentMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.log(`[UTM提取] 正则表达式解析结果: ${JSON.stringify(params)}`);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
// 解析命令行参数
|
||||
function parseCommandLineArgs() {
|
||||
const args = {};
|
||||
process.argv.slice(2).forEach(arg => {
|
||||
if (arg.startsWith('--')) {
|
||||
const [key, value] = arg.substring(2).split('=');
|
||||
args[key] = value || true;
|
||||
}
|
||||
});
|
||||
return args;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseCommandLineArgs();
|
||||
|
||||
// 参数设置
|
||||
const batch_size = parseInt(args['batch-size'] || '1000');
|
||||
const max_records = parseInt(args['max-records'] || '9999999');
|
||||
const timeout_minutes = parseInt(args['timeout'] || '60');
|
||||
const skip_clickhouse_check = args['skip-clickhouse-check'] === 'true';
|
||||
const force_insert = args['force-insert'] !== 'false';
|
||||
const database_override = args['database'] || 'shorturl_analytics';
|
||||
const reset_sync_state = args['reset-sync-state'] === 'true';
|
||||
const debug_utm = args['debug-utm'] === 'true';
|
||||
const start_time = args['start-time'] ? parseInt(args['start-time']) : undefined;
|
||||
const end_time = args['end-time'] ? parseInt(args['end-time']) : undefined;
|
||||
const use_custom_time_range = args['use-custom-time-range'] === 'true';
|
||||
const start_date = args['start-date'];
|
||||
const end_date = args['end-date'];
|
||||
|
||||
const logWithTimestamp = (message) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
|
||||
let customStartTime = start_time;
|
||||
let customEndTime = end_time;
|
||||
let useCustomTimeRange = use_custom_time_range;
|
||||
|
||||
// 处理日期字符串参数，转换为时间戳
|
||||
if (start_date) {
|
||||
try {
|
||||
customStartTime = dateToTimestamp(start_date);
|
||||
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${customStartTime}`);
|
||||
useCustomTimeRange = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (end_date) {
|
||||
try {
|
||||
customEndTime = dateToTimestamp(end_date);
|
||||
// 如果是日期格式，设置为当天结束时间 (23:59:59.999)
|
||||
if (end_date.split('-').length === 3 && end_date.length <= 10) {
|
||||
customEndTime += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${customEndTime}`);
|
||||
} else {
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${customEndTime}`);
|
||||
}
|
||||
useCustomTimeRange = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
|
||||
}
|
||||
if (force_insert) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
|
||||
}
|
||||
if (reset_sync_state) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用重置同步状态，将从头开始同步数据");
|
||||
}
|
||||
if (debug_utm) {
|
||||
logWithTimestamp("已启用UTM参数调试日志");
|
||||
}
|
||||
if (useCustomTimeRange) {
|
||||
if (customStartTime) {
|
||||
logWithTimestamp(`已启用自定义时间范围：开始时间 ${new Date(customStartTime).toISOString()}`);
|
||||
}
|
||||
if (customEndTime) {
|
||||
logWithTimestamp(`已启用自定义时间范围：结束时间 ${new Date(customEndTime).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
// 检查是否超时
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`运行时间超过${timeout_minutes}分钟，暂停执行`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// 获取上次同步状态
|
||||
let lastSyncState = null;
|
||||
if (!reset_sync_state) {
|
||||
try {
|
||||
const rawSyncState = await getVariable("f/shorturl_analytics/mongo_sync_state");
|
||||
if (rawSyncState) {
|
||||
lastSyncState = rawSyncState;
|
||||
}
|
||||
} catch (error) {
|
||||
logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
|
||||
}
|
||||
}
|
||||
|
||||
if (lastSyncState) {
|
||||
logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
|
||||
if (lastSyncState.last_sync_id) {
|
||||
logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("没有找到上次同步状态，将从头开始同步");
|
||||
}
|
||||
|
||||
// 连接MongoDB
|
||||
const client = new MongoClient(mongoConfig.url);
|
||||
try {
|
||||
await client.connect();
|
||||
console.log("MongoDB连接成功");
|
||||
|
||||
const db = client.db(mongoConfig.db);
|
||||
const traceCollection = db.collection("trace");
|
||||
const shortCollection = db.collection("short");
|
||||
|
||||
// 构建查询条件
|
||||
const query = {
|
||||
type: 1 // 只同步type为1的记录
|
||||
};
|
||||
|
||||
// 根据时间范围参数构建查询条件
|
||||
if (useCustomTimeRange) {
|
||||
// 使用自定义时间范围
|
||||
const timeQuery = {};
|
||||
|
||||
if (customStartTime) {
|
||||
timeQuery.$gte = customStartTime;
|
||||
logWithTimestamp(`将只同步createTime >= ${customStartTime} (${new Date(customStartTime).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
if (customEndTime) {
|
||||
timeQuery.$lte = customEndTime;
|
||||
logWithTimestamp(`将只同步createTime <= ${customEndTime} (${new Date(customEndTime).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 只有当至少指定了一个时间限制时才添加时间查询条件
|
||||
if (Object.keys(timeQuery).length > 0) {
|
||||
query.createTime = timeQuery;
|
||||
}
|
||||
}
|
||||
// 如果不使用自定义时间范围，且有上次同步状态，则只获取更新的记录
|
||||
else if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 使用上次同步时间作为过滤条件
|
||||
query.createTime = { $gt: lastSyncState.last_sync_time };
|
||||
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 计算总记录数
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`找到 ${totalRecords} 条新记录需要同步`);
|
||||
|
||||
// 限制此次处理的记录数量
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`本次将处理 ${recordsToProcess} 条记录`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("没有新记录需要同步，任务完成");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "没有新记录需要同步"
|
||||
};
|
||||
}
|
||||
|
||||
// 检查ClickHouse连接状态
|
||||
const checkClickHouseConnection = async () => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，不测试连接");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("测试ClickHouse连接...");
|
||||
const clickhouseUrl = clickhouseConfig.clickhouse_url;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`,
|
||||
},
|
||||
body: `SELECT 1 FROM ${clickhouseConfig.clickhouse_database}.events LIMIT 1`,
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse连接测试成功");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ ClickHouse连接测试失败，请启用skip_clickhouse_check=true参数来跳过连接检查");
|
||||
throw new Error("ClickHouse连接失败，无法继续同步");
|
||||
}
|
||||
|
||||
// 处理记录的函数
|
||||
const processRecords = async (records) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
|
||||
|
||||
// 强制使用所有记录，不检查重复
|
||||
const newRecords = records;
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
|
||||
|
||||
// 获取链接信息
|
||||
const slugIds = newRecords.map(record => new ObjectId(record.slugId));
|
||||
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
|
||||
const shortLinks = await shortCollection.find({
|
||||
_id: { $in: slugIds }
|
||||
}).toArray();
|
||||
|
||||
// 创建映射用于快速查找
|
||||
const shortLinksMap = new Map(shortLinks.map((link) => [link._id.toString(), link]));
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息，${newRecords.length - shortLinks.length} 条数据将使用占位符`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const eventTime = new Date(record.createTime);
|
||||
|
||||
// 获取对应的短链接信息
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString());
|
||||
|
||||
// 提取URL中的UTM参数
|
||||
if (debug_utm && record.url) {
|
||||
logWithTimestamp(`======== UTM参数调试 ========`);
|
||||
logWithTimestamp(`记录ID: ${record._id.toString()}`);
|
||||
logWithTimestamp(`原始URL: ${record.url}`);
|
||||
}
|
||||
|
||||
const utmParams = extractUtmParams(record.url || "", debug_utm);
|
||||
|
||||
if (debug_utm) {
|
||||
logWithTimestamp(`提取的UTM参数: ${JSON.stringify(utmParams)}`);
|
||||
logWithTimestamp(`===========================`);
|
||||
}
|
||||
|
||||
// 保存提取的UTM参数和URL到event_attributes
|
||||
const eventAttributes = {
|
||||
mongo_id: record._id.toString(),
|
||||
url: record.url || "",
|
||||
...(record.url ? { raw_url: record.url } : {})
|
||||
};
|
||||
|
||||
// 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
|
||||
return {
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
event_type: record.type === 1 ? "visit" : "custom",
|
||||
event_attributes: JSON.stringify(eventAttributes),
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
|
||||
link_label: record.label || "",
|
||||
link_title: shortLink?.title || "unknown_title", // 使用占位符
|
||||
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
|
||||
link_created_at: shortLink?.createTime
|
||||
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
link_expires_at: shortLink?.expiresAt
|
||||
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: null,
|
||||
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
|
||||
user_id: shortLink?.user || "unknown_user", // 使用占位符
|
||||
user_name: "unknown_user", // 使用占位符
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
team_id: shortLink?.teamId || "unknown_team", // 使用占位符
|
||||
team_name: "unknown_team", // 使用占位符
|
||||
team_attributes: "{}",
|
||||
project_id: shortLink?.projectId || "unknown_project", // 使用占位符
|
||||
project_name: "unknown_project", // 使用占位符
|
||||
project_attributes: "{}",
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: record._id.toString() + "-" + record.createTime,
|
||||
ip_address: record.ip || "0.0.0.0", // 使用占位符
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "unknown", // 使用占位符
|
||||
os: record.platformOS || "unknown", // 使用占位符
|
||||
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
|
||||
referrer: record.url || "",
|
||||
utm_source: utmParams.utm_source || "",
|
||||
utm_medium: utmParams.utm_medium || "",
|
||||
utm_campaign: utmParams.utm_campaign || "",
|
||||
utm_term: utmParams.utm_term || "",
|
||||
utm_content: utmParams.utm_content || "",
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
conversion_type: "visit",
|
||||
conversion_value: 0,
|
||||
req_full_path: record.url || ""
|
||||
};
|
||||
});
|
||||
|
||||
// 生成ClickHouse插入SQL
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
|
||||
(event_time, event_type, event_attributes, link_id, link_slug, link_label, link_title,
|
||||
link_original_url, link_attributes, link_created_at, link_expires_at, link_tags,
|
||||
user_id, user_name, user_email, user_attributes, team_id, team_name, team_attributes,
|
||||
project_id, project_name, project_attributes, qr_code_id, qr_code_name, qr_code_attributes,
|
||||
visitor_id, session_id, ip_address, country, city, device_type, browser, os, user_agent,
|
||||
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
|
||||
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
|
||||
VALUES ${clickhouseData.map(record => {
|
||||
// 确保所有字符串值都是字符串类型，并安全处理替换
|
||||
const safeReplace = (val) => {
|
||||
// 确保值是字符串，如果是null或undefined则使用空字符串
|
||||
const str = val === null || val === undefined ? "" : String(val);
|
||||
// 安全替换单引号
|
||||
return str.replace(/'/g, "''");
|
||||
};
|
||||
|
||||
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
|
||||
'${record.link_id}', '${safeReplace(record.link_slug)}', '${safeReplace(record.link_label)}', '${safeReplace(record.link_title)}',
|
||||
'${safeReplace(record.link_original_url)}', '${safeReplace(record.link_attributes)}', '${record.link_created_at}',
|
||||
${record.link_expires_at === null ? 'NULL' : `'${record.link_expires_at}'`}, '${safeReplace(record.link_tags)}',
|
||||
'${safeReplace(record.user_id)}', '${safeReplace(record.user_name)}', '${safeReplace(record.user_email)}',
|
||||
'${safeReplace(record.user_attributes)}', '${safeReplace(record.team_id)}', '${safeReplace(record.team_name)}',
|
||||
'${safeReplace(record.team_attributes)}', '${safeReplace(record.project_id)}', '${safeReplace(record.project_name)}',
|
||||
'${safeReplace(record.project_attributes)}', '${safeReplace(record.qr_code_id)}', '${safeReplace(record.qr_code_name)}',
|
||||
'${safeReplace(record.qr_code_attributes)}', '${safeReplace(record.visitor_id)}', '${safeReplace(record.session_id)}',
|
||||
'${safeReplace(record.ip_address)}', '${safeReplace(record.country)}', '${safeReplace(record.city)}',
|
||||
'${safeReplace(record.device_type)}', '${safeReplace(record.browser)}', '${safeReplace(record.os)}',
|
||||
'${safeReplace(record.user_agent)}', '${safeReplace(record.referrer)}', '${safeReplace(record.utm_source)}',
|
||||
'${safeReplace(record.utm_medium)}', '${safeReplace(record.utm_campaign)}', '${safeReplace(record.utm_term)}',
|
||||
'${safeReplace(record.utm_content)}', ${record.time_spent_sec}, ${record.is_bounce}, ${record.is_qr_scan},
|
||||
'${safeReplace(record.conversion_type)}', ${record.conversion_value}, '${safeReplace(record.req_full_path)}')`;
|
||||
}).join(", ")}
|
||||
`;
|
||||
|
||||
if (insertSQL.length === 0) {
|
||||
console.log("没有新记录需要插入");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 发送请求到ClickHouse
|
||||
const clickhouseUrl = clickhouseConfig.clickhouse_url;
|
||||
try {
|
||||
logWithTimestamp("发送插入请求到ClickHouse...");
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${Buffer.from(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`).toString('base64')}`
|
||||
},
|
||||
body: insertSQL,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`向ClickHouse插入数据失败: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// 批量处理记录
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
let lastSyncTime = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录，因超时暂停执行`);
|
||||
break;
|
||||
}
|
||||
|
||||
// 每批次都输出进度
|
||||
logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
|
||||
|
||||
// 查询MongoDB数据
|
||||
const records = await traceCollection.find(query)
|
||||
.sort({ createTime: 1 })
|
||||
.skip(page * batch_size)
|
||||
.limit(batch_size)
|
||||
.toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("没有找到更多数据，同步结束");
|
||||
break;
|
||||
}
|
||||
|
||||
// 找到数据，开始处理
|
||||
logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
|
||||
// 输出当前批次的部分数据信息
|
||||
if (records.length > 0) {
|
||||
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
|
||||
if (records.length > 1) {
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
|
||||
}
|
||||
|
||||
// 如果开启了调试，输出一些URL样本
|
||||
if (debug_utm) {
|
||||
const sampleSize = Math.min(5, records.length);
|
||||
logWithTimestamp(`URL样本 (前${sampleSize}条):`);
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
if (records[i].url) {
|
||||
logWithTimestamp(`样本 ${i+1}: ${records[i].url}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
// 更新最后处理的记录时间和ID
|
||||
if (records.length > 0) {
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
|
||||
}
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
if (processedRecords > 0 && lastSyncTime > 0) {
|
||||
// 只在非自定义时间范围模式下更新同步状态
|
||||
if (!useCustomTimeRange) {
|
||||
// 创建新的同步状态，简化对象结构
|
||||
const newSyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords
|
||||
};
|
||||
|
||||
try {
|
||||
// 保存同步状态
|
||||
await setVariable("f/shorturl_analytics/mongo_sync_state", newSyncState);
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
logWithTimestamp(`更新同步状态失败: ${err.message}，将继续执行`);
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("使用自定义时间范围模式，不更新全局同步状态");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
|
||||
message: useCustomTimeRange ? "自定义时间范围数据同步完成" : "数据同步完成",
|
||||
custom_time_range_used: useCustomTimeRange
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("同步过程中发生错误:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err.message,
|
||||
stack: err.stack
|
||||
};
|
||||
} finally {
|
||||
// 关闭MongoDB连接
|
||||
await client.close();
|
||||
console.log("MongoDB连接已关闭");
|
||||
}
|
||||
}
|
||||
|
||||
// 执行主函数
|
||||
main().then(result => {
|
||||
console.log("任务执行结果:", result);
|
||||
process.exit(result.success ? 0 : 1);
|
||||
}).catch(err => {
|
||||
console.error("执行出错:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,4 +1,12 @@
|
||||
// 从MongoDB的trace表同步数据到ClickHouse的events表
|
||||
//
|
||||
// 支持以下同步模式：
|
||||
// 1. 增量同步：基于上次同步状态，只同步新数据（默认模式）
|
||||
// 2. 自定义时间范围同步：通过指定开始时间和结束时间，同步特定时间范围内的数据
|
||||
// - 可以通过时间戳参数(start_time/end_time)指定范围
|
||||
// - 也可以通过日期字符串参数(start_date/end_date)指定范围，支持ISO格式或yyyy-MM-dd格式
|
||||
//
|
||||
// 使用自定义时间范围时，将不会更新同步状态，避免干扰增量同步进度
|
||||
import { getVariable, setVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
@@ -68,6 +76,33 @@ interface UtmParams {
|
||||
// 同步状态键名
|
||||
const SYNC_STATE_KEY = "f/shorturl_analytics/mongo_sync_state";
|
||||
|
||||
// 日期字符串转时间戳工具函数（接受ISO字符串或yyyy-MM-dd格式）
|
||||
function dateToTimestamp(dateStr: string): number {
|
||||
try {
|
||||
// 尝试直接解析完整的ISO日期字符串
|
||||
const date = new Date(dateStr);
|
||||
|
||||
// 检查是否为有效日期
|
||||
if (isNaN(date.getTime())) {
|
||||
// 尝试解析yyyy-MM-dd格式，默认设置为当天的00:00:00
|
||||
const parts = dateStr.split('-');
|
||||
if (parts.length === 3) {
|
||||
const year = parseInt(parts[0], 10);
|
||||
const month = parseInt(parts[1], 10) - 1; // 月份从0开始
|
||||
const day = parseInt(parts[2], 10);
|
||||
|
||||
const dateObj = new Date(year, month, day, 0, 0, 0);
|
||||
return dateObj.getTime();
|
||||
}
|
||||
throw new Error(`无法解析日期字符串: ${dateStr}`);
|
||||
}
|
||||
|
||||
return date.getTime();
|
||||
} catch (err) {
|
||||
throw new Error(`日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从URL中提取UTM参数的函数，增强版
|
||||
function extractUtmParams(url: string, debug = false): UtmParams {
|
||||
const defaultUtmParams: UtmParams = {
|
||||
@@ -185,10 +220,15 @@ export async function main(
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false,
|
||||
force_insert = true,
|
||||
database_override = "shorturl_analytics", // 添加数据库名称参数，默认为shorturl_analytics
|
||||
reset_sync_state = false, // 添加参数用于重置同步状态
|
||||
debug_utm = false // 添加参数控制UTM调试日志输出
|
||||
debug_utm = false, // 添加参数控制UTM调试日志输出
|
||||
start_time?: number, // 添加参数指定同步的开始时间戳，可选
|
||||
end_time?: number, // 添加参数指定同步的结束时间戳，可选
|
||||
use_custom_time_range = false, // 添加参数控制是否使用自定义时间范围
|
||||
start_date?: string, // 添加开始日期字符串参数（ISO格式或yyyy-MM-dd格式）
|
||||
end_date?: string // 添加结束日期字符串参数（ISO格式或yyyy-MM-dd格式）
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
@@ -197,6 +237,34 @@ export async function main(
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
|
||||
// 处理日期字符串参数，转换为时间戳
|
||||
if (start_date) {
|
||||
try {
|
||||
start_time = dateToTimestamp(start_date);
|
||||
logWithTimestamp(`将开始日期 ${start_date} 转换为时间戳 ${start_time}`);
|
||||
use_custom_time_range = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`开始日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (end_date) {
|
||||
try {
|
||||
end_time = dateToTimestamp(end_date);
|
||||
// 如果是日期格式，设置为当天结束时间 (23:59:59.999)
|
||||
if (end_date.split('-').length === 3 && end_date.length <= 10) {
|
||||
end_time += 24 * 60 * 60 * 1000 - 1; // 加上23:59:59.999
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为当天结束时间戳 ${end_time}`);
|
||||
} else {
|
||||
logWithTimestamp(`将结束日期 ${end_date} 转换为时间戳 ${end_time}`);
|
||||
}
|
||||
use_custom_time_range = true;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`结束日期转换错误: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
|
||||
}
|
||||
@@ -209,6 +277,14 @@ export async function main(
|
||||
if (debug_utm) {
|
||||
logWithTimestamp("已启用UTM参数调试日志");
|
||||
}
|
||||
if (use_custom_time_range) {
|
||||
if (start_time) {
|
||||
logWithTimestamp(`已启用自定义时间范围：开始时间 ${new Date(start_time).toISOString()}`);
|
||||
}
|
||||
if (end_time) {
|
||||
logWithTimestamp(`已启用自定义时间范围：结束时间 ${new Date(end_time).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
@@ -328,11 +404,31 @@ export async function main(
|
||||
|
||||
// 构建查询条件，根据上次同步状态获取新记录
|
||||
const query: Record<string, unknown> = {
|
||||
type: 1 // 只同步type为1的记录
|
||||
// 删除了 type: 1 的条件，将同步所有数据
|
||||
};
|
||||
|
||||
// 如果有上次同步状态，则只获取更新的记录
|
||||
if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 根据时间范围参数构建查询条件
|
||||
if (use_custom_time_range) {
|
||||
// 使用自定义时间范围
|
||||
const timeQuery: Record<string, number> = {};
|
||||
|
||||
if (start_time) {
|
||||
timeQuery.$gte = start_time;
|
||||
logWithTimestamp(`将只同步createTime >= ${start_time} (${new Date(start_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
if (end_time) {
|
||||
timeQuery.$lte = end_time;
|
||||
logWithTimestamp(`将只同步createTime <= ${end_time} (${new Date(end_time).toISOString()}) 的记录`);
|
||||
}
|
||||
|
||||
// 只有当至少指定了一个时间限制时才添加时间查询条件
|
||||
if (Object.keys(timeQuery).length > 0) {
|
||||
query.createTime = timeQuery;
|
||||
}
|
||||
}
|
||||
// 如果不使用自定义时间范围，且有上次同步状态，则只获取更新的记录
|
||||
else if (lastSyncState && lastSyncState.last_sync_time) {
|
||||
// 使用上次同步时间作为过滤条件
|
||||
query.createTime = { $gt: lastSyncState.last_sync_time };
|
||||
logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
|
||||
@@ -391,104 +487,6 @@ export async function main(
|
||||
}
|
||||
};
|
||||
|
||||
// 检查记录是否已经存在于ClickHouse中
|
||||
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
// 如果跳过ClickHouse检查或强制插入，则直接返回所有记录
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`已跳过ClickHouse重复检查，准备处理所有 ${records.length} 条记录`);
|
||||
return records;
|
||||
}
|
||||
|
||||
logWithTimestamp(`正在检查 ${records.length} 条记录是否已存在于ClickHouse中...`);
|
||||
|
||||
try {
|
||||
// 验证数据库名称
|
||||
if (!clickhouseConfig.clickhouse_database || clickhouseConfig.clickhouse_database === "undefined") {
|
||||
throw new Error("数据库名称未定义或无效，请检查配置");
|
||||
}
|
||||
|
||||
// 提取所有记录的ID
|
||||
const recordIds = records.map(record => record.slugId.toString()); // 使用slugId作为link_id查询
|
||||
logWithTimestamp(`待检查的记录ID: ${recordIds.join(', ')}`);
|
||||
|
||||
// 构建查询SQL，检查记录是否已存在，确保添加FORMAT JSON来获取正确的JSON格式响应
|
||||
const query = `
|
||||
SELECT link_id, visitor_id
|
||||
FROM ${clickhouseConfig.clickhouse_database}.events
|
||||
WHERE link_id IN ('${recordIds.join("','")}')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
logWithTimestamp(`执行ClickHouse查询: ${query.replace(/\n\s*/g, ' ')}`);
|
||||
|
||||
// 发送请求到ClickHouse，添加10秒超时
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse查询错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
// 获取响应文本以便记录
|
||||
const responseText = await response.text();
|
||||
logWithTimestamp(`ClickHouse查询响应: ${responseText.slice(0, 200)}${responseText.length > 200 ? '...' : ''}`);
|
||||
|
||||
if (!responseText.trim()) {
|
||||
logWithTimestamp("ClickHouse返回空响应，假定没有记录存在");
|
||||
return records; // 如果响应为空，假设没有记录
|
||||
}
|
||||
|
||||
// 解析结果
|
||||
let result;
|
||||
try {
|
||||
result = JSON.parse(responseText);
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse响应不是有效的JSON: ${responseText}`);
|
||||
throw new Error(`解析ClickHouse响应失败: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// 确保result有正确的结构
|
||||
if (!result.data) {
|
||||
logWithTimestamp(`ClickHouse响应缺少data字段: ${JSON.stringify(result)}`);
|
||||
return records; // 如果没有data字段，假设没有记录
|
||||
}
|
||||
|
||||
// 提取已存在的记录ID
|
||||
const existingIds = new Set(result.data.map((row: { link_id: string }) => row.link_id));
|
||||
|
||||
logWithTimestamp(`检测到 ${existingIds.size} 条记录已存在于ClickHouse中`);
|
||||
if (existingIds.size > 0) {
|
||||
logWithTimestamp(`已存在的记录ID: ${Array.from(existingIds).join(', ')}`);
|
||||
}
|
||||
|
||||
// 过滤出不存在的记录
|
||||
const newRecords = records.filter(record => !existingIds.has(record.slugId.toString())); // 使用slugId匹配link_id
|
||||
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
|
||||
|
||||
return newRecords;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`ClickHouse查询出错: ${error.message}`);
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，将继续处理所有记录");
|
||||
return records;
|
||||
} else {
|
||||
throw error; // 如果没有启用跳过检查，则抛出错误
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
@@ -502,27 +500,10 @@ export async function main(
|
||||
|
||||
logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
|
||||
|
||||
// 检查记录是否已存在
|
||||
let newRecords;
|
||||
try {
|
||||
newRecords = await checkExistingRecords(records);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
|
||||
if (!skip_clickhouse_check && !force_insert) {
|
||||
throw error;
|
||||
}
|
||||
// 如果跳过检查或强制插入，则使用所有记录
|
||||
logWithTimestamp("将使用所有记录进行处理");
|
||||
newRecords = records;
|
||||
}
|
||||
// 强制使用所有记录，不检查重复
|
||||
const newRecords = records;
|
||||
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("所有记录都已存在，跳过处理");
|
||||
return 0;
|
||||
}
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条记录...`);
|
||||
|
||||
// 获取链接信息 - 新增代码
|
||||
const slugIds = newRecords.map(record => record.slugId);
|
||||
@@ -533,7 +514,7 @@ export async function main(
|
||||
|
||||
// 创建映射用于快速查找 - 新增代码
|
||||
const shortLinksMap = new Map(shortLinks.map((link: ShortRecord) => [link._id.toString(), link]));
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息`);
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息，${newRecords.length - shortLinks.length} 条数据将使用占位符`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
@@ -567,14 +548,14 @@ export async function main(
|
||||
return {
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
event_time: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
event_type: record.type === 1 ? "visit" : "custom",
|
||||
event_type: "click", // 将所有event_type都设置为click
|
||||
event_attributes: JSON.stringify(eventAttributes),
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: shortLink?.slug || "",
|
||||
link_slug: shortLink?.slug || "unknown_slug", // 使用占位符
|
||||
link_label: record.label || "",
|
||||
link_title: shortLink?.title || "",
|
||||
link_original_url: shortLink?.origin || "",
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || null }),
|
||||
link_title: shortLink?.title || "unknown_title", // 使用占位符
|
||||
link_original_url: shortLink?.origin || "https://unknown.url", // 使用占位符
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || "unknown_domain" }), // 使用占位符
|
||||
link_created_at: shortLink?.createTime
|
||||
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: eventTime.toISOString().replace('T', ' ').replace('Z', ''),
|
||||
@@ -582,34 +563,34 @@ export async function main(
|
||||
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: null,
|
||||
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]",
|
||||
user_id: shortLink?.user || "",
|
||||
user_name: "",
|
||||
user_id: "3680f452-e404-4339-a3d2-2a8e1ff92102", // 使用占位符
|
||||
user_name: "unknown_user", // 使用占位符
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
team_id: shortLink?.teamId || "",
|
||||
team_name: "",
|
||||
team_id: "e02251eb-eb98-47c8-b5dd-4f6e4fdb1f49", // 使用占位符
|
||||
team_name: "", // 使用占位符
|
||||
team_attributes: "{}",
|
||||
project_id: shortLink?.projectId || "",
|
||||
project_name: "",
|
||||
project_id: "34cdb8b9-8b8e-4033-876a-0632002ef1f9", // 使用占位符
|
||||
project_name: "", // 使用占位符
|
||||
project_attributes: "{}",
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
visitor_id: record._id.toString(), // 使用MongoDB ID作为访客ID
|
||||
session_id: record._id.toString() + "-" + record.createTime, // 创建一个唯一会话ID
|
||||
ip_address: record.ip,
|
||||
country: "", // 这些字段在MongoDB中不存在，使用默认值
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: record._id.toString() + "-" + record.createTime,
|
||||
ip_address: record.ip || "0.0.0.0", // 使用占位符
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "",
|
||||
os: record.platformOS || "",
|
||||
user_agent: record.browser + " " + record.browserVersion,
|
||||
browser: record.browser || "unknown", // 使用占位符
|
||||
os: record.platformOS || "unknown", // 使用占位符
|
||||
user_agent: (record.browser || "unknown") + " " + (record.browserVersion || "unknown"), // 使用占位符
|
||||
referrer: record.url || "",
|
||||
utm_source: utmParams.utm_source,
|
||||
utm_medium: utmParams.utm_medium,
|
||||
utm_campaign: utmParams.utm_campaign,
|
||||
utm_term: utmParams.utm_term,
|
||||
utm_content: utmParams.utm_content,
|
||||
utm_source: utmParams.utm_source || "",
|
||||
utm_medium: utmParams.utm_medium || "",
|
||||
utm_campaign: utmParams.utm_campaign || "",
|
||||
utm_term: utmParams.utm_term || "",
|
||||
utm_content: utmParams.utm_content || "",
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
@@ -630,12 +611,23 @@ export async function main(
|
||||
referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, time_spent_sec,
|
||||
is_bounce, is_qr_scan, conversion_type, conversion_value, req_full_path)
|
||||
VALUES ${clickhouseData.map(record => {
|
||||
// 确保所有字符串值都是字符串类型，并安全处理替换
|
||||
// 增强版安全替换函数，处理所有特殊字符
|
||||
const safeReplace = (val: unknown): string => {
|
||||
// 确保值是字符串，如果是null或undefined则使用空字符串
|
||||
const str = val === null || val === undefined ? "" : String(val);
|
||||
// 安全替换单引号
|
||||
return str.replace(/'/g, "''");
|
||||
|
||||
// 转义所有可能导致SQL注入或格式错误的字符
|
||||
// 1. 先替换所有反斜杠
|
||||
// 2. 再替换单引号
|
||||
// 3. 替换所有控制字符和特殊字符
|
||||
return str
|
||||
.replace(/\\/g, "\\\\") // 转义反斜杠
|
||||
.replace(/'/g, "\\'") // 转义单引号
|
||||
.replace(/\r/g, "\\r") // 转义回车
|
||||
.replace(/\n/g, "\\n") // 转义换行
|
||||
.replace(/\t/g, "\\t") // 转义制表符
|
||||
.replace(/\0/g, "") // 移除空字符
|
||||
.replace(/[\x00-\x1F\x7F-\x9F]/g, ""); // 移除所有控制字符
|
||||
};
|
||||
|
||||
return `('${record.event_time}', '${safeReplace(record.event_type)}', '${safeReplace(record.event_attributes)}',
|
||||
@@ -693,7 +685,6 @@ export async function main(
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
let lastSyncTime = 0;
|
||||
let lastSyncId = "";
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
@@ -750,7 +741,6 @@ export async function main(
|
||||
if (records.length > 0) {
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
|
||||
lastSyncId = lastRecord._id.toString();
|
||||
}
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
@@ -758,11 +748,12 @@ export async function main(
|
||||
|
||||
// 更新同步状态
|
||||
if (processedRecords > 0 && lastSyncTime > 0) {
|
||||
// 创建新的同步状态
|
||||
// 只在非自定义时间范围模式下更新同步状态
|
||||
if (!use_custom_time_range) {
|
||||
// 创建新的同步状态，简化对象结构
|
||||
const newSyncState: SyncState = {
|
||||
last_sync_time: lastSyncTime,
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + totalBatchRecords,
|
||||
last_sync_id: lastSyncId
|
||||
records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + processedRecords, // 使用处理的总记录数，而不是实际插入数
|
||||
};
|
||||
|
||||
try {
|
||||
@@ -771,7 +762,11 @@ export async function main(
|
||||
logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`更新同步状态失败: ${error.message}`);
|
||||
logWithTimestamp(`更新同步状态失败: ${error.message}，将继续执行`);
|
||||
// 不抛出错误，继续执行
|
||||
}
|
||||
} else {
|
||||
logWithTimestamp("使用自定义时间范围模式，不更新全局同步状态");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -780,7 +775,8 @@ export async function main(
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
|
||||
message: "数据同步完成"
|
||||
message: use_custom_time_range ? "自定义时间范围数据同步完成" : "数据同步完成",
|
||||
custom_time_range_used: use_custom_time_range
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("同步过程中发生错误:", err);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// 描述: 此脚本从PostgreSQL数据库获取所有shorturl类型的资源及其关联数据，并同步到ClickHouse
|
||||
|
||||
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
|
||||
import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
|
||||
import { getResource, getVariable, setVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
|
||||
|
||||
// 资源属性接口
|
||||
interface ResourceAttributes {
|
||||
@@ -37,6 +37,15 @@ interface PgConfig {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// 上次同步状态接口
|
||||
interface SyncState {
|
||||
lastSyncTime: string;
|
||||
lastRunTime: string;
|
||||
}
|
||||
|
||||
// 状态变量名称
|
||||
const STATE_VARIABLE_PATH = "f/shorturl_analytics/shorturl_sync_state";
|
||||
|
||||
// Windmill函数定义
|
||||
export async function main(
|
||||
/** PostgreSQL和ClickHouse同步脚本 */
|
||||
@@ -47,9 +56,11 @@ export async function main(
|
||||
includeDeleted?: boolean;
|
||||
/** 是否执行实际写入操作 */
|
||||
dryRun?: boolean;
|
||||
/** 开始时间（ISO格式）*/
|
||||
/** 是否强制全量同步 */
|
||||
forceFullSync?: boolean;
|
||||
/** 手动指定开始时间（ISO格式）- 会覆盖自动增量设置 */
|
||||
startTime?: string;
|
||||
/** 结束时间（ISO格式）*/
|
||||
/** 手动指定结束时间（ISO格式）*/
|
||||
endTime?: string;
|
||||
}
|
||||
) {
|
||||
@@ -57,8 +68,41 @@ export async function main(
|
||||
const limit = params.limit || 500;
|
||||
const includeDeleted = params.includeDeleted || false;
|
||||
const dryRun = params.dryRun || false;
|
||||
const startTime = params.startTime ? new Date(params.startTime) : undefined;
|
||||
const endTime = params.endTime ? new Date(params.endTime) : undefined;
|
||||
const forceFullSync = params.forceFullSync || false;
|
||||
|
||||
// 获取当前时间作为本次运行时间
|
||||
const currentRunTime = new Date().toISOString();
|
||||
|
||||
// 初始化同步状态
|
||||
let syncState: SyncState;
|
||||
let startTime: Date | undefined;
|
||||
const endTime: Date | undefined = params.endTime ? new Date(params.endTime) : new Date();
|
||||
|
||||
// 如果强制全量同步或手动指定了开始时间，则使用指定的开始时间
|
||||
if (forceFullSync || params.startTime) {
|
||||
startTime = params.startTime ? new Date(params.startTime) : undefined;
|
||||
console.log(`使用${params.startTime ? '手动指定' : '全量同步'} - 开始时间: ${startTime ? startTime.toISOString() : '无限制'}`);
|
||||
}
|
||||
// 否则尝试获取上次同步时间作为增量同步的开始时间点
|
||||
else {
|
||||
try {
|
||||
// 获取上次同步状态
|
||||
const stateStr = await getVariable(STATE_VARIABLE_PATH);
|
||||
if (stateStr) {
|
||||
syncState = JSON.parse(stateStr);
|
||||
console.log(`获取到上次同步状态: 同步时间=${syncState.lastSyncTime}, 运行时间=${syncState.lastRunTime}`);
|
||||
|
||||
// 使用上次运行时间作为本次的开始时间 (减去1分钟防止边界问题)
|
||||
const lastRunTime = new Date(syncState.lastRunTime);
|
||||
lastRunTime.setMinutes(lastRunTime.getMinutes() - 1);
|
||||
startTime = lastRunTime;
|
||||
} else {
|
||||
console.log("未找到上次同步状态，将执行全量同步");
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
console.log(`获取同步状态出错: ${error instanceof Error ? error.message : String(error)}，将执行全量同步`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`开始同步PostgreSQL shorturl数据到ClickHouse`);
|
||||
console.log(`参数: limit=${limit}, includeDeleted=${includeDeleted}, dryRun=${dryRun}`);
|
||||
@@ -67,7 +111,7 @@ export async function main(
|
||||
|
||||
// 获取数据库配置
|
||||
console.log("获取PostgreSQL数据库配置...");
|
||||
const pgConfig = await getResource('f/limq/postgresql') as PgConfig;
|
||||
const pgConfig = await getResource('f/limq/production_supabase') as PgConfig;
|
||||
console.log(`数据库连接配置: host=${pgConfig.host}, port=${pgConfig.port}, database=${pgConfig.dbname || 'postgres'}, user=${pgConfig.user}`);
|
||||
|
||||
let pgPool: Pool | null = null;
|
||||
@@ -106,6 +150,8 @@ export async function main(
|
||||
console.log(`获取到 ${shorturls.length} 个shorturl资源`);
|
||||
|
||||
if (shorturls.length === 0) {
|
||||
// 即使没有数据也更新状态
|
||||
await updateSyncState(currentRunTime);
|
||||
return { synced: 0, message: "没有找到需要同步的shorturl资源" };
|
||||
}
|
||||
|
||||
@@ -120,7 +166,11 @@ export async function main(
|
||||
// 写入ClickHouse
|
||||
const inserted = await insertToClickhouse(clickhouseData);
|
||||
console.log(`成功写入 ${inserted} 条记录到ClickHouse`);
|
||||
return { synced: inserted, message: "同步完成" };
|
||||
|
||||
// 更新同步状态
|
||||
await updateSyncState(currentRunTime);
|
||||
|
||||
return { synced: inserted, message: "同步完成", lastSyncTime: currentRunTime };
|
||||
} else {
|
||||
console.log("Dry run模式 - 不执行实际写入");
|
||||
console.log(`将写入 ${clickhouseData.length} 条记录到ClickHouse`);
|
||||
@@ -146,6 +196,22 @@ export async function main(
|
||||
}
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
async function updateSyncState(currentRunTime: string): Promise<void> {
|
||||
try {
|
||||
const syncState: SyncState = {
|
||||
lastSyncTime: new Date().toISOString(), // 记录数据同步完成的时间
|
||||
lastRunTime: currentRunTime // 记录本次运行的时间点
|
||||
};
|
||||
|
||||
console.log(`更新同步状态: ${JSON.stringify(syncState)}`);
|
||||
await setVariable(STATE_VARIABLE_PATH, JSON.stringify(syncState));
|
||||
} catch (error: unknown) {
|
||||
console.error(`更新同步状态失败: ${error instanceof Error ? error.message : String(error)}`);
|
||||
// 不中断主流程，即使状态更新失败
|
||||
}
|
||||
}
|
||||
|
||||
// 从PostgreSQL获取所有shorturl资源
|
||||
async function fetchShorturlResources(
|
||||
pgPool: Pool,
|
||||
@@ -185,8 +251,9 @@ async function fetchShorturlResources(
|
||||
query += ` AND r.deleted_at IS NULL`;
|
||||
}
|
||||
|
||||
// 修改为同时考虑created_at和updated_at，确保捕获自上次同步以来创建或更新的记录
|
||||
if (options.startTime) {
|
||||
query += ` AND r.created_at >= $${paramCount}`;
|
||||
query += ` AND (r.created_at >= $${paramCount} OR r.updated_at >= $${paramCount})`;
|
||||
params.push(options.startTime);
|
||||
paramCount++;
|
||||
}
|
||||
@@ -197,7 +264,8 @@ async function fetchShorturlResources(
|
||||
paramCount++;
|
||||
}
|
||||
|
||||
query += ` ORDER BY r.created_at DESC LIMIT $${paramCount}`;
|
||||
// 优先按更新时间排序，确保最近更新的记录先处理
|
||||
query += ` ORDER BY r.updated_at DESC, r.created_at DESC LIMIT $${paramCount}`;
|
||||
params.push(options.limit);
|
||||
|
||||
const client = await pgPool.connect();
|
||||
|
||||
Reference in New Issue
Block a user