Compare commits
4 Commits
chart
...
92d82b18a0
| Author | SHA1 | Date | |
|---|---|---|---|
| 92d82b18a0 | |||
| 1e9e5928d7 | |||
| 231cf556b0 | |||
| 3413d3e182 |
152
api/events.ts
Normal file
152
api/events.ts
Normal file
@@ -0,0 +1,152 @@
|
||||
import { Router } from 'express';
|
||||
import type { Request, Response } from 'express';
|
||||
import type { ApiResponse, EventsQueryParams } from '../lib/types';
|
||||
import {
|
||||
getEvents,
|
||||
getEventsSummary,
|
||||
getTimeSeriesData,
|
||||
getGeoAnalytics,
|
||||
getDeviceAnalytics
|
||||
} from '../lib/analytics';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// 获取事件列表
|
||||
router.get('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const params: EventsQueryParams = {
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
eventType: req.query.eventType as string,
|
||||
linkId: req.query.linkId as string,
|
||||
linkSlug: req.query.linkSlug as string,
|
||||
userId: req.query.userId as string,
|
||||
teamId: req.query.teamId as string,
|
||||
projectId: req.query.projectId as string,
|
||||
page: req.query.page ? parseInt(req.query.page as string, 10) : 1,
|
||||
pageSize: req.query.pageSize ? parseInt(req.query.pageSize as string, 10) : 20,
|
||||
sortBy: req.query.sortBy as string,
|
||||
sortOrder: req.query.sortOrder as 'asc' | 'desc'
|
||||
};
|
||||
|
||||
const { events, total } = await getEvents(params);
|
||||
|
||||
const response: ApiResponse<typeof events> = {
|
||||
success: true,
|
||||
data: events,
|
||||
meta: {
|
||||
total,
|
||||
page: params.page,
|
||||
pageSize: params.pageSize
|
||||
}
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取事件概览
|
||||
router.get('/summary', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const summary = await getEventsSummary({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof summary> = {
|
||||
success: true,
|
||||
data: summary
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取时间序列数据
|
||||
router.get('/time-series', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const data = await getTimeSeriesData({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string,
|
||||
granularity: (req.query.granularity || 'day') as 'hour' | 'day' | 'week' | 'month'
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取地理位置分析
|
||||
router.get('/geo', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const data = await getGeoAnalytics({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string,
|
||||
groupBy: (req.query.groupBy || 'country') as 'country' | 'city'
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
// 获取设备分析
|
||||
router.get('/devices', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const data = await getDeviceAnalytics({
|
||||
startTime: req.query.startTime as string,
|
||||
endTime: req.query.endTime as string,
|
||||
linkId: req.query.linkId as string
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
res.status(500).json(response);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -1,28 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getDeviceAnalysis } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId') || undefined;
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 获取设备分析详情
|
||||
const analysisData = await getDeviceAnalysis(
|
||||
startDate,
|
||||
endDate,
|
||||
linkId
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(analysisData);
|
||||
} catch (error) {
|
||||
console.error('Error in device-analysis API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch device analysis data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getConversionFunnel } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId');
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 验证必要参数
|
||||
if (!linkId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing required parameter: linkId' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 获取转化漏斗数据
|
||||
const funnelData = await getConversionFunnel(
|
||||
linkId,
|
||||
startDate || undefined,
|
||||
endDate || undefined
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(funnelData);
|
||||
} catch (error) {
|
||||
console.error('Error in funnel API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch funnel data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getLinkPerformance } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId');
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 验证必要参数
|
||||
if (!linkId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing required parameter: linkId' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 获取链接表现数据
|
||||
const performanceData = await getLinkPerformance(
|
||||
linkId,
|
||||
startDate || undefined,
|
||||
endDate || undefined
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(performanceData);
|
||||
} catch (error) {
|
||||
console.error('Error in link-performance API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch link performance data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getLinkStatusDistribution } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const projectId = searchParams.get('projectId') || undefined;
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 获取链接状态分布数据
|
||||
const distributionData = await getLinkStatusDistribution(
|
||||
startDate,
|
||||
endDate,
|
||||
projectId
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(distributionData);
|
||||
} catch (error) {
|
||||
console.error('Error in link-status-distribution API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch link status distribution data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getOverviewCards } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const projectId = searchParams.get('projectId') || undefined;
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 获取概览卡片数据
|
||||
const cardsData = await getOverviewCards(
|
||||
startDate,
|
||||
endDate,
|
||||
projectId
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(cardsData);
|
||||
} catch (error) {
|
||||
console.error('Error in overview-cards API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch overview cards data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getLinkOverview } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId');
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 验证必要参数
|
||||
if (!linkId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing required parameter: linkId' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 获取链接概览数据
|
||||
const overviewData = await getLinkOverview(
|
||||
linkId,
|
||||
startDate || undefined,
|
||||
endDate || undefined
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(overviewData);
|
||||
} catch (error) {
|
||||
console.error('Error in overview API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch overview data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getPlatformDistribution } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId') || undefined;
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 获取平台分布数据
|
||||
const distributionData = await getPlatformDistribution(
|
||||
startDate,
|
||||
endDate,
|
||||
linkId
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(distributionData);
|
||||
} catch (error) {
|
||||
console.error('Error in platform-distribution API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch platform distribution data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getPopularLinks } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const projectId = searchParams.get('projectId') || undefined;
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
const sortBy = searchParams.get('sortBy') as 'visits' | 'uniqueVisitors' | 'conversionRate' || 'visits';
|
||||
const limit = searchParams.get('limit') ? parseInt(searchParams.get('limit') as string, 10) : 10;
|
||||
|
||||
// 获取热门链接数据
|
||||
const linksData = await getPopularLinks(
|
||||
startDate,
|
||||
endDate,
|
||||
projectId,
|
||||
sortBy,
|
||||
limit
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(linksData);
|
||||
} catch (error) {
|
||||
console.error('Error in popular-links API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch popular links data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getPopularReferrers } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId') || undefined;
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
const type = searchParams.get('type') as 'domain' | 'full' || 'domain';
|
||||
const limit = searchParams.get('limit') ? parseInt(searchParams.get('limit') as string, 10) : 10;
|
||||
|
||||
// 获取热门引荐来源数据
|
||||
const referrersData = await getPopularReferrers(
|
||||
startDate,
|
||||
endDate,
|
||||
linkId,
|
||||
type,
|
||||
limit
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(referrersData);
|
||||
} catch (error) {
|
||||
console.error('Error in popular-referrers API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch popular referrers data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getQrCodeAnalysis } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId') || undefined;
|
||||
const qrCodeId = searchParams.get('qrCodeId') || undefined;
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
|
||||
// 获取QR码分析数据
|
||||
const analysisData = await getQrCodeAnalysis(
|
||||
startDate,
|
||||
endDate,
|
||||
linkId,
|
||||
qrCodeId
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(analysisData);
|
||||
} catch (error) {
|
||||
console.error('Error in qr-code-analysis API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch QR code analysis data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { trackEvent, EventType, ConversionType } from '@/lib/analytics';
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
// 解析请求体
|
||||
const body = await request.json();
|
||||
|
||||
// 验证必要字段
|
||||
if (!body.linkId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing required field: linkId' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (!body.eventType || !Object.values(EventType).includes(body.eventType)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Invalid or missing eventType',
|
||||
validValues: Object.values(EventType)
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 验证转化类型（如果提供）
|
||||
if (
|
||||
body.conversionType &&
|
||||
!Object.values(ConversionType).includes(body.conversionType)
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Invalid conversionType',
|
||||
validValues: Object.values(ConversionType)
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 添加客户端IP
|
||||
const clientIp = request.headers.get('x-forwarded-for') ||
|
||||
request.headers.get('x-real-ip') ||
|
||||
'0.0.0.0';
|
||||
|
||||
// 添加用户代理
|
||||
const userAgent = request.headers.get('user-agent') || '';
|
||||
|
||||
// 合并数据
|
||||
const eventData = {
|
||||
...body,
|
||||
ipAddress: body.ipAddress || clientIp,
|
||||
userAgent: body.userAgent || userAgent,
|
||||
};
|
||||
|
||||
// 追踪事件
|
||||
const result = await trackEvent(eventData);
|
||||
|
||||
// 返回结果
|
||||
return NextResponse.json(result);
|
||||
} catch (error) {
|
||||
console.error('Error in track API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to track event' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getVisitTrends, TimeGranularity } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// 获取请求参数
|
||||
const { searchParams } = new URL(request.url);
|
||||
const linkId = searchParams.get('linkId');
|
||||
const startDate = searchParams.get('startDate') || undefined;
|
||||
const endDate = searchParams.get('endDate') || undefined;
|
||||
const granularity = searchParams.get('granularity') as TimeGranularity || TimeGranularity.DAY;
|
||||
|
||||
// 验证必要参数
|
||||
if (!linkId) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing required parameter: linkId' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 验证粒度参数
|
||||
const validGranularities = Object.values(TimeGranularity);
|
||||
if (granularity && !validGranularities.includes(granularity)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Invalid granularity value',
|
||||
validValues: validGranularities
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 获取访问趋势数据
|
||||
const trendsData = await getVisitTrends(
|
||||
linkId,
|
||||
startDate || undefined,
|
||||
endDate || undefined,
|
||||
granularity
|
||||
);
|
||||
|
||||
// 返回数据
|
||||
return NextResponse.json(trendsData);
|
||||
} catch (error) {
|
||||
console.error('Error in trends API:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch trends data' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
28
app/api/events/devices/route.ts
Normal file
28
app/api/events/devices/route.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
import { getDeviceAnalytics } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
const data = await getDeviceAnalytics({
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
29
app/api/events/geo/route.ts
Normal file
29
app/api/events/geo/route.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
import { getGeoAnalytics } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
const data = await getGeoAnalytics({
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
groupBy: (searchParams.get('groupBy') || 'country') as 'country' | 'city'
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
51
app/api/events/route.ts
Normal file
51
app/api/events/route.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse, EventsQueryParams, EventType } from '@/lib/types';
|
||||
import {
|
||||
getEvents,
|
||||
getEventsSummary,
|
||||
getTimeSeriesData,
|
||||
getGeoAnalytics,
|
||||
getDeviceAnalytics
|
||||
} from '@/lib/analytics';
|
||||
|
||||
// 获取事件列表
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
const params: EventsQueryParams = {
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
eventType: searchParams.get('eventType') as EventType || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
linkSlug: searchParams.get('linkSlug') || undefined,
|
||||
userId: searchParams.get('userId') || undefined,
|
||||
teamId: searchParams.get('teamId') || undefined,
|
||||
projectId: searchParams.get('projectId') || undefined,
|
||||
page: searchParams.has('page') ? parseInt(searchParams.get('page')!, 10) : 1,
|
||||
pageSize: searchParams.has('pageSize') ? parseInt(searchParams.get('pageSize')!, 10) : 20,
|
||||
sortBy: searchParams.get('sortBy') || undefined,
|
||||
sortOrder: (searchParams.get('sortOrder') as 'asc' | 'desc') || undefined
|
||||
};
|
||||
|
||||
const { events, total } = await getEvents(params);
|
||||
|
||||
const response: ApiResponse<typeof events> = {
|
||||
success: true,
|
||||
data: events,
|
||||
meta: {
|
||||
total,
|
||||
page: params.page,
|
||||
pageSize: params.pageSize
|
||||
}
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
28
app/api/events/summary/route.ts
Normal file
28
app/api/events/summary/route.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
import { getEventsSummary } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
const summary = await getEventsSummary({
|
||||
startTime: searchParams.get('startTime') || undefined,
|
||||
endTime: searchParams.get('endTime') || undefined,
|
||||
linkId: searchParams.get('linkId') || undefined
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof summary> = {
|
||||
success: true,
|
||||
data: summary
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
38
app/api/events/time-series/route.ts
Normal file
38
app/api/events/time-series/route.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import type { ApiResponse } from '@/lib/types';
|
||||
import { getTimeSeriesData } from '@/lib/analytics';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const startTime = searchParams.get('startTime');
|
||||
const endTime = searchParams.get('endTime');
|
||||
|
||||
if (!startTime || !endTime) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'startTime and endTime are required'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
const data = await getTimeSeriesData({
|
||||
startTime,
|
||||
endTime,
|
||||
linkId: searchParams.get('linkId') || undefined,
|
||||
granularity: (searchParams.get('granularity') || 'day') as 'hour' | 'day' | 'week' | 'month'
|
||||
});
|
||||
|
||||
const response: ApiResponse<typeof data> = {
|
||||
success: true,
|
||||
data
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
};
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getLinkDetailsById } from '@/app/api/links/service';
|
||||
|
||||
// 正确的Next.js 15 API路由处理函数参数类型定义
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
context: { params: Promise<any> }
|
||||
) {
|
||||
try {
|
||||
// 获取参数，支持异步格式
|
||||
const params = await context.params;
|
||||
const linkId = params.linkId;
|
||||
const link = await getLinkDetailsById(linkId);
|
||||
|
||||
if (!link) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Link not found' },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json(link);
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch link details:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch link details', message: (error as Error).message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getLinkById } from '../service';
|
||||
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
context: { params: Promise<any> }
|
||||
) {
|
||||
try {
|
||||
// 获取参数，支持异步格式
|
||||
const params = await context.params;
|
||||
const linkId = params.linkId;
|
||||
const link = await getLinkById(linkId);
|
||||
|
||||
if (!link) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Link not found' },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json(link);
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch link:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch link', message: (error as Error).message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
import { executeQuery, executeQuerySingle } from '@/lib/clickhouse';
|
||||
import { Link, LinkQueryParams } from '../types';
|
||||
|
||||
/**
|
||||
* Find links with filtering options
|
||||
*/
|
||||
export async function findLinks({
|
||||
limit = 10,
|
||||
offset = 0,
|
||||
searchTerm = '',
|
||||
tagFilter = '',
|
||||
isActive = null,
|
||||
}: LinkQueryParams) {
|
||||
// Build WHERE conditions
|
||||
const conditions = [];
|
||||
|
||||
if (searchTerm) {
|
||||
conditions.push(`
|
||||
(lower(title) LIKE lower('%${searchTerm}%') OR
|
||||
lower(original_url) LIKE lower('%${searchTerm}%'))
|
||||
`);
|
||||
}
|
||||
|
||||
if (tagFilter) {
|
||||
conditions.push(`hasAny(tags, ['${tagFilter}'])`);
|
||||
}
|
||||
|
||||
if (isActive !== null) {
|
||||
conditions.push(`is_active = ${isActive ? 'true' : 'false'}`);
|
||||
}
|
||||
|
||||
const whereClause = conditions.length > 0
|
||||
? `WHERE ${conditions.join(' AND ')}`
|
||||
: '';
|
||||
|
||||
// Get total count
|
||||
const countQuery = `
|
||||
SELECT count() as total
|
||||
FROM links
|
||||
${whereClause}
|
||||
`;
|
||||
|
||||
const countData = await executeQuery<{ total: number }>(countQuery);
|
||||
const total = countData.length > 0 ? countData[0].total : 0;
|
||||
|
||||
// 使用左连接获取链接数据和统计信息
|
||||
const linksQuery = `
|
||||
SELECT
|
||||
l.link_id,
|
||||
l.original_url,
|
||||
l.created_at,
|
||||
l.created_by,
|
||||
l.title,
|
||||
l.description,
|
||||
l.tags,
|
||||
l.is_active,
|
||||
l.expires_at,
|
||||
l.team_id,
|
||||
l.project_id,
|
||||
count(le.event_id) as visits,
|
||||
count(DISTINCT le.visitor_id) as unique_visits
|
||||
FROM links l
|
||||
LEFT JOIN link_events le ON l.link_id = le.link_id
|
||||
${whereClause}
|
||||
GROUP BY
|
||||
l.link_id,
|
||||
l.original_url,
|
||||
l.created_at,
|
||||
l.created_by,
|
||||
l.title,
|
||||
l.description,
|
||||
l.tags,
|
||||
l.is_active,
|
||||
l.expires_at,
|
||||
l.team_id,
|
||||
l.project_id
|
||||
ORDER BY l.created_at DESC
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
`;
|
||||
|
||||
const links = await executeQuery<Link>(linksQuery);
|
||||
|
||||
return {
|
||||
links,
|
||||
total,
|
||||
limit,
|
||||
offset,
|
||||
page: Math.floor(offset / limit) + 1,
|
||||
totalPages: Math.ceil(total / limit)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a single link by ID
|
||||
*/
|
||||
export async function findLinkById(linkId: string): Promise<Link | null> {
|
||||
const query = `
|
||||
SELECT
|
||||
l.link_id,
|
||||
l.original_url,
|
||||
l.created_at,
|
||||
l.created_by,
|
||||
l.title,
|
||||
l.description,
|
||||
l.tags,
|
||||
l.is_active,
|
||||
l.expires_at,
|
||||
l.team_id,
|
||||
l.project_id,
|
||||
count(le.event_id) as visits,
|
||||
count(DISTINCT le.visitor_id) as unique_visits
|
||||
FROM links l
|
||||
LEFT JOIN link_events le ON l.link_id = le.link_id
|
||||
WHERE l.link_id = '${linkId}'
|
||||
GROUP BY
|
||||
l.link_id,
|
||||
l.original_url,
|
||||
l.created_at,
|
||||
l.created_by,
|
||||
l.title,
|
||||
l.description,
|
||||
l.tags,
|
||||
l.is_active,
|
||||
l.expires_at,
|
||||
l.team_id,
|
||||
l.project_id
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
return await executeQuerySingle<Link>(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a single link by ID - only basic info without statistics
|
||||
*/
|
||||
export async function findLinkDetailsById(linkId: string): Promise<Omit<Link, 'visits' | 'unique_visits'> | null> {
|
||||
const query = `
|
||||
SELECT
|
||||
link_id,
|
||||
original_url,
|
||||
created_at,
|
||||
created_by,
|
||||
title,
|
||||
description,
|
||||
tags,
|
||||
is_active,
|
||||
expires_at,
|
||||
team_id,
|
||||
project_id
|
||||
FROM links
|
||||
WHERE link_id = '${linkId}'
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
return await executeQuerySingle<Omit<Link, 'visits' | 'unique_visits'>>(query);
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { LinkQueryParams } from '../types';
|
||||
import { getLinks } from './service';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
// Parse request parameters
|
||||
const params: LinkQueryParams = {
|
||||
limit: searchParams.has('limit') ? Number(searchParams.get('limit')) : 10,
|
||||
page: searchParams.has('page') ? Number(searchParams.get('page')) : 1,
|
||||
searchTerm: searchParams.get('search') || '',
|
||||
tagFilter: searchParams.get('tag') || '',
|
||||
};
|
||||
|
||||
// Handle active status filter
|
||||
const activeFilter = searchParams.get('active');
|
||||
if (activeFilter === 'true') params.isActive = true;
|
||||
if (activeFilter === 'false') params.isActive = false;
|
||||
|
||||
// Get link data
|
||||
const result = await getLinks(params);
|
||||
return NextResponse.json(result);
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch links:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch links', message: (error as Error).message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
import { Link, LinkQueryParams, PaginatedResponse } from '../types';
|
||||
import { findLinkById, findLinkDetailsById, findLinks } from './repository';
|
||||
|
||||
/**
|
||||
* Get links with pagination information
|
||||
*/
|
||||
export async function getLinks(params: LinkQueryParams): Promise<PaginatedResponse<Link>> {
|
||||
// Convert page number to offset
|
||||
const { page, limit = 10, ...otherParams } = params;
|
||||
const offset = page ? (page - 1) * limit : params.offset || 0;
|
||||
|
||||
const result = await findLinks({
|
||||
...otherParams,
|
||||
limit,
|
||||
offset
|
||||
});
|
||||
|
||||
return {
|
||||
data: result.links,
|
||||
pagination: {
|
||||
total: result.total,
|
||||
limit: result.limit,
|
||||
offset: result.offset,
|
||||
page: result.page,
|
||||
totalPages: result.totalPages
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single link by ID with full details (including statistics)
|
||||
*/
|
||||
export async function getLinkById(linkId: string): Promise<Link | null> {
|
||||
return await findLinkById(linkId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single link by ID - only basic info without statistics
|
||||
*/
|
||||
export async function getLinkDetailsById(linkId: string): Promise<Omit<Link, 'visits' | 'unique_visits'> | null> {
|
||||
return await findLinkDetailsById(linkId);
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import { executeQuerySingle } from '@/lib/clickhouse';
|
||||
import { StatsOverview } from '../types';
|
||||
|
||||
/**
|
||||
* Get overview statistics for links
|
||||
*/
|
||||
export async function findStatsOverview(): Promise<StatsOverview | null> {
|
||||
const query = `
|
||||
WITH
|
||||
toUInt64(count()) as total_links,
|
||||
toUInt64(countIf(is_active = true)) as active_links
|
||||
FROM links
|
||||
SELECT
|
||||
total_links as totalLinks,
|
||||
active_links as activeLinks,
|
||||
(SELECT count() FROM link_events) as totalVisits,
|
||||
(SELECT count() FROM link_events) / NULLIF(total_links, 0) as conversionRate
|
||||
`;
|
||||
|
||||
return await executeQuerySingle<StatsOverview>(query);
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { getStatsOverview } from './service';
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const stats = await getStatsOverview();
|
||||
return NextResponse.json(stats);
|
||||
} catch (error) {
|
||||
console.error('获取统计概览失败:', error);
|
||||
return NextResponse.json(
|
||||
{ error: '获取统计概览失败', message: (error as Error).message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import { StatsOverview } from '../types';
|
||||
import { findStatsOverview } from './repository';
|
||||
|
||||
/**
|
||||
* Get link statistics overview
|
||||
*/
|
||||
export async function getStatsOverview(): Promise<StatsOverview> {
|
||||
const stats = await findStatsOverview();
|
||||
|
||||
// Return default values if no data
|
||||
if (!stats) {
|
||||
return {
|
||||
totalLinks: 0,
|
||||
activeLinks: 0,
|
||||
totalVisits: 0,
|
||||
conversionRate: 0
|
||||
};
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import { executeQuery } from '@/lib/clickhouse';
|
||||
import { Tag } from '../types';
|
||||
|
||||
/**
|
||||
* Get all tags with usage counts
|
||||
*/
|
||||
export async function findAllTags(): Promise<Tag[]> {
|
||||
const query = `
|
||||
SELECT
|
||||
tag,
|
||||
count() as count
|
||||
FROM links
|
||||
ARRAY JOIN tags as tag
|
||||
GROUP BY tag
|
||||
ORDER BY count DESC
|
||||
`;
|
||||
|
||||
return await executeQuery<Tag>(query);
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { getAllTags } from './service';
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const tags = await getAllTags();
|
||||
return NextResponse.json(tags);
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch tags:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch tags', message: (error as Error).message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
import { Tag } from '../types';
|
||||
import { findAllTags } from './repository';
|
||||
|
||||
/**
|
||||
* Get all available tags
|
||||
*/
|
||||
export async function getAllTags(): Promise<Tag[]> {
|
||||
return await findAllTags();
|
||||
}
|
||||
221
app/api/types.ts
221
app/api/types.ts
@@ -1,221 +0,0 @@
|
||||
// 链接数据类型
|
||||
export interface Link {
|
||||
link_id: string;
|
||||
original_url: string;
|
||||
created_at: string;
|
||||
created_by: string;
|
||||
title: string;
|
||||
description: string;
|
||||
tags: string[];
|
||||
is_active: boolean;
|
||||
expires_at: string | null;
|
||||
team_id: string;
|
||||
project_id: string;
|
||||
visits: number;
|
||||
unique_visits: number;
|
||||
}
|
||||
|
||||
// 分页响应类型
|
||||
export interface PaginatedResponse<T> {
|
||||
data: T[];
|
||||
pagination: {
|
||||
total: number;
|
||||
limit: number;
|
||||
offset: number;
|
||||
page: number;
|
||||
totalPages: number;
|
||||
}
|
||||
}
|
||||
|
||||
// 链接查询参数
|
||||
export interface LinkQueryParams {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
page?: number;
|
||||
searchTerm?: string;
|
||||
tagFilter?: string;
|
||||
isActive?: boolean | null;
|
||||
}
|
||||
|
||||
// 标签类型
|
||||
export interface Tag {
|
||||
tag: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
// 统计概览类型
|
||||
export interface StatsOverview {
|
||||
totalLinks: number;
|
||||
activeLinks: number;
|
||||
totalVisits: number;
|
||||
conversionRate: number;
|
||||
}
|
||||
|
||||
// Analytics数据类型
|
||||
export interface LinkOverviewData {
|
||||
totalVisits: number;
|
||||
uniqueVisitors: number;
|
||||
averageTimeSpent: number;
|
||||
bounceCount: number;
|
||||
conversionCount: number;
|
||||
uniqueReferrers: number;
|
||||
deviceTypes: {
|
||||
mobile: number;
|
||||
tablet: number;
|
||||
desktop: number;
|
||||
other: number;
|
||||
};
|
||||
qrScanCount: number;
|
||||
totalConversionValue: number;
|
||||
}
|
||||
|
||||
export interface FunnelStep {
|
||||
name: string;
|
||||
value: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface ConversionFunnelData {
|
||||
steps: FunnelStep[];
|
||||
totalConversions: number;
|
||||
conversionRate: number;
|
||||
}
|
||||
|
||||
export interface TrendPoint {
|
||||
timestamp: string;
|
||||
visits: number;
|
||||
uniqueVisitors: number;
|
||||
}
|
||||
|
||||
export interface VisitTrendsData {
|
||||
trends: TrendPoint[];
|
||||
totals: {
|
||||
visits: number;
|
||||
uniqueVisitors: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TrackEventRequest {
|
||||
linkId: string;
|
||||
eventType: string;
|
||||
visitorId?: string;
|
||||
sessionId?: string;
|
||||
referrer?: string;
|
||||
userAgent?: string;
|
||||
ipAddress?: string;
|
||||
timeSpent?: number;
|
||||
conversionType?: string;
|
||||
conversionValue?: number;
|
||||
customData?: Record<string, unknown>;
|
||||
isQrScan?: boolean;
|
||||
qrCodeId?: string;
|
||||
utmSource?: string;
|
||||
utmMedium?: string;
|
||||
utmCampaign?: string;
|
||||
}
|
||||
|
||||
export interface TrackEventResponse {
|
||||
success: boolean;
|
||||
eventId: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
// 链接表现数据
|
||||
export interface LinkPerformanceData {
|
||||
totalClicks: number;
|
||||
uniqueVisitors: number;
|
||||
averageTimeSpent: number;
|
||||
bounceRate: number;
|
||||
uniqueReferrers: number;
|
||||
conversionRate: number;
|
||||
activeDays: number;
|
||||
lastClickTime: string | null;
|
||||
deviceDistribution: {
|
||||
mobile: number;
|
||||
desktop: number;
|
||||
};
|
||||
}
|
||||
|
||||
// 平台分布数据
|
||||
export interface PlatformItem {
|
||||
name: string;
|
||||
count: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface PlatformDistributionData {
|
||||
totalVisits: number;
|
||||
platforms: PlatformItem[];
|
||||
browsers: PlatformItem[];
|
||||
}
|
||||
|
||||
// 设备分析数据
|
||||
export interface DeviceItem {
|
||||
name: string;
|
||||
count: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface DeviceModelItem {
|
||||
type: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
count: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface DeviceAnalysisData {
|
||||
totalVisits: number;
|
||||
deviceTypes: DeviceItem[];
|
||||
deviceBrands: DeviceItem[];
|
||||
deviceModels: DeviceModelItem[];
|
||||
}
|
||||
|
||||
// 热门引荐来源数据
|
||||
export interface ReferrerItem {
|
||||
source: string;
|
||||
visitCount: number;
|
||||
uniqueVisitors: number;
|
||||
conversionCount: number;
|
||||
conversionRate: number;
|
||||
averageTimeSpent: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface PopularReferrersData {
|
||||
referrers: ReferrerItem[];
|
||||
totalVisits: number;
|
||||
}
|
||||
|
||||
// QR码分析数据
|
||||
export interface LocationItem {
|
||||
city: string;
|
||||
country: string;
|
||||
scanCount: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface DeviceDistributionItem {
|
||||
type: string;
|
||||
count: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface HourlyDistributionItem {
|
||||
hour: number;
|
||||
scanCount: number;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface QrCodeAnalysisData {
|
||||
overview: {
|
||||
totalScans: number;
|
||||
uniqueScanners: number;
|
||||
conversionCount: number;
|
||||
conversionRate: number;
|
||||
averageTimeSpent: number;
|
||||
};
|
||||
locations: LocationItem[];
|
||||
deviceDistribution: DeviceDistributionItem[];
|
||||
hourlyDistribution: HourlyDistributionItem[];
|
||||
}
|
||||
1526
lib/analytics.ts
1526
lib/analytics.ts
File diff suppressed because it is too large
Load Diff
@@ -1,39 +1,109 @@
|
||||
import { createClient } from '@clickhouse/client';
|
||||
import type { EventsQueryParams } from './types';
|
||||
|
||||
// Create configuration object using the URL approach
|
||||
const config = {
|
||||
// ClickHouse 客户端配置
|
||||
const clickhouse = createClient({
|
||||
url: process.env.CLICKHOUSE_URL || 'http://localhost:8123',
|
||||
username: process.env.CLICKHOUSE_USER || 'default',
|
||||
password: process.env.CLICKHOUSE_PASSWORD || '',
|
||||
database: process.env.CLICKHOUSE_DATABASE || 'limq'
|
||||
};
|
||||
username: process.env.CLICKHOUSE_USER || 'admin',
|
||||
password: process.env.CLICKHOUSE_PASSWORD || 'your_secure_password',
|
||||
database: process.env.CLICKHOUSE_DB || 'shorturl_analytics'
|
||||
});
|
||||
|
||||
// Create ClickHouse client with proper URL format
|
||||
export const clickhouse = createClient(config);
|
||||
// 构建日期过滤条件
|
||||
function buildDateFilter(startTime?: string, endTime?: string): string {
|
||||
const filters = [];
|
||||
|
||||
if (startTime) {
|
||||
filters.push(`event_time >= parseDateTimeBestEffort('${startTime}')`);
|
||||
}
|
||||
|
||||
if (endTime) {
|
||||
filters.push(`event_time <= parseDateTimeBestEffort('${endTime}')`);
|
||||
}
|
||||
|
||||
return filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
|
||||
}
|
||||
|
||||
// 构建通用过滤条件
|
||||
export function buildFilter(params: Partial<EventsQueryParams>): string {
|
||||
const filters = [];
|
||||
|
||||
// 时间范围过滤
|
||||
if (params.startTime || params.endTime) {
|
||||
const dateFilter = buildDateFilter(params.startTime, params.endTime).replace('WHERE ', '');
|
||||
if (dateFilter) {
|
||||
filters.push(dateFilter);
|
||||
}
|
||||
}
|
||||
|
||||
// 事件类型过滤
|
||||
if (params.eventType) {
|
||||
filters.push(`event_type = '${params.eventType}'`);
|
||||
}
|
||||
|
||||
// 链接ID过滤
|
||||
if (params.linkId) {
|
||||
filters.push(`link_id = '${params.linkId}'`);
|
||||
}
|
||||
|
||||
// 链接短码过滤
|
||||
if (params.linkSlug) {
|
||||
filters.push(`link_slug = '${params.linkSlug}'`);
|
||||
}
|
||||
|
||||
// 用户ID过滤
|
||||
if (params.userId) {
|
||||
filters.push(`user_id = '${params.userId}'`);
|
||||
}
|
||||
|
||||
// 团队ID过滤
|
||||
if (params.teamId) {
|
||||
filters.push(`team_id = '${params.teamId}'`);
|
||||
}
|
||||
|
||||
// 项目ID过滤
|
||||
if (params.projectId) {
|
||||
filters.push(`project_id = '${params.projectId}'`);
|
||||
}
|
||||
|
||||
return filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute ClickHouse query and return results
|
||||
*/
|
||||
// 构建分页
|
||||
export function buildPagination(page?: number, pageSize?: number): string {
|
||||
const limit = pageSize || 20;
|
||||
const offset = ((page || 1) - 1) * limit;
|
||||
return `LIMIT ${limit} OFFSET ${offset}`;
|
||||
}
|
||||
|
||||
// 构建排序
|
||||
export function buildOrderBy(sortBy?: string, sortOrder?: 'asc' | 'desc'): string {
|
||||
if (!sortBy) {
|
||||
return 'ORDER BY event_time DESC';
|
||||
}
|
||||
return `ORDER BY ${sortBy} ${sortOrder || 'desc'}`;
|
||||
}
|
||||
|
||||
// 执行查询并处理错误
|
||||
export async function executeQuery<T>(query: string): Promise<T[]> {
|
||||
try {
|
||||
const result = await clickhouse.query({
|
||||
const resultSet = await clickhouse.query({
|
||||
query,
|
||||
format: 'JSONEachRow',
|
||||
format: 'JSONEachRow'
|
||||
});
|
||||
|
||||
const data = await result.json();
|
||||
return data as T[];
|
||||
const rows = await resultSet.json<T>();
|
||||
return Array.isArray(rows) ? rows : [rows];
|
||||
} catch (error) {
|
||||
console.error('ClickHouse query error:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute ClickHouse query and return a single result
|
||||
*/
|
||||
// 执行查询并返回单个结果
|
||||
export async function executeQuerySingle<T>(query: string): Promise<T | null> {
|
||||
const results = await executeQuery<T>(query);
|
||||
return results.length > 0 ? results[0] : null;
|
||||
}
|
||||
}
|
||||
|
||||
export default clickhouse;
|
||||
171
lib/types.ts
Normal file
171
lib/types.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
// 事件类型
|
||||
export enum EventType {
|
||||
CLICK = 'click',
|
||||
REDIRECT = 'redirect',
|
||||
CONVERSION = 'conversion',
|
||||
ERROR = 'error'
|
||||
}
|
||||
|
||||
// 转化类型
|
||||
export enum ConversionType {
|
||||
VISIT = 'visit',
|
||||
STAY = 'stay',
|
||||
INTERACT = 'interact',
|
||||
SIGNUP = 'signup',
|
||||
SUBSCRIPTION = 'subscription',
|
||||
PURCHASE = 'purchase'
|
||||
}
|
||||
|
||||
// 设备类型
|
||||
export enum DeviceType {
|
||||
MOBILE = 'mobile',
|
||||
TABLET = 'tablet',
|
||||
DESKTOP = 'desktop',
|
||||
OTHER = 'other'
|
||||
}
|
||||
|
||||
// API 响应基础接口
|
||||
export interface ApiResponse<T> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: string;
|
||||
meta?: {
|
||||
total?: number;
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
};
|
||||
}
|
||||
|
||||
// 事件查询参数
|
||||
export interface EventsQueryParams {
|
||||
startTime?: string; // ISO 格式时间
|
||||
endTime?: string; // ISO 格式时间
|
||||
eventType?: EventType;
|
||||
linkId?: string;
|
||||
linkSlug?: string;
|
||||
userId?: string;
|
||||
teamId?: string;
|
||||
projectId?: string;
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
sortBy?: string;
|
||||
sortOrder?: 'asc' | 'desc';
|
||||
}
|
||||
|
||||
// 事件基础信息
|
||||
export interface Event {
|
||||
event_id: string;
|
||||
event_time: string;
|
||||
event_type: EventType;
|
||||
event_attributes: Record<string, any>;
|
||||
|
||||
// 链接信息
|
||||
link_id: string;
|
||||
link_slug: string;
|
||||
link_label: string;
|
||||
link_title: string;
|
||||
link_original_url: string;
|
||||
link_attributes: Record<string, any>;
|
||||
link_created_at: string;
|
||||
link_expires_at: string | null;
|
||||
link_tags: string[];
|
||||
|
||||
// 用户信息
|
||||
user_id: string;
|
||||
user_name: string;
|
||||
user_email: string;
|
||||
user_attributes: Record<string, any>;
|
||||
|
||||
// 团队信息
|
||||
team_id: string;
|
||||
team_name: string;
|
||||
team_attributes: Record<string, any>;
|
||||
|
||||
// 项目信息
|
||||
project_id: string;
|
||||
project_name: string;
|
||||
project_attributes: Record<string, any>;
|
||||
|
||||
// 访问者信息
|
||||
visitor_id: string;
|
||||
session_id: string;
|
||||
ip_address: string;
|
||||
country: string;
|
||||
city: string;
|
||||
device_type: DeviceType;
|
||||
browser: string;
|
||||
os: string;
|
||||
user_agent: string;
|
||||
|
||||
// 来源信息
|
||||
referrer: string;
|
||||
utm_source: string;
|
||||
utm_medium: string;
|
||||
utm_campaign: string;
|
||||
|
||||
// 交互信息
|
||||
time_spent_sec: number;
|
||||
is_bounce: boolean;
|
||||
is_qr_scan: boolean;
|
||||
conversion_type: ConversionType;
|
||||
conversion_value: number;
|
||||
}
|
||||
|
||||
// 事件概览数据
|
||||
export interface EventsSummary {
|
||||
totalEvents: number;
|
||||
uniqueVisitors: number;
|
||||
totalConversions: number;
|
||||
averageTimeSpent: number;
|
||||
deviceTypes: {
|
||||
mobile: number;
|
||||
desktop: number;
|
||||
tablet: number;
|
||||
other: number;
|
||||
};
|
||||
browsers: Array<{
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}>;
|
||||
operatingSystems: Array<{
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
// 时间序列数据
|
||||
export interface TimeSeriesData {
|
||||
timestamp: string;
|
||||
events: number;
|
||||
visitors: number;
|
||||
conversions: number;
|
||||
}
|
||||
|
||||
// 地理位置数据
|
||||
export interface GeoData {
|
||||
location: string;
|
||||
visits: number;
|
||||
visitors: number;
|
||||
percentage: number;
|
||||
}
|
||||
|
||||
// 设备分析数据
|
||||
export interface DeviceAnalytics {
|
||||
deviceTypes: Array<{
|
||||
type: DeviceType;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}>;
|
||||
browsers: Array<{
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}>;
|
||||
operatingSystems: Array<{
|
||||
name: string;
|
||||
count: number;
|
||||
percentage: number;
|
||||
}>;
|
||||
}
|
||||
76
scripts/db/sql/clickhouse/create_shorturl_analytics.sql
Normal file
76
scripts/db/sql/clickhouse/create_shorturl_analytics.sql
Normal file
@@ -0,0 +1,76 @@
|
||||
-- 创建数据库（如果不存在）
|
||||
CREATE DATABASE IF NOT EXISTS shorturl_analytics;
|
||||
|
||||
-- 切换到shorturl_analytics数据库
|
||||
USE shorturl_analytics;
|
||||
|
||||
-- 删除已存在的表
|
||||
DROP TABLE IF EXISTS shorturl_analytics.events;
|
||||
|
||||
-- 创建新表
|
||||
CREATE TABLE IF NOT EXISTS shorturl_analytics.events (
|
||||
-- 事件基础信息
|
||||
event_id String,
|
||||
event_time DateTime64(3),
|
||||
-- 精确到毫秒的时间戳
|
||||
event_type String,
|
||||
-- click, redirect, conversion, error
|
||||
event_attributes String DEFAULT '{}',
|
||||
-- 链接基本信息
|
||||
link_id String,
|
||||
link_slug String,
|
||||
-- 新增slug
|
||||
link_label String,
|
||||
-- 新增label
|
||||
link_title String,
|
||||
link_original_url String,
|
||||
link_attributes String DEFAULT '{}',
|
||||
link_created_at DateTime64(3),
|
||||
-- 精确到毫秒的时间戳
|
||||
link_expires_at Nullable(DateTime64(3)),
|
||||
-- 精确到毫秒的时间戳
|
||||
link_tags String DEFAULT '[]',
|
||||
-- Array of {id, name, attributes}
|
||||
-- 用户信息
|
||||
user_id String,
|
||||
user_name String,
|
||||
user_email String,
|
||||
user_attributes String DEFAULT '{}',
|
||||
-- 团队信息
|
||||
team_id String,
|
||||
team_name String,
|
||||
team_attributes String DEFAULT '{}',
|
||||
-- 项目信息
|
||||
project_id String,
|
||||
project_name String,
|
||||
project_attributes String DEFAULT '{}',
|
||||
-- QR码信息
|
||||
qr_code_id String,
|
||||
qr_code_name String,
|
||||
qr_code_attributes String DEFAULT '{}',
|
||||
-- 访问者信息
|
||||
visitor_id String,
|
||||
session_id String,
|
||||
ip_address String,
|
||||
country String,
|
||||
city String,
|
||||
device_type String,
|
||||
-- 改为String类型
|
||||
browser String,
|
||||
os String,
|
||||
user_agent String,
|
||||
-- 来源信息
|
||||
referrer String,
|
||||
utm_source String,
|
||||
utm_medium String,
|
||||
utm_campaign String,
|
||||
-- 交互信息
|
||||
time_spent_sec UInt32 DEFAULT 0,
|
||||
is_bounce Boolean DEFAULT true,
|
||||
is_qr_scan Boolean DEFAULT false,
|
||||
conversion_type String,
|
||||
-- 改为String类型
|
||||
conversion_value Float64 DEFAULT 0
|
||||
) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) -- 直接使用DateTime64进行分区
|
||||
ORDER BY
|
||||
(event_time, link_id, event_id) SETTINGS index_granularity = 8192;
|
||||
@@ -1,146 +0,0 @@
|
||||
-- 添加team、project和qrcode表到limq数据库
|
||||
USE limq;
|
||||
|
||||
-- 团队表
|
||||
CREATE TABLE IF NOT EXISTS limq.teams (
|
||||
team_id String,
|
||||
name String,
|
||||
created_at DateTime,
|
||||
created_by String,
|
||||
description String DEFAULT '',
|
||||
avatar_url String DEFAULT '',
|
||||
is_active Boolean DEFAULT true,
|
||||
plan_type Enum8(
|
||||
'free' = 1,
|
||||
'pro' = 2,
|
||||
'enterprise' = 3
|
||||
),
|
||||
members_count UInt32 DEFAULT 1,
|
||||
PRIMARY KEY (team_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
team_id SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 项目表
|
||||
CREATE TABLE IF NOT EXISTS limq.projects (
|
||||
project_id String,
|
||||
team_id String,
|
||||
name String,
|
||||
created_at DateTime,
|
||||
created_by String,
|
||||
description String DEFAULT '',
|
||||
is_archived Boolean DEFAULT false,
|
||||
links_count UInt32 DEFAULT 0,
|
||||
total_clicks UInt64 DEFAULT 0,
|
||||
last_updated DateTime DEFAULT now(),
|
||||
PRIMARY KEY (project_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
(project_id, team_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- QR码表 (扩展现有的qr_scans表)
|
||||
CREATE TABLE IF NOT EXISTS limq.qrcodes (
|
||||
qr_code_id String,
|
||||
link_id String,
|
||||
team_id String,
|
||||
project_id String DEFAULT '',
|
||||
name String,
|
||||
description String DEFAULT '',
|
||||
created_at DateTime,
|
||||
created_by String,
|
||||
updated_at DateTime DEFAULT now(),
|
||||
qr_type Enum8(
|
||||
'standard' = 1,
|
||||
'custom' = 2,
|
||||
'dynamic' = 3
|
||||
) DEFAULT 'standard',
|
||||
image_url String DEFAULT '',
|
||||
design_config String DEFAULT '{}',
|
||||
is_active Boolean DEFAULT true,
|
||||
total_scans UInt64 DEFAULT 0,
|
||||
unique_scanners UInt32 DEFAULT 0,
|
||||
PRIMARY KEY (qr_code_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
(qr_code_id, link_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 团队成员表
|
||||
CREATE TABLE IF NOT EXISTS limq.team_members (
|
||||
team_id String,
|
||||
user_id String,
|
||||
role Enum8(
|
||||
'owner' = 1,
|
||||
'admin' = 2,
|
||||
'editor' = 3,
|
||||
'viewer' = 4
|
||||
),
|
||||
joined_at DateTime DEFAULT now(),
|
||||
invited_by String,
|
||||
is_active Boolean DEFAULT true,
|
||||
last_active DateTime DEFAULT now(),
|
||||
PRIMARY KEY (team_id, user_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
(team_id, user_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 团队每日统计视图
|
||||
CREATE MATERIALIZED VIEW limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, team_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
l.team_id AS team_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(e.visitor_id) AS unique_visitors,
|
||||
countIf(e.event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(e.link_id) AS links_used,
|
||||
countIf(e.is_qr_scan) AS qr_scan_count
|
||||
FROM
|
||||
limq.link_events e
|
||||
JOIN limq.links l ON e.link_id = l.link_id
|
||||
WHERE
|
||||
l.team_id != ''
|
||||
GROUP BY
|
||||
date,
|
||||
l.team_id;
|
||||
|
||||
-- 项目每日统计视图
|
||||
CREATE MATERIALIZED VIEW limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, project_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
l.project_id AS project_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(e.visitor_id) AS unique_visitors,
|
||||
countIf(e.event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(e.link_id) AS links_used,
|
||||
countIf(e.is_qr_scan) AS qr_scan_count
|
||||
FROM
|
||||
limq.link_events e
|
||||
JOIN limq.links l ON e.link_id = l.link_id
|
||||
WHERE
|
||||
l.project_id != ''
|
||||
GROUP BY
|
||||
date,
|
||||
l.project_id;
|
||||
|
||||
-- QR码每日统计视图
|
||||
CREATE MATERIALIZED VIEW limq.qrcode_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, qr_code_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(scan_time) AS date,
|
||||
qr_code_id,
|
||||
count() AS total_scans,
|
||||
uniqExact(visitor_id) AS unique_scanners,
|
||||
countIf(led_to_conversion) AS conversions,
|
||||
countIf(device_type = 'mobile') AS mobile_scans,
|
||||
countIf(device_type = 'tablet') AS tablet_scans,
|
||||
countIf(device_type = 'desktop') AS desktop_scans,
|
||||
uniqExact(location) AS unique_locations
|
||||
FROM
|
||||
limq.qr_scans
|
||||
GROUP BY
|
||||
date,
|
||||
qr_code_id;
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
# 脚本名称: load-clickhouse-testdata.sh
|
||||
# 用途: 将测试数据加载到ClickHouse数据库中
|
||||
|
||||
# 设置脚本目录路径
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# 设置SQL文件路径
|
||||
SQL_FILE="$SCRIPT_DIR/sql/clickhouse/seed-clickhouse-analytics.sql"
|
||||
|
||||
# 检查SQL文件是否存在
|
||||
if [ ! -f "$SQL_FILE" ]; then
|
||||
echo "错误: SQL文件 '$SQL_FILE' 不存在"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 执行CH查询脚本
|
||||
echo "开始加载测试数据到ClickHouse数据库..."
|
||||
bash "$SCRIPT_DIR/sql/clickhouse/ch-query.sh" -f "$SQL_FILE"
|
||||
|
||||
# 检查执行结果
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "测试数据已成功加载到ClickHouse数据库"
|
||||
else
|
||||
echo "错误: 加载测试数据失败"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -1,997 +0,0 @@
|
||||
-- 移动端点击访问事件
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 10:25:30',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-123',
|
||||
's-456',
|
||||
'click',
|
||||
'103.45.67.89',
|
||||
'China',
|
||||
'Shanghai',
|
||||
'https://www.google.com',
|
||||
'google',
|
||||
'organic',
|
||||
'none',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
45,
|
||||
false,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 11:32:21',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-124',
|
||||
's-457',
|
||||
'click',
|
||||
'43.78.123.45',
|
||||
'Japan',
|
||||
'Tokyo',
|
||||
'https://twitter.com',
|
||||
'twitter',
|
||||
'social',
|
||||
'spring_promo',
|
||||
'Mozilla/5.0 (Android 10)',
|
||||
'mobile',
|
||||
'Chrome',
|
||||
'Android',
|
||||
15,
|
||||
true,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 14:15:45',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-125',
|
||||
's-458',
|
||||
'click',
|
||||
'72.34.67.81',
|
||||
'US',
|
||||
'New York',
|
||||
'https://www.facebook.com',
|
||||
'facebook',
|
||||
'social',
|
||||
'crypto_ad',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
120,
|
||||
false,
|
||||
false,
|
||||
'interact',
|
||||
0
|
||||
);
|
||||
|
||||
-- 桌面设备点击事件
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 08:45:12',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-126',
|
||||
's-459',
|
||||
'click',
|
||||
'89.67.43.21',
|
||||
'Germany',
|
||||
'Berlin',
|
||||
'https://www.reddit.com',
|
||||
'reddit',
|
||||
'referral',
|
||||
'none',
|
||||
'Mozilla/5.0 (Windows NT 10.0)',
|
||||
'desktop',
|
||||
'Chrome',
|
||||
'Windows',
|
||||
300,
|
||||
false,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 16:20:33',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-127',
|
||||
's-460',
|
||||
'click',
|
||||
'178.65.43.12',
|
||||
'UK',
|
||||
'London',
|
||||
'https://www.linkedin.com',
|
||||
'linkedin',
|
||||
'social',
|
||||
'biz_campaign',
|
||||
'Mozilla/5.0 (Macintosh)',
|
||||
'desktop',
|
||||
'Safari',
|
||||
'MacOS',
|
||||
250,
|
||||
false,
|
||||
false,
|
||||
'stay',
|
||||
0
|
||||
);
|
||||
|
||||
-- 平板设备点击事件
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 13:10:55',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-128',
|
||||
's-461',
|
||||
'click',
|
||||
'156.78.34.12',
|
||||
'Canada',
|
||||
'Toronto',
|
||||
'https://www.youtube.com',
|
||||
'youtube',
|
||||
'video',
|
||||
'tutorial',
|
||||
'Mozilla/5.0 (iPad)',
|
||||
'tablet',
|
||||
'Safari',
|
||||
'iOS',
|
||||
180,
|
||||
false,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
-- QR扫描访问事件
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 09:30:22',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_qr',
|
||||
'v-129',
|
||||
's-462',
|
||||
'click',
|
||||
'101.56.78.90',
|
||||
'China',
|
||||
'Beijing',
|
||||
'direct',
|
||||
'qr',
|
||||
'print',
|
||||
'offline_event',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
75,
|
||||
false,
|
||||
true,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
-- 转化事件
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 10:27:45',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-123',
|
||||
's-456',
|
||||
'conversion',
|
||||
'103.45.67.89',
|
||||
'China',
|
||||
'Shanghai',
|
||||
'https://www.google.com',
|
||||
'google',
|
||||
'organic',
|
||||
'none',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
120,
|
||||
false,
|
||||
false,
|
||||
'signup',
|
||||
50
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-15 08:52:18',
|
||||
'2025-03-15',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-126',
|
||||
's-459',
|
||||
'conversion',
|
||||
'89.67.43.21',
|
||||
'Germany',
|
||||
'Berlin',
|
||||
'https://www.reddit.com',
|
||||
'reddit',
|
||||
'referral',
|
||||
'none',
|
||||
'Mozilla/5.0 (Windows NT 10.0)',
|
||||
'desktop',
|
||||
'Chrome',
|
||||
'Windows',
|
||||
450,
|
||||
false,
|
||||
false,
|
||||
'purchase',
|
||||
150.75
|
||||
);
|
||||
|
||||
-- 第二天的数据 (3/16)
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-16 11:15:30',
|
||||
'2025-03-16',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-130',
|
||||
's-463',
|
||||
'click',
|
||||
'178.91.45.67',
|
||||
'France',
|
||||
'Paris',
|
||||
'https://www.google.com',
|
||||
'google',
|
||||
'organic',
|
||||
'none',
|
||||
'Mozilla/5.0 (Android 11)',
|
||||
'mobile',
|
||||
'Chrome',
|
||||
'Android',
|
||||
60,
|
||||
false,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-16 14:22:45',
|
||||
'2025-03-16',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-131',
|
||||
's-464',
|
||||
'click',
|
||||
'89.123.45.78',
|
||||
'Spain',
|
||||
'Madrid',
|
||||
'https://www.instagram.com',
|
||||
'instagram',
|
||||
'social',
|
||||
'influencer',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
90,
|
||||
false,
|
||||
false,
|
||||
'interact',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-16 16:40:12',
|
||||
'2025-03-16',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-131',
|
||||
's-464',
|
||||
'conversion',
|
||||
'89.123.45.78',
|
||||
'Spain',
|
||||
'Madrid',
|
||||
'https://www.instagram.com',
|
||||
'instagram',
|
||||
'social',
|
||||
'influencer',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
200,
|
||||
false,
|
||||
false,
|
||||
'subscription',
|
||||
75.50
|
||||
);
|
||||
|
||||
-- 第三天数据 (3/17)
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-17 09:10:22',
|
||||
'2025-03-17',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-132',
|
||||
's-465',
|
||||
'click',
|
||||
'45.67.89.123',
|
||||
'US',
|
||||
'Los Angeles',
|
||||
'https://www.google.com',
|
||||
'google',
|
||||
'cpc',
|
||||
'spring_sale',
|
||||
'Mozilla/5.0 (Windows NT 10.0)',
|
||||
'desktop',
|
||||
'Edge',
|
||||
'Windows',
|
||||
150,
|
||||
false,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-17 12:30:45',
|
||||
'2025-03-17',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-133',
|
||||
's-466',
|
||||
'click',
|
||||
'67.89.123.45',
|
||||
'Brazil',
|
||||
'Sao Paulo',
|
||||
'https://www.yahoo.com',
|
||||
'yahoo',
|
||||
'organic',
|
||||
'none',
|
||||
'Mozilla/5.0 (iPad)',
|
||||
'tablet',
|
||||
'Safari',
|
||||
'iOS',
|
||||
120,
|
||||
false,
|
||||
false,
|
||||
'stay',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-17 15:45:33',
|
||||
'2025-03-17',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-132',
|
||||
's-465',
|
||||
'conversion',
|
||||
'45.67.89.123',
|
||||
'US',
|
||||
'Los Angeles',
|
||||
'https://www.google.com',
|
||||
'google',
|
||||
'cpc',
|
||||
'spring_sale',
|
||||
'Mozilla/5.0 (Windows NT 10.0)',
|
||||
'desktop',
|
||||
'Edge',
|
||||
'Windows',
|
||||
300,
|
||||
false,
|
||||
false,
|
||||
'purchase',
|
||||
225.50
|
||||
);
|
||||
|
||||
-- 添加一周前的数据 (对比期)
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-08 10:25:30',
|
||||
'2025-03-08',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-140',
|
||||
's-470',
|
||||
'click',
|
||||
'103.45.67.89',
|
||||
'China',
|
||||
'Shanghai',
|
||||
'https://www.google.com',
|
||||
'google',
|
||||
'organic',
|
||||
'none',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
30,
|
||||
false,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-08 11:32:21',
|
||||
'2025-03-08',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-141',
|
||||
's-471',
|
||||
'click',
|
||||
'89.67.43.21',
|
||||
'Germany',
|
||||
'Berlin',
|
||||
'https://www.reddit.com',
|
||||
'reddit',
|
||||
'referral',
|
||||
'none',
|
||||
'Mozilla/5.0 (Windows NT 10.0)',
|
||||
'desktop',
|
||||
'Chrome',
|
||||
'Windows',
|
||||
200,
|
||||
false,
|
||||
false,
|
||||
'visit',
|
||||
0
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
conversion_type,
|
||||
conversion_value
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
generateUUIDv4(),
|
||||
'2025-03-08 13:10:55',
|
||||
'2025-03-08',
|
||||
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
|
||||
'ch_main',
|
||||
'v-140',
|
||||
's-470',
|
||||
'conversion',
|
||||
'103.45.67.89',
|
||||
'China',
|
||||
'Shanghai',
|
||||
'https://www.google.com',
|
||||
'google',
|
||||
'organic',
|
||||
'none',
|
||||
'Mozilla/5.0 (iPhone)',
|
||||
'mobile',
|
||||
'Safari',
|
||||
'iOS',
|
||||
100,
|
||||
false,
|
||||
false,
|
||||
'purchase',
|
||||
100.00
|
||||
);
|
||||
@@ -1,122 +0,0 @@
|
||||
-- 修改设备类型字段从枚举类型更改为字符串类型
|
||||
-- 先删除依赖于link_events表的物化视图
|
||||
DROP TABLE IF EXISTS limq.platform_distribution;
|
||||
|
||||
DROP TABLE IF EXISTS limq.link_hourly_patterns;
|
||||
|
||||
DROP TABLE IF EXISTS limq.link_daily_stats;
|
||||
|
||||
DROP TABLE IF EXISTS limq.team_daily_stats;
|
||||
|
||||
DROP TABLE IF EXISTS limq.project_daily_stats;
|
||||
|
||||
-- 修改link_events表的device_type字段
|
||||
ALTER TABLE
|
||||
limq.link_events
|
||||
MODIFY
|
||||
COLUMN device_type String;
|
||||
|
||||
-- 重新创建物化视图
|
||||
-- 每日链接汇总视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, link_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
link_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(visitor_id) AS unique_visitors,
|
||||
uniqExact(session_id) AS unique_sessions,
|
||||
sum(time_spent_sec) AS total_time_spent,
|
||||
avg(time_spent_sec) AS avg_time_spent,
|
||||
countIf(is_bounce) AS bounce_count,
|
||||
countIf(event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(referrer) AS unique_referrers,
|
||||
countIf(device_type = 'mobile') AS mobile_count,
|
||||
countIf(device_type = 'tablet') AS tablet_count,
|
||||
countIf(device_type = 'desktop') AS desktop_count,
|
||||
countIf(is_qr_scan) AS qr_scan_count,
|
||||
sum(conversion_value) AS total_conversion_value
|
||||
FROM
|
||||
limq.link_events
|
||||
GROUP BY
|
||||
date,
|
||||
link_id;
|
||||
|
||||
-- 每小时访问模式视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_hourly_patterns ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, hour, link_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
toHour(event_time) AS hour,
|
||||
link_id,
|
||||
count() AS visits,
|
||||
uniqExact(visitor_id) AS unique_visitors
|
||||
FROM
|
||||
limq.link_events
|
||||
GROUP BY
|
||||
date,
|
||||
hour,
|
||||
link_id;
|
||||
|
||||
-- 平台分布视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.platform_distribution ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, utm_source, device_type) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
utm_source,
|
||||
device_type,
|
||||
count() AS visits,
|
||||
uniqExact(visitor_id) AS unique_visitors
|
||||
FROM
|
||||
limq.link_events
|
||||
WHERE
|
||||
utm_source != ''
|
||||
GROUP BY
|
||||
date,
|
||||
utm_source,
|
||||
device_type;
|
||||
|
||||
-- 团队每日统计视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, team_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
l.team_id AS team_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(e.visitor_id) AS unique_visitors,
|
||||
countIf(e.event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(e.link_id) AS links_used,
|
||||
countIf(e.is_qr_scan) AS qr_scan_count
|
||||
FROM
|
||||
limq.link_events e
|
||||
JOIN limq.links l ON e.link_id = l.link_id
|
||||
WHERE
|
||||
l.team_id != ''
|
||||
GROUP BY
|
||||
date,
|
||||
l.team_id;
|
||||
|
||||
-- 项目每日统计视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, project_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
l.project_id AS project_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(e.visitor_id) AS unique_visitors,
|
||||
countIf(e.event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(e.link_id) AS links_used,
|
||||
countIf(e.is_qr_scan) AS qr_scan_count
|
||||
FROM
|
||||
limq.link_events e
|
||||
JOIN limq.links l ON e.link_id = l.link_id
|
||||
WHERE
|
||||
l.project_id != ''
|
||||
GROUP BY
|
||||
date,
|
||||
l.project_id;
|
||||
@@ -1,379 +0,0 @@
|
||||
-- 删除所有物化视图（需要先删除视图，因为它们依赖于表）
|
||||
DROP TABLE IF EXISTS limq.platform_distribution;
|
||||
|
||||
DROP TABLE IF EXISTS limq.link_hourly_patterns;
|
||||
|
||||
DROP TABLE IF EXISTS limq.link_daily_stats;
|
||||
|
||||
DROP TABLE IF EXISTS limq.team_daily_stats;
|
||||
|
||||
DROP TABLE IF EXISTS limq.project_daily_stats;
|
||||
|
||||
DROP TABLE IF EXISTS limq.qrcode_daily_stats;
|
||||
|
||||
-- 删除所有表
|
||||
DROP TABLE IF EXISTS limq.qr_scans;
|
||||
|
||||
DROP TABLE IF EXISTS limq.sessions;
|
||||
|
||||
DROP TABLE IF EXISTS limq.link_events;
|
||||
|
||||
DROP TABLE IF EXISTS limq.links;
|
||||
|
||||
DROP TABLE IF EXISTS limq.teams;
|
||||
|
||||
DROP TABLE IF EXISTS limq.projects;
|
||||
|
||||
DROP TABLE IF EXISTS limq.qrcodes;
|
||||
|
||||
DROP TABLE IF EXISTS limq.team_members;
|
||||
|
||||
DROP TABLE IF EXISTS limq.users;
|
||||
|
||||
-- 创建数据库（如果不存在）
|
||||
CREATE DATABASE IF NOT EXISTS limq;
|
||||
|
||||
-- 切换到limq数据库
|
||||
USE limq;
|
||||
|
||||
-- 创建短链接访问事件表
|
||||
CREATE TABLE IF NOT EXISTS limq.link_events (
|
||||
event_id UUID DEFAULT generateUUIDv4(),
|
||||
event_time DateTime64(3) DEFAULT now64(),
|
||||
date Date DEFAULT toDate(event_time),
|
||||
link_id String,
|
||||
channel_id String,
|
||||
visitor_id String,
|
||||
session_id String,
|
||||
event_type Enum8(
|
||||
'click' = 1,
|
||||
'redirect' = 2,
|
||||
'conversion' = 3,
|
||||
'error' = 4
|
||||
),
|
||||
-- 访问者信息
|
||||
ip_address String,
|
||||
country String,
|
||||
city String,
|
||||
-- 来源信息
|
||||
referrer String,
|
||||
utm_source String,
|
||||
utm_medium String,
|
||||
utm_campaign String,
|
||||
-- 设备信息
|
||||
user_agent String,
|
||||
device_type Enum8(
|
||||
'mobile' = 1,
|
||||
'tablet' = 2,
|
||||
'desktop' = 3,
|
||||
'other' = 4
|
||||
),
|
||||
browser String,
|
||||
os String,
|
||||
-- 交互信息
|
||||
time_spent_sec UInt32 DEFAULT 0,
|
||||
is_bounce Boolean DEFAULT true,
|
||||
-- QR码相关
|
||||
is_qr_scan Boolean DEFAULT false,
|
||||
qr_code_id String DEFAULT '',
|
||||
-- 转化数据
|
||||
conversion_type Enum8(
|
||||
'visit' = 1,
|
||||
'stay' = 2,
|
||||
'interact' = 3,
|
||||
'signup' = 4,
|
||||
'subscription' = 5,
|
||||
'purchase' = 6
|
||||
) DEFAULT 'visit',
|
||||
conversion_value Float64 DEFAULT 0,
|
||||
-- 其他属性
|
||||
custom_data String DEFAULT '{}'
|
||||
) ENGINE = MergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, link_id, event_time) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 短链接维度表
|
||||
CREATE TABLE IF NOT EXISTS limq.links (
|
||||
link_id String,
|
||||
original_url String,
|
||||
created_at DateTime64(3),
|
||||
created_by String,
|
||||
title String,
|
||||
description String,
|
||||
tags Array(String),
|
||||
is_active Boolean DEFAULT true,
|
||||
expires_at Nullable(DateTime64(3)),
|
||||
team_id String DEFAULT '',
|
||||
project_id String DEFAULT '',
|
||||
PRIMARY KEY (link_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
link_id SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 会话跟踪表
|
||||
CREATE TABLE IF NOT EXISTS limq.sessions (
|
||||
session_id String,
|
||||
visitor_id String,
|
||||
link_id String,
|
||||
started_at DateTime64(3),
|
||||
last_activity DateTime64(3),
|
||||
ended_at Nullable(DateTime64(3)),
|
||||
duration_sec UInt32 DEFAULT 0,
|
||||
session_pages UInt8 DEFAULT 1,
|
||||
is_completed Boolean DEFAULT false,
|
||||
PRIMARY KEY (session_id)
|
||||
) ENGINE = ReplacingMergeTree(last_activity)
|
||||
ORDER BY
|
||||
(session_id, link_id, visitor_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- QR码统计表
|
||||
CREATE TABLE IF NOT EXISTS limq.qr_scans (
|
||||
scan_id UUID DEFAULT generateUUIDv4(),
|
||||
qr_code_id String,
|
||||
link_id String,
|
||||
scan_time DateTime64(3),
|
||||
visitor_id String,
|
||||
location String,
|
||||
device_type Enum8(
|
||||
'mobile' = 1,
|
||||
'tablet' = 2,
|
||||
'desktop' = 3,
|
||||
'other' = 4
|
||||
),
|
||||
led_to_conversion Boolean DEFAULT false,
|
||||
PRIMARY KEY (scan_id)
|
||||
) ENGINE = MergeTree() PARTITION BY toYYYYMM(scan_time)
|
||||
ORDER BY
|
||||
scan_id SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 团队表
|
||||
CREATE TABLE IF NOT EXISTS limq.teams (
|
||||
team_id String,
|
||||
name String,
|
||||
created_at DateTime,
|
||||
created_by String,
|
||||
description String DEFAULT '',
|
||||
avatar_url String DEFAULT '',
|
||||
is_active Boolean DEFAULT true,
|
||||
plan_type Enum8(
|
||||
'free' = 1,
|
||||
'pro' = 2,
|
||||
'enterprise' = 3
|
||||
),
|
||||
members_count UInt32 DEFAULT 1,
|
||||
PRIMARY KEY (team_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
team_id SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 项目表
|
||||
CREATE TABLE IF NOT EXISTS limq.projects (
|
||||
project_id String,
|
||||
team_id String,
|
||||
name String,
|
||||
created_at DateTime,
|
||||
created_by String,
|
||||
description String DEFAULT '',
|
||||
is_archived Boolean DEFAULT false,
|
||||
links_count UInt32 DEFAULT 0,
|
||||
total_clicks UInt64 DEFAULT 0,
|
||||
last_updated DateTime DEFAULT now(),
|
||||
PRIMARY KEY (project_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
(project_id, team_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- QR码表
|
||||
CREATE TABLE IF NOT EXISTS limq.qrcodes (
|
||||
qr_code_id String,
|
||||
link_id String,
|
||||
team_id String,
|
||||
project_id String DEFAULT '',
|
||||
name String,
|
||||
description String DEFAULT '',
|
||||
created_at DateTime,
|
||||
created_by String,
|
||||
updated_at DateTime DEFAULT now(),
|
||||
qr_type Enum8(
|
||||
'standard' = 1,
|
||||
'custom' = 2,
|
||||
'dynamic' = 3
|
||||
) DEFAULT 'standard',
|
||||
image_url String DEFAULT '',
|
||||
design_config String DEFAULT '{}',
|
||||
is_active Boolean DEFAULT true,
|
||||
total_scans UInt64 DEFAULT 0,
|
||||
unique_scanners UInt32 DEFAULT 0,
|
||||
PRIMARY KEY (qr_code_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
(qr_code_id, link_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 团队成员表
|
||||
CREATE TABLE IF NOT EXISTS limq.team_members (
|
||||
team_id String,
|
||||
user_id String,
|
||||
role Enum8(
|
||||
'owner' = 1,
|
||||
'admin' = 2,
|
||||
'editor' = 3,
|
||||
'viewer' = 4
|
||||
),
|
||||
joined_at DateTime DEFAULT now(),
|
||||
invited_by String,
|
||||
is_active Boolean DEFAULT true,
|
||||
last_active DateTime DEFAULT now(),
|
||||
PRIMARY KEY (team_id, user_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
(team_id, user_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 用户表
|
||||
CREATE TABLE IF NOT EXISTS limq.users (
|
||||
user_id String,
|
||||
username String,
|
||||
email String,
|
||||
full_name String,
|
||||
avatar_url String DEFAULT '',
|
||||
created_at DateTime,
|
||||
last_login DateTime DEFAULT now(),
|
||||
is_active Boolean DEFAULT true,
|
||||
is_verified Boolean DEFAULT false,
|
||||
auth_provider Enum8(
|
||||
'email' = 1,
|
||||
'google' = 2,
|
||||
'github' = 3,
|
||||
'microsoft' = 4
|
||||
) DEFAULT 'email',
|
||||
roles Array(String) DEFAULT [ 'user' ],
|
||||
preferences String DEFAULT '{}',
|
||||
teams_count UInt32 DEFAULT 0,
|
||||
links_created UInt32 DEFAULT 0,
|
||||
PRIMARY KEY (user_id)
|
||||
) ENGINE = ReplacingMergeTree()
|
||||
ORDER BY
|
||||
user_id SETTINGS index_granularity = 8192;
|
||||
|
||||
-- 每日链接汇总视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, link_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
link_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(visitor_id) AS unique_visitors,
|
||||
uniqExact(session_id) AS unique_sessions,
|
||||
sum(time_spent_sec) AS total_time_spent,
|
||||
avg(time_spent_sec) AS avg_time_spent,
|
||||
countIf(is_bounce) AS bounce_count,
|
||||
countIf(event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(referrer) AS unique_referrers,
|
||||
countIf(device_type = 'mobile') AS mobile_count,
|
||||
countIf(device_type = 'tablet') AS tablet_count,
|
||||
countIf(device_type = 'desktop') AS desktop_count,
|
||||
countIf(is_qr_scan) AS qr_scan_count,
|
||||
sum(conversion_value) AS total_conversion_value
|
||||
FROM
|
||||
limq.link_events
|
||||
GROUP BY
|
||||
date,
|
||||
link_id;
|
||||
|
||||
-- 每小时访问模式视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_hourly_patterns ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, hour, link_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
toHour(event_time) AS hour,
|
||||
link_id,
|
||||
count() AS visits,
|
||||
uniqExact(visitor_id) AS unique_visitors
|
||||
FROM
|
||||
limq.link_events
|
||||
GROUP BY
|
||||
date,
|
||||
hour,
|
||||
link_id;
|
||||
|
||||
-- 平台分布视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.platform_distribution ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, utm_source, device_type) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
utm_source,
|
||||
device_type,
|
||||
count() AS visits,
|
||||
uniqExact(visitor_id) AS unique_visitors
|
||||
FROM
|
||||
limq.link_events
|
||||
WHERE
|
||||
utm_source != ''
|
||||
GROUP BY
|
||||
date,
|
||||
utm_source,
|
||||
device_type;
|
||||
|
||||
-- 团队每日统计视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, team_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
l.team_id AS team_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(e.visitor_id) AS unique_visitors,
|
||||
countIf(e.event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(e.link_id) AS links_used,
|
||||
countIf(e.is_qr_scan) AS qr_scan_count
|
||||
FROM
|
||||
limq.link_events e
|
||||
JOIN limq.links l ON e.link_id = l.link_id
|
||||
WHERE
|
||||
l.team_id != ''
|
||||
GROUP BY
|
||||
date,
|
||||
l.team_id;
|
||||
|
||||
-- 项目每日统计视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, project_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(event_time) AS date,
|
||||
l.project_id AS project_id,
|
||||
count() AS total_clicks,
|
||||
uniqExact(e.visitor_id) AS unique_visitors,
|
||||
countIf(e.event_type = 'conversion') AS conversion_count,
|
||||
uniqExact(e.link_id) AS links_used,
|
||||
countIf(e.is_qr_scan) AS qr_scan_count
|
||||
FROM
|
||||
limq.link_events e
|
||||
JOIN limq.links l ON e.link_id = l.link_id
|
||||
WHERE
|
||||
l.project_id != ''
|
||||
GROUP BY
|
||||
date,
|
||||
l.project_id;
|
||||
|
||||
-- QR码每日统计视图
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.qrcode_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
|
||||
ORDER BY
|
||||
(date, qr_code_id) SETTINGS index_granularity = 8192 AS
|
||||
SELECT
|
||||
toDate(scan_time) AS date,
|
||||
qr_code_id,
|
||||
count() AS total_scans,
|
||||
uniqExact(visitor_id) AS unique_scanners,
|
||||
countIf(led_to_conversion) AS conversions,
|
||||
countIf(device_type = 'mobile') AS mobile_scans,
|
||||
countIf(device_type = 'tablet') AS tablet_scans,
|
||||
countIf(device_type = 'desktop') AS desktop_scans,
|
||||
uniqExact(location) AS unique_locations
|
||||
FROM
|
||||
limq.qr_scans
|
||||
GROUP BY
|
||||
date,
|
||||
qr_code_id;
|
||||
@@ -1,828 +0,0 @@
|
||||
-- 清空现有数据（可选）
|
||||
TRUNCATE TABLE IF EXISTS limq.link_events;
|
||||
|
||||
TRUNCATE TABLE IF EXISTS limq.link_daily_stats;
|
||||
|
||||
TRUNCATE TABLE IF EXISTS limq.link_hourly_patterns;
|
||||
|
||||
TRUNCATE TABLE IF EXISTS limq.links;
|
||||
|
||||
-- 使用固定的UUID值插入链接
|
||||
INSERT INTO
|
||||
limq.links (
|
||||
link_id,
|
||||
original_url,
|
||||
created_at,
|
||||
created_by,
|
||||
title,
|
||||
description,
|
||||
tags,
|
||||
is_active
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
'11111111-1111-1111-1111-111111111111',
|
||||
'https://example.com/page1',
|
||||
now(),
|
||||
'user-1',
|
||||
'产品页面',
|
||||
'我们的主要产品页面',
|
||||
[ '产品',
|
||||
'营销' ],
|
||||
true
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.links (
|
||||
link_id,
|
||||
original_url,
|
||||
created_at,
|
||||
created_by,
|
||||
title,
|
||||
description,
|
||||
tags,
|
||||
is_active
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
'22222222-2222-2222-2222-222222222222',
|
||||
'https://example.com/promo',
|
||||
now(),
|
||||
'user-1',
|
||||
'促销活动',
|
||||
'夏季特别促销活动',
|
||||
[ '促销',
|
||||
'活动' ],
|
||||
true
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.links (
|
||||
link_id,
|
||||
original_url,
|
||||
created_at,
|
||||
created_by,
|
||||
title,
|
||||
description,
|
||||
tags,
|
||||
is_active
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
'33333333-3333-3333-3333-333333333333',
|
||||
'https://example.com/blog',
|
||||
now(),
|
||||
'user-2',
|
||||
'公司博客',
|
||||
'公司新闻和更新',
|
||||
[ '博客',
|
||||
'内容' ],
|
||||
true
|
||||
);
|
||||
|
||||
INSERT INTO
|
||||
limq.links (
|
||||
link_id,
|
||||
original_url,
|
||||
created_at,
|
||||
created_by,
|
||||
title,
|
||||
description,
|
||||
tags,
|
||||
is_active
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
'44444444-4444-4444-4444-444444444444',
|
||||
'https://example.com/signup',
|
||||
now(),
|
||||
'user-2',
|
||||
'注册页面',
|
||||
'新用户注册页面',
|
||||
[ '转化',
|
||||
'注册' ],
|
||||
true
|
||||
);
|
||||
|
||||
-- 为第一个链接创建500条记录
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
qr_code_id,
|
||||
conversion_type,
|
||||
conversion_value,
|
||||
custom_data
|
||||
)
|
||||
SELECT
|
||||
generateUUIDv4() AS event_id,
|
||||
subtractDays(now(), rand() % 30) AS event_time,
|
||||
toDate(event_time) AS date,
|
||||
'11111111-1111-1111-1111-111111111111' AS link_id,
|
||||
'channel-1' AS channel_id,
|
||||
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
|
||||
concat('session-', toString(number % 50 + 1)) AS session_id,
|
||||
multiIf(
|
||||
rand() % 100 < 70,
|
||||
'click',
|
||||
rand() % 100 < 90,
|
||||
'redirect',
|
||||
rand() % 100 < 98,
|
||||
'conversion',
|
||||
'error'
|
||||
) AS event_type,
|
||||
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'China',
|
||||
rand() % 100 < 85,
|
||||
'US',
|
||||
rand() % 100 < 95,
|
||||
'Japan',
|
||||
'Other'
|
||||
) AS country,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'Beijing',
|
||||
rand() % 100 < 85,
|
||||
'New York',
|
||||
rand() % 100 < 95,
|
||||
'Tokyo',
|
||||
'Other'
|
||||
) AS city,
|
||||
multiIf(
|
||||
rand() % 100 < 30,
|
||||
'https://google.com',
|
||||
rand() % 100 < 50,
|
||||
'https://facebook.com',
|
||||
rand() % 100 < 65,
|
||||
'https://twitter.com',
|
||||
rand() % 100 < 75,
|
||||
'https://instagram.com',
|
||||
rand() % 100 < 85,
|
||||
'https://linkedin.com',
|
||||
rand() % 100 < 90,
|
||||
'https://bing.com',
|
||||
rand() % 100 < 95,
|
||||
'https://baidu.com',
|
||||
'direct'
|
||||
) AS referrer,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'google',
|
||||
rand() % 100 < 70,
|
||||
'facebook',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_source,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'cpc',
|
||||
rand() % 100 < 70,
|
||||
'social',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_medium,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'summer_sale',
|
||||
rand() % 100 < 70,
|
||||
'product_launch',
|
||||
rand() % 100 < 90,
|
||||
'newsletter',
|
||||
'brand'
|
||||
) AS utm_campaign,
|
||||
'Mozilla/5.0' AS user_agent,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'mobile',
|
||||
rand() % 100 < 85,
|
||||
'desktop',
|
||||
rand() % 100 < 95,
|
||||
'tablet',
|
||||
'other'
|
||||
) AS device_type,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'Chrome',
|
||||
rand() % 100 < 80,
|
||||
'Safari',
|
||||
rand() % 100 < 95,
|
||||
'Firefox',
|
||||
'Edge'
|
||||
) AS browser,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'iOS',
|
||||
rand() % 100 < 90,
|
||||
'Android',
|
||||
'Windows'
|
||||
) AS os,
|
||||
rand() % 300 AS time_spent_sec,
|
||||
rand() % 100 < 25 AS is_bounce,
|
||||
rand() % 100 < 20 AS is_qr_scan,
|
||||
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'visit',
|
||||
rand() % 100 < 70,
|
||||
'stay',
|
||||
rand() % 100 < 85,
|
||||
'interact',
|
||||
rand() % 100 < 93,
|
||||
'signup',
|
||||
rand() % 100 < 97,
|
||||
'subscription',
|
||||
'purchase'
|
||||
) AS conversion_type,
|
||||
rand() % 100 * 1.5 AS conversion_value,
|
||||
'{}' AS custom_data
|
||||
FROM
|
||||
numbers(500);
|
||||
|
||||
-- 为第二个链接创建300条记录
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
qr_code_id,
|
||||
conversion_type,
|
||||
conversion_value,
|
||||
custom_data
|
||||
)
|
||||
SELECT
|
||||
generateUUIDv4() AS event_id,
|
||||
subtractDays(now(), rand() % 30) AS event_time,
|
||||
toDate(event_time) AS date,
|
||||
'22222222-2222-2222-2222-222222222222' AS link_id,
|
||||
'channel-1' AS channel_id,
|
||||
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
|
||||
concat('session-', toString(number % 40 + 1)) AS session_id,
|
||||
multiIf(
|
||||
rand() % 100 < 70,
|
||||
'click',
|
||||
rand() % 100 < 90,
|
||||
'redirect',
|
||||
rand() % 100 < 98,
|
||||
'conversion',
|
||||
'error'
|
||||
) AS event_type,
|
||||
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'China',
|
||||
rand() % 100 < 85,
|
||||
'US',
|
||||
rand() % 100 < 95,
|
||||
'Japan',
|
||||
'Other'
|
||||
) AS country,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'Beijing',
|
||||
rand() % 100 < 85,
|
||||
'New York',
|
||||
rand() % 100 < 95,
|
||||
'Tokyo',
|
||||
'Other'
|
||||
) AS city,
|
||||
multiIf(
|
||||
rand() % 100 < 30,
|
||||
'https://google.com',
|
||||
rand() % 100 < 50,
|
||||
'https://facebook.com',
|
||||
rand() % 100 < 65,
|
||||
'https://twitter.com',
|
||||
rand() % 100 < 75,
|
||||
'https://instagram.com',
|
||||
rand() % 100 < 85,
|
||||
'https://linkedin.com',
|
||||
rand() % 100 < 90,
|
||||
'https://bing.com',
|
||||
rand() % 100 < 95,
|
||||
'https://baidu.com',
|
||||
'direct'
|
||||
) AS referrer,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'google',
|
||||
rand() % 100 < 70,
|
||||
'facebook',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_source,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'cpc',
|
||||
rand() % 100 < 70,
|
||||
'social',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_medium,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'summer_sale',
|
||||
rand() % 100 < 70,
|
||||
'product_launch',
|
||||
rand() % 100 < 90,
|
||||
'newsletter',
|
||||
'brand'
|
||||
) AS utm_campaign,
|
||||
'Mozilla/5.0' AS user_agent,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'mobile',
|
||||
rand() % 100 < 85,
|
||||
'desktop',
|
||||
rand() % 100 < 95,
|
||||
'tablet',
|
||||
'other'
|
||||
) AS device_type,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'Chrome',
|
||||
rand() % 100 < 80,
|
||||
'Safari',
|
||||
rand() % 100 < 95,
|
||||
'Firefox',
|
||||
'Edge'
|
||||
) AS browser,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'iOS',
|
||||
rand() % 100 < 90,
|
||||
'Android',
|
||||
'Windows'
|
||||
) AS os,
|
||||
rand() % 300 AS time_spent_sec,
|
||||
rand() % 100 < 25 AS is_bounce,
|
||||
rand() % 100 < 15 AS is_qr_scan,
|
||||
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'visit',
|
||||
rand() % 100 < 70,
|
||||
'stay',
|
||||
rand() % 100 < 85,
|
||||
'interact',
|
||||
rand() % 100 < 93,
|
||||
'signup',
|
||||
rand() % 100 < 97,
|
||||
'subscription',
|
||||
'purchase'
|
||||
) AS conversion_type,
|
||||
rand() % 100 * 2.5 AS conversion_value,
|
||||
'{}' AS custom_data
|
||||
FROM
|
||||
numbers(300);
|
||||
|
||||
-- 为第三个链接创建200条记录
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
qr_code_id,
|
||||
conversion_type,
|
||||
conversion_value,
|
||||
custom_data
|
||||
)
|
||||
SELECT
|
||||
generateUUIDv4() AS event_id,
|
||||
subtractDays(now(), rand() % 30) AS event_time,
|
||||
toDate(event_time) AS date,
|
||||
'33333333-3333-3333-3333-333333333333' AS link_id,
|
||||
'channel-2' AS channel_id,
|
||||
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
|
||||
concat('session-', toString(number % 30 + 1)) AS session_id,
|
||||
multiIf(
|
||||
rand() % 100 < 70,
|
||||
'click',
|
||||
rand() % 100 < 90,
|
||||
'redirect',
|
||||
rand() % 100 < 98,
|
||||
'conversion',
|
||||
'error'
|
||||
) AS event_type,
|
||||
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'China',
|
||||
rand() % 100 < 85,
|
||||
'US',
|
||||
rand() % 100 < 95,
|
||||
'Japan',
|
||||
'Other'
|
||||
) AS country,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'Beijing',
|
||||
rand() % 100 < 85,
|
||||
'New York',
|
||||
rand() % 100 < 95,
|
||||
'Tokyo',
|
||||
'Other'
|
||||
) AS city,
|
||||
multiIf(
|
||||
rand() % 100 < 30,
|
||||
'https://google.com',
|
||||
rand() % 100 < 50,
|
||||
'https://facebook.com',
|
||||
rand() % 100 < 65,
|
||||
'https://twitter.com',
|
||||
rand() % 100 < 75,
|
||||
'https://instagram.com',
|
||||
rand() % 100 < 85,
|
||||
'https://linkedin.com',
|
||||
rand() % 100 < 90,
|
||||
'https://bing.com',
|
||||
rand() % 100 < 95,
|
||||
'https://baidu.com',
|
||||
'direct'
|
||||
) AS referrer,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'google',
|
||||
rand() % 100 < 70,
|
||||
'facebook',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_source,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'cpc',
|
||||
rand() % 100 < 70,
|
||||
'social',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_medium,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'summer_sale',
|
||||
rand() % 100 < 70,
|
||||
'product_launch',
|
||||
rand() % 100 < 90,
|
||||
'newsletter',
|
||||
'brand'
|
||||
) AS utm_campaign,
|
||||
'Mozilla/5.0' AS user_agent,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'mobile',
|
||||
rand() % 100 < 85,
|
||||
'desktop',
|
||||
rand() % 100 < 95,
|
||||
'tablet',
|
||||
'other'
|
||||
) AS device_type,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'Chrome',
|
||||
rand() % 100 < 80,
|
||||
'Safari',
|
||||
rand() % 100 < 95,
|
||||
'Firefox',
|
||||
'Edge'
|
||||
) AS browser,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'iOS',
|
||||
rand() % 100 < 90,
|
||||
'Android',
|
||||
'Windows'
|
||||
) AS os,
|
||||
rand() % 600 AS time_spent_sec,
|
||||
rand() % 100 < 15 AS is_bounce,
|
||||
rand() % 100 < 10 AS is_qr_scan,
|
||||
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'visit',
|
||||
rand() % 100 < 70,
|
||||
'stay',
|
||||
rand() % 100 < 85,
|
||||
'interact',
|
||||
rand() % 100 < 93,
|
||||
'signup',
|
||||
rand() % 100 < 97,
|
||||
'subscription',
|
||||
'purchase'
|
||||
) AS conversion_type,
|
||||
rand() % 100 * 1.2 AS conversion_value,
|
||||
'{}' AS custom_data
|
||||
FROM
|
||||
numbers(200);
|
||||
|
||||
-- 为第四个链接创建400条记录
|
||||
INSERT INTO
|
||||
limq.link_events (
|
||||
event_id,
|
||||
event_time,
|
||||
date,
|
||||
link_id,
|
||||
channel_id,
|
||||
visitor_id,
|
||||
session_id,
|
||||
event_type,
|
||||
ip_address,
|
||||
country,
|
||||
city,
|
||||
referrer,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
user_agent,
|
||||
device_type,
|
||||
browser,
|
||||
os,
|
||||
time_spent_sec,
|
||||
is_bounce,
|
||||
is_qr_scan,
|
||||
qr_code_id,
|
||||
conversion_type,
|
||||
conversion_value,
|
||||
custom_data
|
||||
)
|
||||
SELECT
|
||||
generateUUIDv4() AS event_id,
|
||||
subtractDays(now(), rand() % 30) AS event_time,
|
||||
toDate(event_time) AS date,
|
||||
'44444444-4444-4444-4444-444444444444' AS link_id,
|
||||
'channel-2' AS channel_id,
|
||||
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
|
||||
concat('session-', toString(number % 60 + 1)) AS session_id,
|
||||
multiIf(
|
||||
rand() % 100 < 70,
|
||||
'click',
|
||||
rand() % 100 < 90,
|
||||
'redirect',
|
||||
rand() % 100 < 98,
|
||||
'conversion',
|
||||
'error'
|
||||
) AS event_type,
|
||||
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'China',
|
||||
rand() % 100 < 85,
|
||||
'US',
|
||||
rand() % 100 < 95,
|
||||
'Japan',
|
||||
'Other'
|
||||
) AS country,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'Beijing',
|
||||
rand() % 100 < 85,
|
||||
'New York',
|
||||
rand() % 100 < 95,
|
||||
'Tokyo',
|
||||
'Other'
|
||||
) AS city,
|
||||
multiIf(
|
||||
rand() % 100 < 30,
|
||||
'https://google.com',
|
||||
rand() % 100 < 50,
|
||||
'https://facebook.com',
|
||||
rand() % 100 < 65,
|
||||
'https://twitter.com',
|
||||
rand() % 100 < 75,
|
||||
'https://instagram.com',
|
||||
rand() % 100 < 85,
|
||||
'https://linkedin.com',
|
||||
rand() % 100 < 90,
|
||||
'https://bing.com',
|
||||
rand() % 100 < 95,
|
||||
'https://baidu.com',
|
||||
'direct'
|
||||
) AS referrer,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'google',
|
||||
rand() % 100 < 70,
|
||||
'facebook',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_source,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'cpc',
|
||||
rand() % 100 < 70,
|
||||
'social',
|
||||
rand() % 100 < 90,
|
||||
'email',
|
||||
'direct'
|
||||
) AS utm_medium,
|
||||
multiIf(
|
||||
rand() % 100 < 40,
|
||||
'summer_sale',
|
||||
rand() % 100 < 70,
|
||||
'product_launch',
|
||||
rand() % 100 < 90,
|
||||
'newsletter',
|
||||
'brand'
|
||||
) AS utm_campaign,
|
||||
'Mozilla/5.0' AS user_agent,
|
||||
multiIf(
|
||||
rand() % 100 < 60,
|
||||
'mobile',
|
||||
rand() % 100 < 85,
|
||||
'desktop',
|
||||
rand() % 100 < 95,
|
||||
'tablet',
|
||||
'other'
|
||||
) AS device_type,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'Chrome',
|
||||
rand() % 100 < 80,
|
||||
'Safari',
|
||||
rand() % 100 < 95,
|
||||
'Firefox',
|
||||
'Edge'
|
||||
) AS browser,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'iOS',
|
||||
rand() % 100 < 90,
|
||||
'Android',
|
||||
'Windows'
|
||||
) AS os,
|
||||
rand() % 400 AS time_spent_sec,
|
||||
rand() % 100 < 20 AS is_bounce,
|
||||
rand() % 100 < 25 AS is_qr_scan,
|
||||
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
|
||||
multiIf(
|
||||
rand() % 100 < 50,
|
||||
'visit',
|
||||
rand() % 100 < 70,
|
||||
'stay',
|
||||
rand() % 100 < 85,
|
||||
'interact',
|
||||
rand() % 100 < 93,
|
||||
'signup',
|
||||
rand() % 100 < 97,
|
||||
'subscription',
|
||||
'purchase'
|
||||
) AS conversion_type,
|
||||
rand() % 100 * 3.5 AS conversion_value,
|
||||
'{}' AS custom_data
|
||||
FROM
|
||||
numbers(400);
|
||||
|
||||
-- 插入link_daily_stats表数据
|
||||
INSERT INTO
|
||||
limq.link_daily_stats (
|
||||
date,
|
||||
link_id,
|
||||
total_clicks,
|
||||
unique_visitors,
|
||||
unique_sessions,
|
||||
total_time_spent,
|
||||
avg_time_spent,
|
||||
bounce_count,
|
||||
conversion_count,
|
||||
unique_referrers,
|
||||
mobile_count,
|
||||
tablet_count,
|
||||
desktop_count,
|
||||
qr_scan_count,
|
||||
total_conversion_value
|
||||
)
|
||||
SELECT
|
||||
subtractDays(today(), number) AS date,
|
||||
multiIf(
|
||||
number % 4 = 0,
|
||||
'11111111-1111-1111-1111-111111111111',
|
||||
number % 4 = 1,
|
||||
'22222222-2222-2222-2222-222222222222',
|
||||
number % 4 = 2,
|
||||
'33333333-3333-3333-3333-333333333333',
|
||||
'44444444-4444-4444-4444-444444444444'
|
||||
) AS link_id,
|
||||
50 + rand() % 100 AS total_clicks,
|
||||
30 + rand() % 50 AS unique_visitors,
|
||||
20 + rand() % 40 AS unique_sessions,
|
||||
(500 + rand() % 1000) * 60 AS total_time_spent,
|
||||
(rand() % 10) * 60 + rand() % 60 AS avg_time_spent,
|
||||
5 + rand() % 20 AS bounce_count,
|
||||
rand() % 30 AS conversion_count,
|
||||
3 + rand() % 8 AS unique_referrers,
|
||||
20 + rand() % 40 AS mobile_count,
|
||||
5 + rand() % 15 AS tablet_count,
|
||||
15 + rand() % 30 AS desktop_count,
|
||||
rand() % 10 AS qr_scan_count,
|
||||
rand() % 1000 * 2.5 AS total_conversion_value
|
||||
FROM
|
||||
numbers(30)
|
||||
WHERE
|
||||
number < 30;
|
||||
|
||||
-- 插入link_hourly_patterns表数据
|
||||
INSERT INTO
|
||||
limq.link_hourly_patterns (date, hour, link_id, visits, unique_visitors)
|
||||
SELECT
|
||||
subtractDays(today(), number % 7) AS date,
|
||||
number % 24 AS hour,
|
||||
multiIf(
|
||||
intDiv(number, 24) % 4 = 0,
|
||||
'11111111-1111-1111-1111-111111111111',
|
||||
intDiv(number, 24) % 4 = 1,
|
||||
'22222222-2222-2222-2222-222222222222',
|
||||
intDiv(number, 24) % 4 = 2,
|
||||
'33333333-3333-3333-3333-333333333333',
|
||||
'44444444-4444-4444-4444-444444444444'
|
||||
) AS link_id,
|
||||
5 + rand() % 20 AS visits,
|
||||
3 + rand() % 10 AS unique_visitors
|
||||
FROM
|
||||
numbers(672) -- 7天 x 24小时 x 4个链接
|
||||
WHERE
|
||||
number < 672;
|
||||
|
||||
-- 显示数据行数，验证插入成功
|
||||
SELECT
|
||||
'link_events 表行数:' AS metric,
|
||||
count() AS value
|
||||
FROM
|
||||
limq.link_events
|
||||
UNION
|
||||
ALL
|
||||
SELECT
|
||||
'link_daily_stats 表行数:',
|
||||
count()
|
||||
FROM
|
||||
limq.link_daily_stats
|
||||
UNION
|
||||
ALL
|
||||
SELECT
|
||||
'link_hourly_patterns 表行数:',
|
||||
count()
|
||||
FROM
|
||||
limq.link_hourly_patterns;
|
||||
364
scripts/db/sync_mongo_to_events.ts
Normal file
364
scripts/db/sync_mongo_to_events.ts
Normal file
@@ -0,0 +1,364 @@
|
||||
// Sync data from MongoDB trace table to ClickHouse events table
|
||||
import { getVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
interface MongoConfig {
|
||||
host: string;
|
||||
port: string;
|
||||
db: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
interface ClickHouseConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_database: string;
|
||||
clickhouse_url: string;
|
||||
}
|
||||
|
||||
interface TraceRecord {
|
||||
_id: ObjectId;
|
||||
slugId: ObjectId;
|
||||
label: string | null;
|
||||
ip: string;
|
||||
type: number;
|
||||
platform: string;
|
||||
platformOS: string;
|
||||
browser: string;
|
||||
browserVersion: string;
|
||||
url: string;
|
||||
createTime: number;
|
||||
}
|
||||
|
||||
export async function main(
|
||||
batch_size = 1000,
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
|
||||
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
|
||||
|
||||
// Set timeout
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// Get MongoDB and ClickHouse connection info
|
||||
let mongoConfig: MongoConfig;
|
||||
let clickhouseConfig: ClickHouseConfig;
|
||||
|
||||
try {
|
||||
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
|
||||
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
|
||||
|
||||
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
|
||||
} catch (error) {
|
||||
console.error("Failed to get config:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Build MongoDB connection URL
|
||||
let mongoUrl = "mongodb://";
|
||||
if (mongoConfig.username && mongoConfig.password) {
|
||||
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
|
||||
}
|
||||
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
|
||||
|
||||
// Connect to MongoDB
|
||||
const client = new MongoClient();
|
||||
try {
|
||||
await client.connect(mongoUrl);
|
||||
console.log("MongoDB connected successfully");
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const traceCollection = db.collection<TraceRecord>("trace");
|
||||
|
||||
// Build query conditions
|
||||
const query: Record<string, unknown> = {
|
||||
type: 1 // Only sync records with type 1
|
||||
};
|
||||
|
||||
// Count total records
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`Found ${totalRecords} records to sync`);
|
||||
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`Will process ${recordsToProcess} records`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("No records to sync, task completed");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "No records to sync"
|
||||
};
|
||||
}
|
||||
|
||||
// Check ClickHouse connection
|
||||
const checkClickHouseConnection = async (): Promise<boolean> => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("Skipping ClickHouse connection check");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("Testing ClickHouse connection...");
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
|
||||
},
|
||||
body: "SELECT 1",
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse connection test successful");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Check if records exist in ClickHouse
|
||||
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
|
||||
return records;
|
||||
}
|
||||
|
||||
try {
|
||||
const recordIds = records.map(record => record._id.toString());
|
||||
|
||||
const query = `
|
||||
SELECT event_id
|
||||
FROM ${clickhouseConfig.clickhouse_database}.events
|
||||
WHERE event_attributes LIKE '%"mongo_id":"%'
|
||||
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
const existingIds = new Set(result.data.map((row: any) => {
|
||||
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
|
||||
return matches ? matches[1] : null;
|
||||
}).filter(Boolean));
|
||||
|
||||
return records.filter(record => !existingIds.has(record._id.toString()));
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
|
||||
return skip_clickhouse_check ? records : [];
|
||||
}
|
||||
};
|
||||
|
||||
// Process records function
|
||||
const processRecords = async (records: TraceRecord[]) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
const newRecords = await checkExistingRecords(records);
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("All records already exist, skipping");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare ClickHouse insert data
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const eventTime = new Date(record.createTime).toISOString();
|
||||
return {
|
||||
event_time: eventTime,
|
||||
event_type: "click",
|
||||
event_attributes: JSON.stringify({
|
||||
mongo_id: record._id.toString(),
|
||||
original_type: record.type
|
||||
}),
|
||||
|
||||
// Link information
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: "",
|
||||
link_label: record.label || "",
|
||||
link_title: "",
|
||||
link_original_url: record.url || "",
|
||||
link_attributes: "{}",
|
||||
link_created_at: eventTime,
|
||||
link_expires_at: null,
|
||||
link_tags: "[]",
|
||||
|
||||
// User information (empty as not available in trace)
|
||||
user_id: "",
|
||||
user_name: "",
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
|
||||
// Team information (empty as not available in trace)
|
||||
team_id: "",
|
||||
team_name: "",
|
||||
team_attributes: "{}",
|
||||
|
||||
// Project information (empty as not available in trace)
|
||||
project_id: "",
|
||||
project_name: "",
|
||||
project_attributes: "{}",
|
||||
|
||||
// QR code information (empty as not available in trace)
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
|
||||
// Visitor information
|
||||
visitor_id: record._id.toString(),
|
||||
session_id: `${record._id.toString()}-${record.createTime}`,
|
||||
ip_address: record.ip || "",
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "",
|
||||
os: record.platformOS || "",
|
||||
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
|
||||
|
||||
// Source information
|
||||
referrer: record.url || "",
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
|
||||
// Interaction information
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
conversion_type: "visit",
|
||||
conversion_value: 0
|
||||
};
|
||||
});
|
||||
|
||||
// Generate ClickHouse insert SQL
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
|
||||
FORMAT JSONEachRow
|
||||
${JSON.stringify(clickhouseData)}
|
||||
`;
|
||||
|
||||
try {
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: insertSQL,
|
||||
signal: AbortSignal.timeout(20000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// Check ClickHouse connection before processing
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
throw new Error("ClickHouse connection failed, cannot continue sync");
|
||||
}
|
||||
|
||||
// Process records in batches
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
|
||||
break;
|
||||
}
|
||||
|
||||
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
const records = await traceCollection.find(
|
||||
query,
|
||||
{
|
||||
allowDiskUse: true,
|
||||
sort: { createTime: 1 },
|
||||
skip: page * batch_size,
|
||||
limit: batch_size
|
||||
}
|
||||
).toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("No more records found, sync complete");
|
||||
break;
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
message: "Data sync completed"
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("Error during sync:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
stack: err instanceof Error ? err.stack : undefined
|
||||
};
|
||||
} finally {
|
||||
await client.close();
|
||||
console.log("MongoDB connection closed");
|
||||
}
|
||||
}
|
||||
409
windmill/sync_mongo_to_events.ts
Normal file
409
windmill/sync_mongo_to_events.ts
Normal file
@@ -0,0 +1,409 @@
|
||||
// Sync data from MongoDB trace table to ClickHouse events table
|
||||
import { getVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
interface MongoConfig {
|
||||
host: string;
|
||||
port: string;
|
||||
db: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
interface ClickHouseConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_url: string;
|
||||
}
|
||||
|
||||
interface TraceRecord {
|
||||
_id: ObjectId;
|
||||
slugId: ObjectId;
|
||||
label: string | null;
|
||||
ip: string;
|
||||
type: number;
|
||||
platform: string;
|
||||
platformOS: string;
|
||||
browser: string;
|
||||
browserVersion: string;
|
||||
url: string;
|
||||
createTime: number;
|
||||
}
|
||||
|
||||
interface ShortRecord {
|
||||
_id: ObjectId;
|
||||
slug: string; // 短链接的slug部分
|
||||
origin: string; // 原始URL
|
||||
domain?: string; // 域名
|
||||
createTime: number; // 创建时间戳
|
||||
user?: string; // 创建用户
|
||||
title?: string; // 标题
|
||||
description?: string; // 描述
|
||||
tags?: string[]; // 标签
|
||||
active?: boolean; // 是否活跃
|
||||
expiresAt?: number; // 过期时间戳
|
||||
teamId?: string; // 团队ID
|
||||
projectId?: string; // 项目ID
|
||||
}
|
||||
|
||||
interface ClickHouseRow {
|
||||
event_id: string;
|
||||
event_attributes: string;
|
||||
}
|
||||
|
||||
export async function main(
|
||||
batch_size = 1000,
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
|
||||
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
|
||||
|
||||
// Set timeout
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// Get MongoDB and ClickHouse connection info
|
||||
let mongoConfig: MongoConfig;
|
||||
let clickhouseConfig: ClickHouseConfig;
|
||||
|
||||
try {
|
||||
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
|
||||
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
|
||||
|
||||
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
|
||||
} catch (error) {
|
||||
console.error("Failed to get config:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Build MongoDB connection URL
|
||||
let mongoUrl = "mongodb://";
|
||||
if (mongoConfig.username && mongoConfig.password) {
|
||||
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
|
||||
}
|
||||
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
|
||||
|
||||
// Connect to MongoDB
|
||||
const client = new MongoClient();
|
||||
try {
|
||||
await client.connect(mongoUrl);
|
||||
console.log("MongoDB connected successfully");
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const traceCollection = db.collection<TraceRecord>("trace");
|
||||
const shortCollection = db.collection<ShortRecord>("short");
|
||||
|
||||
// Build query conditions
|
||||
const query: Record<string, unknown> = {
|
||||
type: 1 // Only sync records with type 1
|
||||
};
|
||||
|
||||
// Count total records
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`Found ${totalRecords} records to sync`);
|
||||
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`Will process ${recordsToProcess} records`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("No records to sync, task completed");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "No records to sync"
|
||||
};
|
||||
}
|
||||
|
||||
// Check ClickHouse connection
|
||||
const checkClickHouseConnection = async (): Promise<boolean> => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("Skipping ClickHouse connection check");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("Testing ClickHouse connection...");
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
|
||||
},
|
||||
body: "SELECT 1",
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse connection test successful");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Check if records exist in ClickHouse
|
||||
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
|
||||
return records;
|
||||
}
|
||||
|
||||
try {
|
||||
const recordIds = records.map(record => record._id.toString());
|
||||
|
||||
const query = `
|
||||
SELECT event_id
|
||||
FROM shorturl_analytics.events
|
||||
WHERE event_attributes LIKE '%"mongo_id":"%'
|
||||
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
const existingIds = new Set(result.data.map((row: ClickHouseRow) => {
|
||||
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
|
||||
return matches ? matches[1] : null;
|
||||
}).filter(Boolean));
|
||||
|
||||
return records.filter(record => !existingIds.has(record._id.toString()));
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
|
||||
return skip_clickhouse_check ? records : [];
|
||||
}
|
||||
};
|
||||
|
||||
// Process records function
|
||||
const processRecords = async (records: TraceRecord[]) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
const newRecords = await checkExistingRecords(records);
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("All records already exist, skipping");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Get link information for all records
|
||||
const slugIds = newRecords.map(record => record.slugId);
|
||||
const shortLinks = await shortCollection.find({
|
||||
_id: { $in: slugIds }
|
||||
}).toArray();
|
||||
|
||||
// Create a map for quick lookup
|
||||
const shortLinksMap = new Map(shortLinks.map(link => [link._id.toString(), link]));
|
||||
|
||||
// Prepare ClickHouse insert data
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString());
|
||||
|
||||
// 将毫秒时间戳转换为 DateTime64(3) 格式
|
||||
const formatDateTime = (timestamp: number) => {
|
||||
return new Date(timestamp).toISOString().replace('T', ' ').replace('Z', '');
|
||||
};
|
||||
|
||||
return {
|
||||
// Event base information
|
||||
event_id: record._id.toString(),
|
||||
event_time: formatDateTime(record.createTime),
|
||||
event_type: "click",
|
||||
event_attributes: JSON.stringify({
|
||||
original_type: record.type
|
||||
}),
|
||||
|
||||
// Link information from short collection
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: shortLink?.slug || "",
|
||||
link_label: record.label || "",
|
||||
link_title: "",
|
||||
link_original_url: shortLink?.origin || "",
|
||||
link_attributes: JSON.stringify({
|
||||
domain: shortLink?.domain || null
|
||||
}),
|
||||
link_created_at: shortLink?.createTime ? formatDateTime(shortLink.createTime) : formatDateTime(record.createTime),
|
||||
link_expires_at: shortLink?.expiresAt ? formatDateTime(shortLink.expiresAt) : null,
|
||||
link_tags: "[]", // Empty array as default
|
||||
|
||||
// User information
|
||||
user_id: shortLink?.user || "",
|
||||
user_name: "",
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
|
||||
// Team information
|
||||
team_id: shortLink?.teamId || "",
|
||||
team_name: "",
|
||||
team_attributes: "{}",
|
||||
|
||||
// Project information
|
||||
project_id: shortLink?.projectId || "",
|
||||
project_name: "",
|
||||
project_attributes: "{}",
|
||||
|
||||
// QR code information
|
||||
qr_code_id: "",
|
||||
qr_code_name: "",
|
||||
qr_code_attributes: "{}",
|
||||
|
||||
// Visitor information
|
||||
visitor_id: "", // Empty string as default
|
||||
session_id: `${record.slugId.toString()}-${record.createTime}`,
|
||||
ip_address: record.ip || "",
|
||||
country: "",
|
||||
city: "",
|
||||
device_type: record.platform || "",
|
||||
browser: record.browser || "",
|
||||
os: record.platformOS || "",
|
||||
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
|
||||
|
||||
// Source information
|
||||
referrer: record.url || "",
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
|
||||
// Interaction information
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
conversion_type: "visit",
|
||||
conversion_value: 0
|
||||
};
|
||||
});
|
||||
|
||||
// Generate ClickHouse insert SQL
|
||||
const rows = clickhouseData.map(row => {
|
||||
// 只需要处理JSON字符串的转义
|
||||
const formattedRow = {
|
||||
...row,
|
||||
event_attributes: row.event_attributes.replace(/\\/g, '\\\\'),
|
||||
link_attributes: row.link_attributes.replace(/\\/g, '\\\\')
|
||||
};
|
||||
return JSON.stringify(formattedRow);
|
||||
}).join('\n');
|
||||
|
||||
const insertSQL = `INSERT INTO shorturl_analytics.events FORMAT JSONEachRow\n${rows}`;
|
||||
|
||||
try {
|
||||
const response = await fetch(clickhouseConfig.clickhouse_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: insertSQL,
|
||||
signal: AbortSignal.timeout(20000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// Check ClickHouse connection before processing
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
throw new Error("ClickHouse connection failed, cannot continue sync");
|
||||
}
|
||||
|
||||
// Process records in batches
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
|
||||
break;
|
||||
}
|
||||
|
||||
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
const records = await traceCollection.find(
|
||||
query,
|
||||
{
|
||||
allowDiskUse: true,
|
||||
sort: { createTime: 1 },
|
||||
skip: page * batch_size,
|
||||
limit: batch_size
|
||||
}
|
||||
).toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("No more records found, sync complete");
|
||||
break;
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
message: "Data sync completed"
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("Error during sync:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
stack: err instanceof Error ? err.stack : undefined
|
||||
};
|
||||
} finally {
|
||||
await client.close();
|
||||
console.log("MongoDB connection closed");
|
||||
}
|
||||
}
|
||||
@@ -1,474 +0,0 @@
|
||||
// 从MongoDB的trace表同步数据到ClickHouse的link_events表
|
||||
import { getVariable, setVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
interface MongoConfig {
|
||||
host: string;
|
||||
port: string;
|
||||
db: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
interface ClickHouseConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_database: string;
|
||||
clickhouse_url: string;
|
||||
}
|
||||
|
||||
interface TraceRecord {
|
||||
_id: ObjectId;
|
||||
slugId: ObjectId;
|
||||
label: string | null;
|
||||
ip: string;
|
||||
type: number;
|
||||
platform: string;
|
||||
platformOS: string;
|
||||
browser: string;
|
||||
browserVersion: string;
|
||||
url: string;
|
||||
createTime: number;
|
||||
}
|
||||
|
||||
interface SyncState {
|
||||
last_sync_time: number;
|
||||
records_synced: number;
|
||||
last_sync_id?: string;
|
||||
}
|
||||
|
||||
export async function main(
|
||||
batch_size = 1000,
|
||||
max_records = 9999999,
|
||||
timeout_minutes = 60,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
|
||||
}
|
||||
if (force_insert) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
// 检查是否超时
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`运行时间超过${timeout_minutes}分钟，暂停执行`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// 获取MongoDB和ClickHouse的连接信息
|
||||
let mongoConfig: MongoConfig;
|
||||
let clickhouseConfig: ClickHouseConfig;
|
||||
|
||||
try {
|
||||
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
|
||||
console.log("原始MongoDB配置:", JSON.stringify(rawMongoConfig));
|
||||
|
||||
// 尝试解析配置，如果是字符串形式
|
||||
if (typeof rawMongoConfig === "string") {
|
||||
try {
|
||||
mongoConfig = JSON.parse(rawMongoConfig);
|
||||
} catch (e) {
|
||||
console.error("MongoDB配置解析失败:", e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
mongoConfig = rawMongoConfig as MongoConfig;
|
||||
}
|
||||
|
||||
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
console.log("原始ClickHouse配置:", JSON.stringify(rawClickhouseConfig));
|
||||
|
||||
// 尝试解析配置，如果是字符串形式
|
||||
if (typeof rawClickhouseConfig === "string") {
|
||||
try {
|
||||
clickhouseConfig = JSON.parse(rawClickhouseConfig);
|
||||
} catch (e) {
|
||||
console.error("ClickHouse配置解析失败:", e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
clickhouseConfig = rawClickhouseConfig as ClickHouseConfig;
|
||||
}
|
||||
|
||||
console.log("MongoDB配置解析为:", JSON.stringify(mongoConfig));
|
||||
console.log("ClickHouse配置解析为:", JSON.stringify(clickhouseConfig));
|
||||
} catch (error) {
|
||||
console.error("获取配置失败:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// 构建MongoDB连接URL
|
||||
let mongoUrl = "mongodb://";
|
||||
if (mongoConfig.username && mongoConfig.password) {
|
||||
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
|
||||
}
|
||||
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
|
||||
|
||||
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
|
||||
|
||||
// 连接MongoDB
|
||||
const client = new MongoClient();
|
||||
try {
|
||||
await client.connect(mongoUrl);
|
||||
console.log("MongoDB连接成功");
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const traceCollection = db.collection<TraceRecord>("trace");
|
||||
|
||||
// 构建查询条件，获取所有记录
|
||||
const query: Record<string, unknown> = {
|
||||
type: 1 // 只同步type为1的记录
|
||||
};
|
||||
|
||||
// 计算总记录数
|
||||
const totalRecords = await traceCollection.countDocuments(query);
|
||||
console.log(`找到 ${totalRecords} 条记录需要同步`);
|
||||
|
||||
// 限制此次处理的记录数量
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`本次将处理 ${recordsToProcess} 条记录`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("没有记录需要同步，任务完成");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
message: "没有记录需要同步"
|
||||
};
|
||||
}
|
||||
|
||||
// 检查ClickHouse连接状态
|
||||
const checkClickHouseConnection = async (): Promise<boolean> => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，不测试连接");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("测试ClickHouse连接...");
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
|
||||
},
|
||||
body: "SELECT 1",
|
||||
// 设置5秒超时
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse连接测试成功");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// 检查记录是否已经存在于ClickHouse中
|
||||
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
// 如果跳过ClickHouse检查或强制插入，则直接返回所有记录
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`已跳过ClickHouse重复检查，准备处理所有 ${records.length} 条记录`);
|
||||
return records;
|
||||
}
|
||||
|
||||
logWithTimestamp(`正在检查 ${records.length} 条记录是否已存在于ClickHouse中...`);
|
||||
|
||||
try {
|
||||
// 提取所有记录的ID
|
||||
const recordIds = records.map(record => record.slugId.toString()); // 使用slugId作为link_id查询
|
||||
logWithTimestamp(`待检查的记录ID: ${recordIds.join(', ')}`);
|
||||
|
||||
// 构建查询SQL，检查记录是否已存在，确保添加FORMAT JSON来获取正确的JSON格式响应
|
||||
const query = `
|
||||
SELECT link_id
|
||||
FROM ${clickhouseConfig.clickhouse_database}.link_events
|
||||
WHERE link_id IN ('${recordIds.join("','")}')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
logWithTimestamp(`执行ClickHouse查询: ${query.replace(/\n\s*/g, ' ')}`);
|
||||
|
||||
// 发送请求到ClickHouse，添加10秒超时
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse查询错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
// 获取响应文本以便记录
|
||||
const responseText = await response.text();
|
||||
logWithTimestamp(`ClickHouse查询响应: ${responseText.slice(0, 200)}${responseText.length > 200 ? '...' : ''}`);
|
||||
|
||||
if (!responseText.trim()) {
|
||||
logWithTimestamp("ClickHouse返回空响应，假定没有记录存在");
|
||||
return records; // 如果响应为空，假设没有记录
|
||||
}
|
||||
|
||||
// 解析结果
|
||||
let result;
|
||||
try {
|
||||
result = JSON.parse(responseText);
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse响应不是有效的JSON: ${responseText}`);
|
||||
throw new Error(`解析ClickHouse响应失败: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// 确保result有正确的结构
|
||||
if (!result.data) {
|
||||
logWithTimestamp(`ClickHouse响应缺少data字段: ${JSON.stringify(result)}`);
|
||||
return records; // 如果没有data字段，假设没有记录
|
||||
}
|
||||
|
||||
// 提取已存在的记录ID
|
||||
const existingIds = new Set(result.data.map((row: { link_id: string }) => row.link_id));
|
||||
|
||||
logWithTimestamp(`检测到 ${existingIds.size} 条记录已存在于ClickHouse中`);
|
||||
if (existingIds.size > 0) {
|
||||
logWithTimestamp(`已存在的记录ID: ${Array.from(existingIds).join(', ')}`);
|
||||
}
|
||||
|
||||
// 过滤出不存在的记录
|
||||
const newRecords = records.filter(record => !existingIds.has(record.slugId.toString())); // 使用slugId匹配link_id
|
||||
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
|
||||
|
||||
return newRecords;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`ClickHouse查询出错: ${error.message}`);
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，将继续处理所有记录");
|
||||
return records;
|
||||
} else {
|
||||
throw error; // 如果没有启用跳过检查，则抛出错误
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ ClickHouse连接测试失败，请启用skip_clickhouse_check=true参数来跳过连接检查");
|
||||
throw new Error("ClickHouse连接失败，无法继续同步");
|
||||
}
|
||||
|
||||
// 处理记录的函数
|
||||
const processRecords = async (records: TraceRecord[]) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
logWithTimestamp(`开始处理批次数据，共 ${records.length} 条记录...`);
|
||||
|
||||
// 检查记录是否已存在
|
||||
let newRecords;
|
||||
try {
|
||||
newRecords = await checkExistingRecords(records);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
|
||||
if (!skip_clickhouse_check && !force_insert) {
|
||||
throw error;
|
||||
}
|
||||
// 如果跳过检查或强制插入，则使用所有记录
|
||||
logWithTimestamp("将使用所有记录进行处理");
|
||||
newRecords = records;
|
||||
}
|
||||
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("所有记录都已存在，跳过处理");
|
||||
return 0;
|
||||
}
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
// 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
|
||||
return {
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
link_id: record.slugId.toString(),
|
||||
channel_id: record.label || "",
|
||||
visitor_id: record._id.toString(), // 使用MongoDB ID作为访客ID
|
||||
session_id: record._id.toString() + "-" + record.createTime, // 创建一个唯一会话ID
|
||||
event_type: record.type <= 4 ? record.type : 1, // 确保event_type在枚举范围内
|
||||
ip_address: record.ip,
|
||||
country: "", // 这些字段在MongoDB中不存在，使用默认值
|
||||
city: "",
|
||||
referrer: record.url || "",
|
||||
utm_source: "",
|
||||
utm_medium: "",
|
||||
utm_campaign: "",
|
||||
user_agent: record.browser + " " + record.browserVersion,
|
||||
device_type: record.platform || "unknown",
|
||||
browser: record.browser || "",
|
||||
os: record.platformOS || "",
|
||||
time_spent_sec: 0,
|
||||
is_bounce: true,
|
||||
is_qr_scan: false,
|
||||
qr_code_id: "",
|
||||
conversion_type: 1, // 默认为'visit'
|
||||
conversion_value: 0,
|
||||
custom_data: `{"mongo_id":"${record._id.toString()}"}`
|
||||
};
|
||||
});
|
||||
|
||||
// 生成ClickHouse插入SQL
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.link_events
|
||||
(link_id, channel_id, visitor_id, session_id, event_type, ip_address, country, city,
|
||||
referrer, utm_source, utm_medium, utm_campaign, user_agent, device_type, browser, os,
|
||||
time_spent_sec, is_bounce, is_qr_scan, qr_code_id, conversion_type, conversion_value, custom_data)
|
||||
VALUES ${clickhouseData.map(record => {
|
||||
// 确保所有字符串值都是字符串类型，并安全处理替换
|
||||
const safeReplace = (val: any): string => {
|
||||
// 确保值是字符串，如果是null或undefined则使用空字符串
|
||||
const str = val === null || val === undefined ? "" : String(val);
|
||||
// 安全替换单引号
|
||||
return str.replace(/'/g, "''");
|
||||
};
|
||||
|
||||
return `('${record.link_id}', '${safeReplace(record.channel_id)}', '${record.visitor_id}', '${record.session_id}',
|
||||
${record.event_type}, '${safeReplace(record.ip_address)}', '', '',
|
||||
'${safeReplace(record.referrer)}', '', '', '', '${safeReplace(record.user_agent)}', '${safeReplace(record.device_type)}',
|
||||
'${safeReplace(record.browser)}', '${safeReplace(record.os)}',
|
||||
0, true, false, '', 1, 0, '${safeReplace(record.custom_data)}')`;
|
||||
}).join(", ")}
|
||||
`;
|
||||
|
||||
if (insertSQL.length === 0) {
|
||||
console.log("没有新记录需要插入");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 发送请求到ClickHouse，添加20秒超时
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
try {
|
||||
logWithTimestamp("发送插入请求到ClickHouse...");
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: insertSQL,
|
||||
signal: AbortSignal.timeout(20000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`向ClickHouse插入数据失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// 批量处理记录
|
||||
let processedRecords = 0;
|
||||
let totalBatchRecords = 0;
|
||||
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录，因超时暂停执行`);
|
||||
break;
|
||||
}
|
||||
|
||||
// 每批次都输出进度
|
||||
logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
|
||||
const records = await traceCollection.find(
|
||||
query,
|
||||
{
|
||||
allowDiskUse: true,
|
||||
sort: { createTime: 1 },
|
||||
skip: page * batch_size,
|
||||
limit: batch_size
|
||||
}
|
||||
).toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp("没有找到更多数据，同步结束");
|
||||
break;
|
||||
}
|
||||
|
||||
// 找到数据，开始处理
|
||||
logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
|
||||
// 输出当前批次的部分数据信息
|
||||
if (records.length > 0) {
|
||||
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
|
||||
if (records.length > 1) {
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length;
|
||||
totalBatchRecords += batchSize;
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
message: "数据同步完成"
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("同步过程中发生错误:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
stack: err instanceof Error ? err.stack : undefined
|
||||
};
|
||||
} finally {
|
||||
// 关闭MongoDB连接
|
||||
await client.close();
|
||||
console.log("MongoDB连接已关闭");
|
||||
}
|
||||
}
|
||||
@@ -1,532 +0,0 @@
|
||||
// 从MongoDB的short表同步数据到ClickHouse的links表
|
||||
import { getVariable, setVariable } from "npm:windmill-client@1";
|
||||
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
|
||||
|
||||
interface MongoConfig {
|
||||
host: string;
|
||||
port: string;
|
||||
db: string;
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
interface ClickHouseConfig {
|
||||
clickhouse_host: string;
|
||||
clickhouse_port: number;
|
||||
clickhouse_user: string;
|
||||
clickhouse_password: string;
|
||||
clickhouse_database: string;
|
||||
clickhouse_url: string;
|
||||
}
|
||||
|
||||
interface ShortRecord {
|
||||
_id: ObjectId;
|
||||
slug: string; // 短链接的slug部分
|
||||
origin: string; // 原始URL
|
||||
domain?: string; // 域名
|
||||
createTime: number; // 创建时间戳
|
||||
user?: string; // 创建用户
|
||||
title?: string; // 标题
|
||||
description?: string; // 描述
|
||||
tags?: string[]; // 标签
|
||||
active?: boolean; // 是否活跃
|
||||
expiresAt?: number; // 过期时间戳
|
||||
teamId?: string; // 团队ID
|
||||
projectId?: string; // 项目ID
|
||||
}
|
||||
|
||||
interface SyncState {
|
||||
last_sync_time: number;
|
||||
records_synced: number;
|
||||
last_sync_id?: string;
|
||||
}
|
||||
|
||||
export async function main(
|
||||
batch_size = 100,
|
||||
initial_sync = false,
|
||||
max_records = 999999,
|
||||
timeout_minutes = 30,
|
||||
skip_clickhouse_check = false,
|
||||
force_insert = false
|
||||
) {
|
||||
const logWithTimestamp = (message: string) => {
|
||||
const now = new Date();
|
||||
console.log(`[${now.toISOString()}] ${message}`);
|
||||
};
|
||||
|
||||
logWithTimestamp("开始执行MongoDB到ClickHouse的短链接同步任务...");
|
||||
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式，不会检查记录是否已存在");
|
||||
}
|
||||
if (force_insert) {
|
||||
logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
|
||||
}
|
||||
|
||||
// 设置超时
|
||||
const startTime = Date.now();
|
||||
const timeoutMs = timeout_minutes * 60 * 1000;
|
||||
|
||||
// 检查是否超时
|
||||
const checkTimeout = () => {
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
console.log(`运行时间超过${timeout_minutes}分钟，暂停执行`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// 获取MongoDB和ClickHouse的连接信息
|
||||
let mongoConfig: MongoConfig;
|
||||
let clickhouseConfig: ClickHouseConfig;
|
||||
|
||||
try {
|
||||
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
|
||||
console.log("原始MongoDB配置:", typeof rawMongoConfig === "string" ? rawMongoConfig : JSON.stringify(rawMongoConfig));
|
||||
|
||||
// 尝试解析配置，如果是字符串形式
|
||||
if (typeof rawMongoConfig === "string") {
|
||||
try {
|
||||
mongoConfig = JSON.parse(rawMongoConfig);
|
||||
} catch (e) {
|
||||
console.error("MongoDB配置解析失败:", e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
mongoConfig = rawMongoConfig as MongoConfig;
|
||||
}
|
||||
|
||||
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
|
||||
console.log("原始ClickHouse配置:", typeof rawClickhouseConfig === "string" ? rawClickhouseConfig : JSON.stringify(rawClickhouseConfig));
|
||||
|
||||
// 尝试解析配置，如果是字符串形式
|
||||
if (typeof rawClickhouseConfig === "string") {
|
||||
try {
|
||||
clickhouseConfig = JSON.parse(rawClickhouseConfig);
|
||||
} catch (e) {
|
||||
console.error("ClickHouse配置解析失败:", e);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
clickhouseConfig = rawClickhouseConfig as ClickHouseConfig;
|
||||
}
|
||||
|
||||
console.log("MongoDB配置解析为:", JSON.stringify(mongoConfig));
|
||||
console.log("ClickHouse配置解析为:", JSON.stringify(clickhouseConfig));
|
||||
} catch (error) {
|
||||
console.error("获取配置失败:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// 构建MongoDB连接URL
|
||||
let mongoUrl = "mongodb://";
|
||||
if (mongoConfig.username && mongoConfig.password) {
|
||||
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
|
||||
}
|
||||
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
|
||||
|
||||
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
|
||||
|
||||
// 获取上次同步的状态
|
||||
let syncState: SyncState;
|
||||
try {
|
||||
const rawSyncState = await getVariable<string>("f/shorturl_analytics/clickhouse/shorturl_links_sync_state");
|
||||
try {
|
||||
syncState = JSON.parse(rawSyncState);
|
||||
console.log(`获取同步状态成功: 上次同步时间 ${new Date(syncState.last_sync_time).toISOString()}`);
|
||||
} catch (parseError) {
|
||||
console.error("解析同步状态失败:", parseError);
|
||||
throw parseError;
|
||||
}
|
||||
} catch (_unused_error) {
|
||||
console.log("未找到同步状态，创建初始同步状态");
|
||||
syncState = {
|
||||
last_sync_time: 0,
|
||||
records_synced: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// 如果强制从头开始同步
|
||||
if (initial_sync) {
|
||||
console.log("强制从头开始同步");
|
||||
syncState = {
|
||||
last_sync_time: 0,
|
||||
records_synced: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// 连接MongoDB
|
||||
const client = new MongoClient();
|
||||
try {
|
||||
await client.connect(mongoUrl);
|
||||
console.log("MongoDB连接成功");
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const shortCollection = db.collection<ShortRecord>("short");
|
||||
|
||||
// 构建查询条件，只查询新的记录
|
||||
const query: Record<string, unknown> = {};
|
||||
|
||||
if (syncState.last_sync_time > 0) {
|
||||
query.createTime = { $gt: syncState.last_sync_time };
|
||||
}
|
||||
|
||||
if (syncState.last_sync_id) {
|
||||
// 如果有上次同步的ID，则从该ID之后开始查询
|
||||
query._id = { $gt: new ObjectId(syncState.last_sync_id) };
|
||||
}
|
||||
|
||||
// 计算总记录数
|
||||
const totalRecords = await shortCollection.countDocuments(query);
|
||||
console.log(`找到 ${totalRecords} 条新短链接记录需要同步`);
|
||||
|
||||
// 限制此次处理的记录数量
|
||||
const recordsToProcess = Math.min(totalRecords, max_records);
|
||||
console.log(`本次将处理 ${recordsToProcess} 条记录`);
|
||||
|
||||
if (totalRecords === 0) {
|
||||
console.log("没有新记录需要同步，任务完成");
|
||||
return {
|
||||
success: true,
|
||||
records_synced: 0,
|
||||
total_synced: syncState.records_synced,
|
||||
message: "没有新记录需要同步"
|
||||
};
|
||||
}
|
||||
|
||||
// 分批处理记录
|
||||
let processedRecords = 0;
|
||||
let lastId: string | undefined;
|
||||
let lastCreateTime = syncState.last_sync_time;
|
||||
let totalBatchRecords = 0;
|
||||
|
||||
// 检查ClickHouse连接状态
|
||||
const checkClickHouseConnection = async (): Promise<boolean> => {
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，不测试连接");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
logWithTimestamp("测试ClickHouse连接...");
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
|
||||
},
|
||||
body: "SELECT 1 FORMAT JSON",
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
logWithTimestamp("ClickHouse连接测试成功");
|
||||
return true;
|
||||
} else {
|
||||
const errorText = await response.text();
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`ClickHouse连接测试失败: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// 检查记录是否已经存在于ClickHouse中
|
||||
const checkExistingRecords = async (records: ShortRecord[]): Promise<ShortRecord[]> => {
|
||||
if (records.length === 0) return [];
|
||||
|
||||
// 如果跳过ClickHouse检查或强制插入，则直接返回所有记录
|
||||
if (skip_clickhouse_check || force_insert) {
|
||||
logWithTimestamp(`已跳过ClickHouse重复检查，准备处理所有 ${records.length} 条记录`);
|
||||
return records;
|
||||
}
|
||||
|
||||
logWithTimestamp(`正在检查 ${records.length} 条短链接记录是否已存在于ClickHouse中...`);
|
||||
|
||||
try {
|
||||
// 提取所有记录的ID
|
||||
const recordIds = records.map(record => record._id.toString());
|
||||
logWithTimestamp(`待检查的短链接ID: ${recordIds.join(', ')}`);
|
||||
|
||||
// 构建查询SQL，检查记录是否已存在
|
||||
const query = `
|
||||
SELECT link_id
|
||||
FROM ${clickhouseConfig.clickhouse_database}.links
|
||||
WHERE link_id IN ('${recordIds.join("','")}')
|
||||
FORMAT JSON
|
||||
`;
|
||||
|
||||
logWithTimestamp(`执行ClickHouse查询: ${query.replace(/\n\s*/g, ' ')}`);
|
||||
|
||||
// 发送请求到ClickHouse
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: query,
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse查询错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
// 获取响应文本
|
||||
const responseText = await response.text();
|
||||
logWithTimestamp(`ClickHouse查询响应: ${responseText.slice(0, 200)}${responseText.length > 200 ? '...' : ''}`);
|
||||
|
||||
if (!responseText.trim()) {
|
||||
logWithTimestamp("ClickHouse返回空响应，假定没有记录存在");
|
||||
return records;
|
||||
}
|
||||
|
||||
// 解析结果
|
||||
let result;
|
||||
try {
|
||||
result = JSON.parse(responseText);
|
||||
} catch (err) {
|
||||
logWithTimestamp(`ClickHouse响应不是有效的JSON: ${responseText}`);
|
||||
throw new Error(`解析ClickHouse响应失败: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// 确保result有正确的结构
|
||||
if (!result.data) {
|
||||
logWithTimestamp(`ClickHouse响应缺少data字段: ${JSON.stringify(result)}`);
|
||||
return records;
|
||||
}
|
||||
|
||||
// 提取已存在的记录ID
|
||||
const existingIds = new Set(result.data.map((row: { link_id: string }) => row.link_id));
|
||||
|
||||
logWithTimestamp(`检测到 ${existingIds.size} 条记录已存在于ClickHouse中`);
|
||||
if (existingIds.size > 0) {
|
||||
logWithTimestamp(`已存在的记录ID: ${Array.from(existingIds).join(', ')}`);
|
||||
}
|
||||
|
||||
// 过滤出不存在的记录
|
||||
const newRecords = records.filter(record => !existingIds.has(record._id.toString()));
|
||||
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
|
||||
|
||||
return newRecords;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`ClickHouse查询出错: ${error.message}`);
|
||||
if (skip_clickhouse_check) {
|
||||
logWithTimestamp("已启用跳过ClickHouse检查，将继续处理所有记录");
|
||||
return records;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 在处理记录前先检查ClickHouse连接
|
||||
const clickhouseConnected = await checkClickHouseConnection();
|
||||
if (!clickhouseConnected && !skip_clickhouse_check) {
|
||||
logWithTimestamp("⚠️ ClickHouse连接测试失败，请启用skip_clickhouse_check=true参数来跳过连接检查");
|
||||
throw new Error("ClickHouse连接失败，无法继续同步");
|
||||
}
|
||||
|
||||
// 处理记录的函数
|
||||
const processRecords = async (records: ShortRecord[]) => {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
logWithTimestamp(`开始处理批次数据，共 ${records.length} 条短链接记录...`);
|
||||
|
||||
// 检查记录是否已存在
|
||||
let newRecords;
|
||||
try {
|
||||
newRecords = await checkExistingRecords(records);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
|
||||
if (!skip_clickhouse_check && !force_insert) {
|
||||
throw error;
|
||||
}
|
||||
// 如果跳过检查或强制插入，则使用所有记录
|
||||
logWithTimestamp("将使用所有记录进行处理");
|
||||
newRecords = records;
|
||||
}
|
||||
|
||||
if (newRecords.length === 0) {
|
||||
logWithTimestamp("所有记录都已存在，跳过处理");
|
||||
// 更新同步状态，即使没有新增记录
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastId = lastRecord._id.toString();
|
||||
lastCreateTime = lastRecord.createTime;
|
||||
return 0;
|
||||
}
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条新短链接记录...`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
// 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
|
||||
// 处理日期时间，移除ISO格式中的Z以使ClickHouse正确解析
|
||||
const createdAtStr = new Date(record.createTime).toISOString().replace('Z', '');
|
||||
const expiresAtStr = record.expiresAt ? new Date(record.expiresAt).toISOString().replace('Z', '') : null;
|
||||
|
||||
return {
|
||||
link_id: record._id.toString(), // 使用MongoDB的_id作为link_id
|
||||
original_url: record.origin || "",
|
||||
created_at: createdAtStr,
|
||||
created_by: record.user || "unknown",
|
||||
title: record.slug, // 使用slug作为title
|
||||
description: record.description || "",
|
||||
tags: record.tags || [],
|
||||
is_active: record.active !== undefined ? record.active : true,
|
||||
expires_at: expiresAtStr,
|
||||
team_id: record.teamId || "",
|
||||
project_id: record.projectId || ""
|
||||
};
|
||||
});
|
||||
|
||||
// 更新同步状态（使用原始records的最后一条，以确保进度正确）
|
||||
const lastRecord = records[records.length - 1];
|
||||
lastId = lastRecord._id.toString();
|
||||
lastCreateTime = lastRecord.createTime;
|
||||
logWithTimestamp(`更新同步位置到: ID=${lastId}, 时间=${new Date(lastCreateTime).toISOString()}`);
|
||||
|
||||
// 生成ClickHouse插入SQL
|
||||
// 注意：Array类型需要特殊处理，这里将tags作为JSON字符串处理
|
||||
const insertSQL = `
|
||||
INSERT INTO ${clickhouseConfig.clickhouse_database}.links
|
||||
(link_id, original_url, created_at, created_by, title, description, tags, is_active, expires_at, team_id, project_id)
|
||||
VALUES
|
||||
${clickhouseData.map(record => {
|
||||
// 处理tags数组
|
||||
const tagsStr = JSON.stringify(record.tags || []);
|
||||
|
||||
// 处理expires_at可能为null的情况
|
||||
const expiresAt = record.expires_at ? `'${record.expires_at}'` : "NULL";
|
||||
|
||||
// 确保所有字段在使用replace前都有默认值
|
||||
const safeOriginalUrl = (record.original_url || "").replace(/'/g, "''");
|
||||
const safeCreatedBy = (record.created_by || "unknown").replace(/'/g, "''");
|
||||
const safeTitle = (record.title || "无标题").replace(/'/g, "''");
|
||||
const safeDescription = (record.description || "").replace(/'/g, "''");
|
||||
const safeTeamId = record.team_id || "";
|
||||
const safeProjectId = record.project_id || "";
|
||||
|
||||
return `('${record.link_id}', '${safeOriginalUrl}', '${record.created_at}', '${safeCreatedBy}', '${safeTitle}', '${safeDescription}', ${tagsStr}, ${record.is_active}, ${expiresAt}, '${safeTeamId}', '${safeProjectId}')`;
|
||||
}).join(", ")}
|
||||
`;
|
||||
|
||||
if (clickhouseData.length === 0) {
|
||||
console.log("没有新记录需要插入");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 发送请求到ClickHouse
|
||||
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
|
||||
try {
|
||||
logWithTimestamp("发送插入请求到ClickHouse...");
|
||||
const response = await fetch(clickhouseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
|
||||
},
|
||||
body: insertSQL,
|
||||
signal: AbortSignal.timeout(20000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
|
||||
}
|
||||
|
||||
logWithTimestamp(`成功插入 ${newRecords.length} 条短链接记录到ClickHouse`);
|
||||
return newRecords.length;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logWithTimestamp(`向ClickHouse插入数据失败: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// 批量处理记录
|
||||
for (let page = 0; processedRecords < recordsToProcess; page++) {
|
||||
// 检查超时
|
||||
if (checkTimeout()) {
|
||||
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录，因超时暂停执行`);
|
||||
break;
|
||||
}
|
||||
|
||||
// 每批次都输出进度
|
||||
logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
|
||||
const records = await shortCollection.find(query)
|
||||
.sort({ createTime: 1, _id: 1 })
|
||||
.skip(page * batch_size)
|
||||
.limit(batch_size)
|
||||
.toArray();
|
||||
|
||||
if (records.length === 0) {
|
||||
logWithTimestamp(`第 ${page+1} 批次没有找到数据，结束处理`);
|
||||
break;
|
||||
}
|
||||
|
||||
logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
|
||||
// 输出当前批次的部分数据信息
|
||||
if (records.length > 0) {
|
||||
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, Slug=${records[0].slug}, 时间=${new Date(records[0].createTime).toISOString()}`);
|
||||
if (records.length > 1) {
|
||||
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, Slug=${records[records.length-1].slug}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
const batchSize = await processRecords(records);
|
||||
processedRecords += records.length; // 总是增加处理的记录数，即使有些记录已存在
|
||||
totalBatchRecords += batchSize; // 只增加实际插入的记录数
|
||||
|
||||
logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
|
||||
|
||||
// 更新查询条件，以便下一批次查询
|
||||
query.createTime = { $gt: lastCreateTime };
|
||||
if (lastId) {
|
||||
query._id = { $gt: new ObjectId(lastId) };
|
||||
}
|
||||
logWithTimestamp(`更新查询条件: 创建时间 > ${new Date(lastCreateTime).toISOString()}, ID > ${lastId || 'none'}`);
|
||||
}
|
||||
|
||||
// 更新同步状态
|
||||
const newSyncState: SyncState = {
|
||||
last_sync_time: lastCreateTime,
|
||||
records_synced: syncState.records_synced + totalBatchRecords,
|
||||
last_sync_id: lastId
|
||||
};
|
||||
|
||||
await setVariable("f/shorturl_analytics/clickhouse/shorturl_links_sync_state", JSON.stringify(newSyncState));
|
||||
console.log(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 总同步记录数 ${newSyncState.records_synced}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
records_processed: processedRecords,
|
||||
records_synced: totalBatchRecords,
|
||||
total_synced: newSyncState.records_synced,
|
||||
last_sync_time: new Date(newSyncState.last_sync_time).toISOString(),
|
||||
message: "短链接数据同步完成"
|
||||
};
|
||||
} catch (err) {
|
||||
console.error("同步过程中发生错误:", err);
|
||||
return {
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
stack: err instanceof Error ? err.stack : undefined
|
||||
};
|
||||
} finally {
|
||||
// 关闭MongoDB连接
|
||||
await client.close();
|
||||
console.log("MongoDB连接已关闭");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user