17 Commits

Author SHA1 Message Date
7a03396cdd add event 2025-03-26 20:19:37 +08:00
e9b9950ed3 add event example desc 2025-03-26 19:21:57 +08:00
f5b14bf936 event track api 2025-03-26 18:19:37 +08:00
ca8a7d56f1 swagerr doc 2025-03-26 17:39:58 +08:00
913c9cd289 swagger configure 2025-03-26 17:17:47 +08:00
e916eab92c mv folder 2025-03-26 16:39:04 +08:00
63a578ef38 event api 2025-03-26 12:04:53 +08:00
b4aa765c17 event api 2025-03-26 11:26:53 +08:00
c0e5a9ccb2 add pie chart 2025-03-26 11:18:36 +08:00
1755b44a39 style 2025-03-25 22:24:52 +08:00
e0ac87fb25 events 2025-03-25 21:12:03 +08:00
ecf21a812f dashboard page good 2025-03-25 21:02:17 +08:00
efdfe8bf8e front 2025-03-25 20:54:02 +08:00
92d82b18a0 events apis 2025-03-25 17:26:04 +08:00
1e9e5928d7 sync trace & short to clickhouse events 2025-03-25 14:35:01 +08:00
231cf556b0 table sql 2025-03-25 13:03:01 +08:00
3413d3e182 table sql 2025-03-25 12:56:20 +08:00
66 changed files with 7218 additions and 5838 deletions

View File

@@ -0,0 +1,47 @@
# Date Format Handling for ClickHouse Events API
## Problem Description
The event tracking API was experiencing issues with date format compatibility when inserting records into the ClickHouse database. ClickHouse has specific requirements for datetime formats, particularly for its `DateTime64` type fields, which weren't being properly addressed in the original implementation.
## Root Cause
- JavaScript's default date serialization (`toISOString()`) produces formats like `2023-08-24T12:34:56.789Z`, which include `T` as a separator and `Z` as the UTC timezone indicator
- ClickHouse prefers datetime values in the format `YYYY-MM-DD HH:MM:SS.SSS` for seamless parsing
- The mismatch between these formats was causing insertion errors in the database
## Solution Implemented
We created a `formatDateTime` utility function that properly formats JavaScript Date objects for ClickHouse compatibility:
```typescript
const formatDateTime = (date: Date) => {
return date.toISOString().replace('T', ' ').replace('Z', '');
};
```
This function:
1. Takes a JavaScript Date object as input
2. Converts it to ISO format string
3. Replaces the 'T' separator with a space
4. Removes the trailing 'Z' UTC indicator
The solution was applied to all date fields in the event payload:
- `event_time`
- `link_created_at`
- `link_expires_at`
## Additional Improvements
- We standardized date handling by using a consistent `currentTime` variable
- Added type checking for JSON fields to ensure proper serialization
- Improved error handling for date parsing failures
## Best Practices for ClickHouse Date Handling
1. Always format dates as `YYYY-MM-DD HH:MM:SS.SSS` when inserting into ClickHouse
2. Use consistent date handling utilities across your application
3. Consider timezone handling explicitly when needed
4. For query parameters, use ClickHouse's `parseDateTimeBestEffort` function when possible
5. Test with various date formats and edge cases to ensure robustness

View File

@@ -0,0 +1,309 @@
"use client";
import { useState, useEffect, useRef } from 'react';
import { fetchData } from '@/app/api/utils';
import { DeviceAnalytics } from '@/app/api/types';
import { Chart, PieController, ArcElement, Tooltip, Legend, CategoryScale, LinearScale } from 'chart.js';
// 注册Chart.js组件
Chart.register(PieController, ArcElement, Tooltip, Legend, CategoryScale, LinearScale);
export default function DeviceAnalyticsPage() {
const [deviceData, setDeviceData] = useState<DeviceAnalytics | null>(null);
const [isLoading, setIsLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [dateRange, setDateRange] = useState({
from: new Date('2024-02-01'),
to: new Date('2025-03-05')
});
// 创建图表引用
const deviceTypesChartRef = useRef<HTMLCanvasElement>(null);
const browsersChartRef = useRef<HTMLCanvasElement>(null);
const osChartRef = useRef<HTMLCanvasElement>(null);
// 图表实例引用
const deviceTypesChartInstance = useRef<Chart | null>(null);
const browsersChartInstance = useRef<Chart | null>(null);
const osChartInstance = useRef<Chart | null>(null);
// 颜色配置
const COLORS = {
deviceTypes: ['rgba(59, 130, 246, 0.8)', 'rgba(96, 165, 250, 0.8)', 'rgba(147, 197, 253, 0.8)', 'rgba(191, 219, 254, 0.8)', 'rgba(219, 234, 254, 0.8)'],
browsers: ['rgba(16, 185, 129, 0.8)', 'rgba(52, 211, 153, 0.8)', 'rgba(110, 231, 183, 0.8)', 'rgba(167, 243, 208, 0.8)', 'rgba(209, 250, 229, 0.8)'],
os: ['rgba(239, 68, 68, 0.8)', 'rgba(248, 113, 113, 0.8)', 'rgba(252, 165, 165, 0.8)', 'rgba(254, 202, 202, 0.8)', 'rgba(254, 226, 226, 0.8)']
};
useEffect(() => {
const fetchDeviceData = async () => {
try {
setIsLoading(true);
setError(null);
const response = await fetch(`/api/events/devices?startTime=${dateRange.from.toISOString().split('T')[0]}T00:00:00Z&endTime=${dateRange.to.toISOString().split('T')[0]}T23:59:59Z`);
if (!response.ok) throw new Error('Failed to fetch device data');
const data = await response.json();
setDeviceData(data.data);
} catch (err) {
setError(err instanceof Error ? err.message : 'An error occurred');
} finally {
setIsLoading(false);
}
};
fetchDeviceData();
}, [dateRange]);
// 初始化和更新图表
useEffect(() => {
if (!deviceData || isLoading) return;
// 销毁旧的图表实例
if (deviceTypesChartInstance.current) {
deviceTypesChartInstance.current.destroy();
}
if (browsersChartInstance.current) {
browsersChartInstance.current.destroy();
}
if (osChartInstance.current) {
osChartInstance.current.destroy();
}
// 创建设备类型图表
if (deviceTypesChartRef.current && deviceData.deviceTypes.length > 0) {
const ctx = deviceTypesChartRef.current.getContext('2d');
if (ctx) {
deviceTypesChartInstance.current = new Chart(ctx, {
type: 'pie',
data: {
labels: deviceData.deviceTypes.map(item => item.type),
datasets: [{
data: deviceData.deviceTypes.map(item => item.count),
backgroundColor: COLORS.deviceTypes,
borderColor: COLORS.deviceTypes.map(color => color.replace('0.8', '1')),
borderWidth: 1
}]
},
options: {
responsive: true,
plugins: {
legend: {
position: 'bottom',
labels: {
color: 'white'
}
},
tooltip: {
callbacks: {
label: function(context) {
const label = context.label || '';
const value = context.raw as number;
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
const percentage = Math.round((value * 100) / total);
return `${label}: ${value} (${percentage}%)`;
}
}
}
}
}
});
}
}
// 创建浏览器图表
if (browsersChartRef.current && deviceData.browsers.length > 0) {
const ctx = browsersChartRef.current.getContext('2d');
if (ctx) {
browsersChartInstance.current = new Chart(ctx, {
type: 'pie',
data: {
labels: deviceData.browsers.map(item => item.name),
datasets: [{
data: deviceData.browsers.map(item => item.count),
backgroundColor: COLORS.browsers,
borderColor: COLORS.browsers.map(color => color.replace('0.8', '1')),
borderWidth: 1
}]
},
options: {
responsive: true,
plugins: {
legend: {
position: 'bottom',
labels: {
color: 'white'
}
},
tooltip: {
callbacks: {
label: function(context) {
const label = context.label || '';
const value = context.raw as number;
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
const percentage = Math.round((value * 100) / total);
return `${label}: ${value} (${percentage}%)`;
}
}
}
}
}
});
}
}
// 创建操作系统图表
if (osChartRef.current && deviceData.operatingSystems.length > 0) {
const ctx = osChartRef.current.getContext('2d');
if (ctx) {
osChartInstance.current = new Chart(ctx, {
type: 'pie',
data: {
labels: deviceData.operatingSystems.map(item => item.name),
datasets: [{
data: deviceData.operatingSystems.map(item => item.count),
backgroundColor: COLORS.os,
borderColor: COLORS.os.map(color => color.replace('0.8', '1')),
borderWidth: 1
}]
},
options: {
responsive: true,
plugins: {
legend: {
position: 'bottom',
labels: {
color: 'white'
}
},
tooltip: {
callbacks: {
label: function(context) {
const label = context.label || '';
const value = context.raw as number;
const total = (context.chart.data.datasets[0].data as number[]).reduce((a, b) => (a as number) + (b as number), 0);
const percentage = Math.round((value * 100) / total);
return `${label}: ${value} (${percentage}%)`;
}
}
}
}
}
});
}
}
// 清理函数
return () => {
if (deviceTypesChartInstance.current) {
deviceTypesChartInstance.current.destroy();
}
if (browsersChartInstance.current) {
browsersChartInstance.current.destroy();
}
if (osChartInstance.current) {
osChartInstance.current.destroy();
}
};
}, [deviceData, isLoading]);
return (
<div className="container mx-auto px-4 py-8">
{/* 页面标题 */}
<div className="mb-8">
<h1 className="text-2xl font-bold text-white">Device Analytics</h1>
<p className="mt-2 text-white">Analyze visitor distribution by devices, browsers, and operating systems</p>
</div>
{/* 时间范围选择器 */}
<div className="bg-card-bg rounded-xl p-6 mb-8">
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label className="block text-sm font-medium text-text-secondary mb-1">Start Date</label>
<input
type="date"
className="block w-full px-3 py-2 bg-background border border-card-border rounded-md text-sm"
value={dateRange.from.toISOString().split('T')[0]}
onChange={e => setDateRange(prev => ({ ...prev, from: new Date(e.target.value) }))}
/>
</div>
<div>
<label className="block text-sm font-medium text-text-secondary mb-1">End Date</label>
<input
type="date"
className="block w-full px-3 py-2 bg-background border border-card-border rounded-md text-sm"
value={dateRange.to.toISOString().split('T')[0]}
onChange={e => setDateRange(prev => ({ ...prev, to: new Date(e.target.value) }))}
/>
</div>
</div>
</div>
{/* 设备类型分析 */}
<div className="grid grid-cols-1 lg:grid-cols-3 gap-8 mb-8">
{/* 设备类型 */}
<div className="bg-card-bg rounded-xl p-6">
<h3 className="text-lg font-medium text-white mb-4">Device Types</h3>
{deviceData && deviceData.deviceTypes.length > 0 ? (
<div className="h-64">
<canvas ref={deviceTypesChartRef} />
</div>
) : (
<div className="flex justify-center items-center h-64 text-text-secondary">
No data available
</div>
)}
</div>
{/* 浏览器 */}
<div className="bg-card-bg rounded-xl p-6">
<h3 className="text-lg font-medium text-white mb-4">Browsers</h3>
{deviceData && deviceData.browsers.length > 0 ? (
<div className="h-64">
<canvas ref={browsersChartRef} />
</div>
) : (
<div className="flex justify-center items-center h-64 text-text-secondary">
No data available
</div>
)}
</div>
{/* 操作系统 */}
<div className="bg-card-bg rounded-xl p-6">
<h3 className="text-lg font-medium text-white mb-4">Operating Systems</h3>
{deviceData && deviceData.operatingSystems.length > 0 ? (
<div className="h-64">
<canvas ref={osChartRef} />
</div>
) : (
<div className="flex justify-center items-center h-64 text-text-secondary">
No data available
</div>
)}
</div>
</div>
{/* 加载状态 */}
{isLoading && (
<div className="flex justify-center items-center p-8">
<div className="w-8 h-8 border-t-2 border-b-2 border-accent-blue rounded-full animate-spin"></div>
</div>
)}
{/* 错误状态 */}
{error && (
<div className="flex justify-center items-center p-8 text-accent-red">
<p>{error}</p>
</div>
)}
{/* 无数据状态 */}
{!isLoading && !error && !deviceData && (
<div className="flex justify-center items-center p-8 text-text-secondary">
<p>No device data available</p>
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,137 @@
"use client";
import { useState, useEffect } from 'react';
import { GeoData } from '../../api/types';
export default function GeoAnalyticsPage() {
const [geoData, setGeoData] = useState<GeoData[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [dateRange, setDateRange] = useState({
from: new Date('2024-02-01'),
to: new Date('2025-03-05')
});
useEffect(() => {
const fetchGeoData = async () => {
try {
setIsLoading(true);
setError(null);
const response = await fetch(`/api/events/geo?startTime=${dateRange.from.toISOString().split('T')[0]}T00:00:00Z&endTime=${dateRange.to.toISOString().split('T')[0]}T23:59:59Z`);
if (!response.ok) throw new Error('Failed to fetch geographic data');
const data = await response.json();
setGeoData(data.data);
} catch (err) {
setError(err instanceof Error ? err.message : 'An error occurred');
} finally {
setIsLoading(false);
}
};
fetchGeoData();
}, [dateRange]);
return (
<div className="container mx-auto px-4 py-8">
{/* 页面标题 */}
<div className="mb-8">
<h1 className="text-2xl font-bold text-white">Geographic Analysis</h1>
<p className="mt-2 text-white">Analyze visitor distribution by location</p>
</div>
{/* 时间范围选择器 */}
<div className="bg-card-bg rounded-xl p-6 mb-8">
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label className="block text-sm font-medium text-white mb-1">Start Date</label>
<input
type="date"
className="block w-full px-3 py-2 bg-background border border-card-border rounded-md text-sm "
value={dateRange.from.toISOString().split('T')[0]}
onChange={e => setDateRange(prev => ({ ...prev, from: new Date(e.target.value) }))}
/>
</div>
<div>
<label className="block text-sm font-medium text-white mb-1">End Date</label>
<input
type="date"
className="block w-full px-3 py-2 bg-background border border-card-border rounded-md text-sm "
value={dateRange.to.toISOString().split('T')[0]}
onChange={e => setDateRange(prev => ({ ...prev, to: new Date(e.target.value) }))}
/>
</div>
</div>
</div>
{/* 地理数据表格 */}
<div className="bg-card-bg rounded-xl shadow-sm overflow-hidden">
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-card-border">
<thead>
<tr className="bg-background/50">
<th className="px-6 py-3 text-left text-xs font-medium text-white uppercase tracking-wider">Location</th>
<th className="px-6 py-3 text-left text-xs font-medium text-white uppercase tracking-wider">Visits</th>
<th className="px-6 py-3 text-left text-xs font-medium text-white uppercase tracking-wider">Unique Visitors</th>
<th className="px-6 py-3 text-left text-xs font-medium text-white uppercase tracking-wider">Percentage</th>
</tr>
</thead>
<tbody className="divide-y divide-card-border">
{geoData.map(item => (
<tr key={item.location} className="hover:bg-background/50">
<td className="px-6 py-4 whitespace-nowrap text-sm text-white">
{item.location}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-white">
{item.visits}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-white">
{item.visitors}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">
<div className="flex items-center">
<div className="w-full bg-background rounded-full h-2 mr-2">
<div
className="bg-blue-500 h-2 rounded-full"
style={{ width: `${item.percentage}%` }}
/>
</div>
<span className="text-white">{item.percentage.toFixed(1)}%</span>
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
{/* 加载状态 */}
{isLoading && (
<div className="flex justify-center items-center p-8">
<div className="w-8 h-8 border-t-2 border-b-2 border-accent-blue rounded-full animate-spin"></div>
</div>
)}
{/* 错误状态 */}
{error && (
<div className="flex justify-center items-center p-8 text-accent-red">
<p>{error}</p>
</div>
)}
{/* 无数据状态 */}
{!isLoading && !error && geoData.length === 0 && (
<div className="flex justify-center items-center p-8 text-white">
<p>No geographic data available</p>
</div>
)}
</div>
{/* 提示信息 */}
<div className="mt-4 text-sm text-white">
<p>Note: Geographic data is based on IP addresses and may not be 100% accurate.</p>
</div>
</div>
);
}

View File

@@ -0,0 +1,140 @@
"use client";
import { useState, useEffect } from 'react';
import { addDays, format } from 'date-fns';
import { DateRangePicker } from '@/app/components/ui/DateRangePicker';
import TimeSeriesChart from '@/app/components/charts/TimeSeriesChart';
import GeoAnalytics from '@/app/components/analytics/GeoAnalytics';
import DeviceAnalytics from '@/app/components/analytics/DeviceAnalytics';
import { EventsSummary, TimeSeriesData, GeoData, DeviceAnalytics as DeviceAnalyticsType } from '@/app/api/types';
export default function DashboardPage() {
const [dateRange, setDateRange] = useState({
from: new Date('2024-02-01'),
to: new Date('2025-03-05')
});
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [summary, setSummary] = useState<EventsSummary | null>(null);
const [timeSeriesData, setTimeSeriesData] = useState<TimeSeriesData[]>([]);
const [geoData, setGeoData] = useState<GeoData[]>([]);
const [deviceData, setDeviceData] = useState<DeviceAnalyticsType | null>(null);
useEffect(() => {
const fetchData = async () => {
setLoading(true);
setError(null);
try {
const startTime = format(dateRange.from, "yyyy-MM-dd'T'HH:mm:ss'Z'");
const endTime = format(dateRange.to, "yyyy-MM-dd'T'HH:mm:ss'Z'");
// 并行获取所有数据
const [summaryRes, timeSeriesRes, geoRes, deviceRes] = await Promise.all([
fetch(`/api/events/summary?startTime=${startTime}&endTime=${endTime}`),
fetch(`/api/events/time-series?startTime=${startTime}&endTime=${endTime}`),
fetch(`/api/events/geo?startTime=${startTime}&endTime=${endTime}`),
fetch(`/api/events/devices?startTime=${startTime}&endTime=${endTime}`)
]);
const [summaryData, timeSeriesData, geoData, deviceData] = await Promise.all([
summaryRes.json(),
timeSeriesRes.json(),
geoRes.json(),
deviceRes.json()
]);
if (!summaryRes.ok) throw new Error(summaryData.error || 'Failed to fetch summary data');
if (!timeSeriesRes.ok) throw new Error(timeSeriesData.error || 'Failed to fetch time series data');
if (!geoRes.ok) throw new Error(geoData.error || 'Failed to fetch geo data');
if (!deviceRes.ok) throw new Error(deviceData.error || 'Failed to fetch device data');
setSummary(summaryData);
setTimeSeriesData(timeSeriesData.data);
setGeoData(geoData.data);
setDeviceData(deviceData.data);
} catch (err) {
setError(err instanceof Error ? err.message : 'An error occurred while fetching data');
} finally {
setLoading(false);
}
};
fetchData();
}, [dateRange]);
if (loading) {
return (
<div className="flex items-center justify-center min-h-screen">
<div className="animate-spin rounded-full h-12 w-12 border-t-2 border-b-2 border-blue-500" />
</div>
);
}
if (error) {
return (
<div className="flex items-center justify-center min-h-screen">
<div className="text-red-500">{error}</div>
</div>
);
}
return (
<div className="container mx-auto px-4 py-8">
<div className="flex justify-between items-center mb-8">
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">Analytics Dashboard</h1>
<DateRangePicker
value={dateRange}
onChange={setDateRange}
/>
</div>
{summary && (
<div className="grid grid-cols-1 md:grid-cols-4 gap-6 mb-8">
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
<h3 className="text-sm font-medium text-gray-500 dark:text-gray-400">Total Events</h3>
<p className="text-2xl font-semibold text-gray-900 dark:text-gray-100">
{typeof summary.totalEvents === 'number' ? summary.totalEvents.toLocaleString() : summary.totalEvents}
</p>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
<h3 className="text-sm font-medium text-gray-500 dark:text-gray-400">Unique Visitors</h3>
<p className="text-2xl font-semibold text-gray-900 dark:text-gray-100">
{typeof summary.uniqueVisitors === 'number' ? summary.uniqueVisitors.toLocaleString() : summary.uniqueVisitors}
</p>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
<h3 className="text-sm font-medium text-gray-500 dark:text-gray-400">Total Conversions</h3>
<p className="text-2xl font-semibold text-gray-900 dark:text-gray-100">
{typeof summary.totalConversions === 'number' ? summary.totalConversions.toLocaleString() : summary.totalConversions}
</p>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
<h3 className="text-sm font-medium text-gray-500 dark:text-gray-400">Avg. Time Spent</h3>
<p className="text-2xl font-semibold text-gray-900 dark:text-gray-100">
{summary.averageTimeSpent?.toFixed(1) || '0'}s
</p>
</div>
</div>
)}
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 mb-8">
<h2 className="text-lg font-semibold text-gray-900 dark:text-gray-100 mb-4">Event Trends</h2>
<div className="h-96">
<TimeSeriesChart data={timeSeriesData} />
</div>
</div>
<div className="mb-8">
<h2 className="text-lg font-semibold text-gray-900 dark:text-gray-100 mb-4">Device Analytics</h2>
{deviceData && <DeviceAnalytics data={deviceData} />}
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
<h2 className="text-lg font-semibold text-gray-900 dark:text-gray-100 mb-4">Geographic Distribution</h2>
<GeoAnalytics data={geoData} />
</div>
</div>
);
}

257
app/(app)/events/page.tsx Normal file
View File

@@ -0,0 +1,257 @@
"use client";
import { useState, useEffect } from 'react';
import { addDays, format } from 'date-fns';
import { DateRangePicker } from '@/app/components/ui/DateRangePicker';
import { Event } from '@/app/api/types';
export default function EventsPage() {
const [dateRange, setDateRange] = useState({
from: new Date('2024-02-01'),
to: new Date('2025-03-05')
});
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [events, setEvents] = useState<Event[]>([]);
const [page, setPage] = useState(1);
const [hasMore, setHasMore] = useState(true);
const [filter, setFilter] = useState({
eventType: '',
linkId: '',
linkSlug: ''
});
const [filters, setFilters] = useState({
startTime: format(new Date('2024-02-01'), "yyyy-MM-dd'T'HH:mm:ss'Z'"),
endTime: format(new Date('2025-03-05'), "yyyy-MM-dd'T'HH:mm:ss'Z'"),
page: 1,
pageSize: 20
});
const [summary, setSummary] = useState<any>(null);
const fetchEvents = async (pageNum: number) => {
try {
const startTime = format(dateRange.from, "yyyy-MM-dd'T'HH:mm:ss'Z'");
const endTime = format(dateRange.to, "yyyy-MM-dd'T'HH:mm:ss'Z'");
const params = new URLSearchParams({
startTime,
endTime,
page: pageNum.toString(),
pageSize: '50'
});
if (filter.eventType) params.append('eventType', filter.eventType);
if (filter.linkId) params.append('linkId', filter.linkId);
if (filter.linkSlug) params.append('linkSlug', filter.linkSlug);
const response = await fetch(`/api/events?${params.toString()}`);
const data = await response.json();
if (!response.ok) {
throw new Error(data.error || 'Failed to fetch events');
}
const eventsData = data.data || data.events || [];
if (pageNum === 1) {
setEvents(eventsData);
} else {
setEvents(prev => [...prev, ...eventsData]);
}
setHasMore(Array.isArray(eventsData) && eventsData.length === 50);
} catch (err) {
console.error("Error fetching events:", err);
setError(err instanceof Error ? err.message : 'An error occurred while fetching events');
setEvents([]);
} finally {
setLoading(false);
}
};
useEffect(() => {
setPage(1);
setEvents([]);
setLoading(true);
fetchEvents(1);
}, [dateRange, filter]);
const loadMore = () => {
if (!loading && hasMore) {
const nextPage = page + 1;
setPage(nextPage);
fetchEvents(nextPage);
}
};
const formatDate = (dateString: string) => {
const date = new Date(dateString);
return date.toLocaleString();
};
if (error) {
return (
<div className="flex items-center justify-center min-h-screen">
<div className="text-red-500">{error}</div>
</div>
);
}
return (
<div className="container mx-auto px-4 py-8">
<div className="flex justify-between items-center mb-8">
<h1 className="text-2xl font-bold text-gray-900 dark:text-gray-100">Events</h1>
<DateRangePicker
value={dateRange}
onChange={setDateRange}
/>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 mb-8">
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
<div>
<label className="block text-sm font-medium text-gray-500 dark:text-gray-400 mb-1">
Event Type
</label>
<select
value={filter.eventType}
onChange={e => setFilter(prev => ({ ...prev, eventType: e.target.value }))}
className="block w-full px-3 py-2 bg-white dark:bg-gray-800 border border-gray-300 dark:border-gray-600 rounded-md text-sm text-gray-900 dark:text-gray-100"
>
<option value="">All Types</option>
<option value="click">Click</option>
<option value="conversion">Conversion</option>
</select>
</div>
<div>
<label className="block text-sm font-medium text-gray-500 dark:text-gray-400 mb-1">
Link ID
</label>
<input
type="text"
value={filter.linkId}
onChange={e => setFilter(prev => ({ ...prev, linkId: e.target.value }))}
placeholder="Enter Link ID"
className="block w-full px-3 py-2 bg-white dark:bg-gray-800 border border-gray-300 dark:border-gray-600 rounded-md text-sm text-gray-900 dark:text-gray-100"
/>
</div>
<div>
<label className="block text-sm font-medium text-gray-500 dark:text-gray-400 mb-1">
Link Slug
</label>
<input
type="text"
value={filter.linkSlug}
onChange={e => setFilter(prev => ({ ...prev, linkSlug: e.target.value }))}
placeholder="Enter Link Slug"
className="block w-full px-3 py-2 bg-white dark:bg-gray-800 border border-gray-300 dark:border-gray-600 rounded-md text-sm text-gray-900 dark:text-gray-100"
/>
</div>
</div>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg shadow overflow-hidden">
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-gray-200 dark:divide-gray-700">
<thead className="bg-gray-50 dark:bg-gray-900">
<tr>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Time
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Type
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Link
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Visitor
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Location
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Referrer
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Conversion
</th>
</tr>
</thead>
<tbody className="bg-white dark:bg-gray-800 divide-y divide-gray-200 dark:divide-gray-700">
{Array.isArray(events) && events.map((event, index) => (
<tr key={event.event_id || index} className={index % 2 === 0 ? 'bg-white dark:bg-gray-800' : 'bg-gray-50 dark:bg-gray-900'}>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
{event.event_time && formatDate(event.event_time)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm">
<span className={`inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium ${
event.event_type === 'conversion' ? 'bg-green-100 text-green-800 dark:bg-green-800 dark:text-green-100' : 'bg-blue-100 text-blue-800 dark:bg-blue-800 dark:text-blue-100'
}`}>
{event.event_type || 'unknown'}
</span>
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
<div>
<div className="font-medium">{event.link_slug || '-'}</div>
<div className="text-gray-500 dark:text-gray-400 text-xs">{event.link_original_url || '-'}</div>
</div>
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
<div>
<div>{event.browser || '-'}</div>
<div className="text-gray-500 dark:text-gray-400 text-xs">{event.os || '-'} / {event.device_type || '-'}</div>
</div>
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
<div>
<div>{event.city || '-'}</div>
<div className="text-gray-500 dark:text-gray-400 text-xs">{event.country || '-'}</div>
</div>
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500 dark:text-gray-400">
{event.referrer || '-'}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
<div>
<div>{event.conversion_type || '-'}</div>
{event.conversion_value > 0 && (
<div className="text-gray-500 dark:text-gray-400 text-xs">Value: {event.conversion_value}</div>
)}
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
{loading && (
<div className="flex justify-center p-4">
<div className="animate-spin rounded-full h-8 w-8 border-t-2 border-b-2 border-blue-500" />
</div>
)}
{!loading && hasMore && (
<div className="flex justify-center p-4">
<button
onClick={loadMore}
className="px-4 py-2 border border-gray-300 dark:border-gray-600 rounded-md text-sm font-medium text-gray-700 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700"
>
Load More
</button>
</div>
)}
{!loading && Array.isArray(events) && events.length === 0 && (
<div className="flex justify-center p-8 text-gray-500 dark:text-gray-400">
No events found
</div>
)}
</div>
</div>
);
}

66
app/(app)/layout.tsx Normal file
View File

@@ -0,0 +1,66 @@
import '../globals.css';
import type { Metadata } from 'next';
import { Inter } from 'next/font/google';
import Link from 'next/link';
const inter = Inter({ subsets: ['latin'] });
export const metadata: Metadata = {
title: 'ShortURL Analytics',
description: 'Analytics dashboard for ShortURL service',
};
export default function AppLayout({
children,
}: {
children: React.ReactNode;
}) {
return (
<div className={inter.className}>
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
<nav className="bg-white dark:bg-gray-800 border-b border-gray-200 dark:border-gray-700">
<div className="container mx-auto px-4">
<div className="flex items-center justify-between h-16">
<div className="flex items-center">
<Link href="/" className="text-xl font-bold text-gray-900 dark:text-gray-100">
ShortURL Analytics
</Link>
<div className="hidden md:block ml-10">
<div className="flex items-baseline space-x-4">
<Link
href="/dashboard"
className="text-gray-500 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100 px-3 py-2 rounded-md text-sm font-medium"
>
Dashboard
</Link>
<Link
href="/events"
className="text-gray-500 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100 px-3 py-2 rounded-md text-sm font-medium"
>
Events
</Link>
<Link
href="/analytics/geo"
className="text-gray-500 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100 px-3 py-2 rounded-md text-sm font-medium"
>
Geographic
</Link>
<Link
href="/analytics/devices"
className="text-gray-500 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100 px-3 py-2 rounded-md text-sm font-medium"
>
Devices
</Link>
</div>
</div>
</div>
</div>
</div>
</nav>
<main className="py-10">
{children}
</main>
</div>
</div>
);
}

59
app/(app)/page.tsx Normal file
View File

@@ -0,0 +1,59 @@
import Link from 'next/link';
export default function Home() {
return (
<div className="container mx-auto px-4">
<div className="max-w-2xl mx-auto py-16">
<h1 className="text-4xl font-bold text-gray-900 dark:text-gray-100 mb-8">
Welcome to ShortURL Analytics
</h1>
<div className="grid gap-6">
<Link
href="/dashboard"
className="block p-6 bg-white dark:bg-gray-800 rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 dark:text-gray-100 mb-2">
Dashboard
</h2>
<p className="text-gray-600 dark:text-gray-400">
View your overall analytics and key metrics
</p>
</Link>
<Link
href="/events"
className="block p-6 bg-white dark:bg-gray-800 rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 dark:text-gray-100 mb-2">
Events
</h2>
<p className="text-gray-600 dark:text-gray-400">
Track and analyze event data
</p>
</Link>
<Link
href="/analytics/geo"
className="block p-6 bg-white dark:bg-gray-800 rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 dark:text-gray-100 mb-2">
Geographic Analysis
</h2>
<p className="text-gray-600 dark:text-gray-400">
Explore visitor locations and geographic patterns
</p>
</Link>
<Link
href="/analytics/devices"
className="block p-6 bg-white dark:bg-gray-800 rounded-lg shadow hover:shadow-md transition-shadow"
>
<h2 className="text-xl font-semibold text-gray-900 dark:text-gray-100 mb-2">
Device Analytics
</h2>
<p className="text-gray-600 dark:text-gray-400">
Understand how users access your links
</p>
</Link>
</div>
</div>
</div>
);
}

View File

@@ -0,0 +1,997 @@
"use client";
import { useEffect } from 'react';
import SwaggerUI from 'swagger-ui-react';
import 'swagger-ui-react/swagger-ui.css';
export default function SwaggerPage() {
useEffect(() => {
// 设置页面标题
document.title = 'API Documentation - ShortURL Analytics';
}, []);
// Swagger配置
const swaggerConfig = {
openapi: '3.0.0',
info: {
title: 'ShortURL Analytics API',
version: '1.0.0',
description: 'API documentation for ShortURL Analytics service',
contact: {
name: 'API Support',
email: 'support@example.com',
},
license: {
name: 'MIT',
url: 'https://opensource.org/licenses/MIT',
},
},
servers: [
{
url: '/api',
description: 'API Server',
},
],
tags: [
{
name: 'events',
description: 'Event tracking and analytics endpoints',
},
],
paths: {
'/events/track': {
post: {
tags: ['events'],
summary: 'Track new event',
description: 'Record a new event in the analytics system',
requestBody: {
required: true,
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/EventInput',
},
examples: {
clickEvent: {
summary: 'Basic click event',
value: {
event_type: 'click',
link_id: 'link_123',
link_slug: 'promo2023',
link_original_url: 'https://example.com/promotion',
visitor_id: '6ec0bd7f-11c0-43da-975e-2a8ad9ebae0b'
}
},
conversionEvent: {
summary: 'Conversion event',
value: {
event_type: 'conversion',
link_id: 'link_123',
link_slug: 'promo2023',
visitor_id: '6ec0bd7f-11c0-43da-975e-2a8ad9ebae0b',
conversion_type: 'purchase',
conversion_value: 99.99
}
},
completeEvent: {
summary: 'Complete event with all fields',
value: {
// Core event fields
event_id: '123e4567-e89b-12d3-a456-426614174000',
event_time: '2025-03-26T10:30:00.000Z',
event_type: 'click',
event_attributes: '{"source":"email_campaign","campaign_id":"spring_sale_2025"}',
// Link information
link_id: 'link_abc123',
link_slug: 'summer-promo',
link_label: 'Summer Promotion 2025',
link_title: 'Summer Sale 50% Off',
link_original_url: 'https://example.com/summer-sale-2025',
link_attributes: '{"utm_campaign":"summer_2025","discount_code":"SUMMER50"}',
link_created_at: '2025-03-20T08:00:00.000Z',
link_expires_at: '2025-09-30T23:59:59.000Z',
link_tags: '["promotion","summer","sale"]',
// User information
user_id: 'user_12345',
user_name: 'John Doe',
user_email: 'john.doe@example.com',
user_attributes: '{"subscription_tier":"premium","account_created":"2024-01-15"}',
// Team information
team_id: 'team_67890',
team_name: 'Marketing Team',
team_attributes: '{"department":"marketing","region":"APAC"}',
// Project information
project_id: 'proj_54321',
project_name: 'Summer Campaign 2025',
project_attributes: '{"goals":"increase_sales","budget":"10000"}',
// QR code information
qr_code_id: 'qr_98765',
qr_code_name: 'Summer Flyer QR',
qr_code_attributes: '{"size":"large","color":"#FF5500","logo":true}',
// Visitor information
visitor_id: '6ec0bd7f-11c0-43da-975e-2a8ad9ebae0b',
session_id: '7fc1bd8f-22d1-54eb-986f-3b9be5ecaf1c',
ip_address: '203.0.113.42',
country: 'United States',
city: 'San Francisco',
device_type: 'mobile',
browser: 'Chrome',
os: 'iOS',
user_agent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1',
// Referrer information
referrer: 'https://www.google.com/search?q=summer+sale',
utm_source: 'google',
utm_medium: 'organic',
utm_campaign: 'summer_promotion',
// Interaction information
time_spent_sec: 145,
is_bounce: false,
is_qr_scan: true,
conversion_type: 'signup',
conversion_value: 0
}
}
}
}
}
},
responses: {
'201': {
description: 'Event successfully tracked',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
success: {
type: 'boolean',
example: true
},
message: {
type: 'string',
example: 'Event tracked successfully'
},
event_id: {
type: 'string',
format: 'uuid',
example: '123e4567-e89b-12d3-a456-426614174000'
}
}
}
}
}
},
'400': {
description: 'Bad request',
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/Error'
},
example: {
error: 'Missing required field: event_type'
}
}
}
},
'500': {
description: 'Server error',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
error: {
type: 'string'
},
details: {
type: 'string'
}
}
},
example: {
error: 'Failed to track event',
details: 'Database connection error'
}
}
}
}
}
}
},
'/events': {
get: {
tags: ['events'],
summary: 'Get events',
description: 'Retrieve events within a specified time range with pagination support',
parameters: [
{
name: 'startTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'Start time for events query (ISO 8601 format)',
},
{
name: 'endTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'End time for events query (ISO 8601 format)',
},
{
name: 'page',
in: 'query',
schema: {
type: 'integer',
default: 1,
minimum: 1,
},
description: 'Page number for pagination',
},
{
name: 'pageSize',
in: 'query',
schema: {
type: 'integer',
default: 50,
minimum: 1,
maximum: 100,
},
description: 'Number of items per page',
},
],
responses: {
'200': {
description: 'Successful response',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
data: {
type: 'array',
items: {
$ref: '#/components/schemas/Event',
},
},
pagination: {
$ref: '#/components/schemas/Pagination',
},
},
},
},
},
},
'400': {
description: 'Bad request',
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/Error',
},
},
},
},
},
},
},
'/events/summary': {
get: {
tags: ['events'],
summary: 'Get events summary',
description: 'Get aggregated statistics for events within a specified time range',
parameters: [
{
name: 'startTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'Start time for summary (ISO 8601 format)',
},
{
name: 'endTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'End time for summary (ISO 8601 format)',
},
],
responses: {
'200': {
description: 'Successful response',
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/EventsSummary',
},
},
},
},
'400': {
description: 'Bad request',
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/Error',
},
},
},
},
},
},
},
'/events/time-series': {
get: {
tags: ['events'],
summary: 'Get time series data',
description: 'Get time-based analytics data for events',
parameters: [
{
name: 'startTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'Start time for time series data (ISO 8601 format)',
},
{
name: 'endTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'End time for time series data (ISO 8601 format)',
},
],
responses: {
'200': {
description: 'Successful response',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
data: {
type: 'array',
items: {
$ref: '#/components/schemas/TimeSeriesData',
},
},
},
},
},
},
},
'400': {
description: 'Bad request',
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/Error',
},
},
},
},
},
},
},
'/events/geo': {
get: {
tags: ['events'],
summary: 'Get geographic data',
description: 'Get geographic distribution of events',
parameters: [
{
name: 'startTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'Start time for geographic data (ISO 8601 format)',
},
{
name: 'endTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'End time for geographic data (ISO 8601 format)',
},
],
responses: {
'200': {
description: 'Successful response',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
data: {
type: 'array',
items: {
$ref: '#/components/schemas/GeoData',
},
},
},
},
},
},
},
'400': {
description: 'Bad request',
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/Error',
},
},
},
},
},
},
},
'/events/devices': {
get: {
tags: ['events'],
summary: 'Get device analytics data',
description: 'Get device-related analytics for events',
parameters: [
{
name: 'startTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'Start time for device analytics (ISO 8601 format)',
},
{
name: 'endTime',
in: 'query',
required: true,
schema: {
type: 'string',
format: 'date-time',
},
description: 'End time for device analytics (ISO 8601 format)',
},
],
responses: {
'200': {
description: 'Successful response',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
data: {
$ref: '#/components/schemas/DeviceAnalytics',
},
},
},
},
},
},
'400': {
description: 'Bad request',
content: {
'application/json': {
schema: {
$ref: '#/components/schemas/Error',
},
},
},
},
},
},
},
},
components: {
schemas: {
EventInput: {
type: 'object',
required: ['event_type'],
properties: {
// Core event fields
event_id: {
type: 'string',
format: 'uuid',
description: '事件唯一标识符用于唯一标识事件记录。若不提供则自动生成UUID'
},
event_time: {
type: 'string',
format: 'date-time',
description: '事件发生的时间戳ISO 8601格式记录事件发生的精确时间。若不提供则使用当前服务器时间'
},
event_type: {
type: 'string',
enum: ['click', 'conversion', 'redirect', 'error'],
description: '事件类型用于分类不同的用户交互行为。click表示点击事件conversion表示转化事件redirect表示重定向事件error表示错误事件'
},
event_attributes: {
type: 'string',
description: '事件附加属性的JSON字符串用于存储与特定事件相关的自定义数据例如事件来源、关联活动ID等'
},
// Link information
link_id: {
type: 'string',
description: '短链接的唯一标识符,用于关联事件与特定短链接'
},
link_slug: {
type: 'string',
description: '短链接的短码/slug部分即URL路径中的短字符串用于生成短链接URL'
},
link_label: {
type: 'string',
description: '短链接的标签名称,用于分类和组织管理短链接'
},
link_title: {
type: 'string',
description: '短链接的标题,用于在管理界面或分析报告中显示链接的易读名称'
},
link_original_url: {
type: 'string',
format: 'uri',
description: '短链接对应的原始目标URL即用户访问短链接后将被重定向到的实际URL'
},
link_attributes: {
type: 'string',
description: '链接附加属性的JSON字符串用于存储与链接相关的自定义数据如营销活动信息、目标受众等'
},
link_created_at: {
type: 'string',
format: 'date-time',
description: '短链接创建时间,记录链接何时被创建'
},
link_expires_at: {
type: 'string',
format: 'date-time',
nullable: true,
description: '短链接过期时间指定链接何时失效值为null表示永不过期'
},
link_tags: {
type: 'string',
description: '链接标签的JSON数组字符串用于通过标签对链接进行分类和过滤'
},
// User information
user_id: {
type: 'string',
description: '创建链接的用户ID用于跟踪哪个用户创建了短链接'
},
user_name: {
type: 'string',
description: '用户名称,用于在报表中展示更易读的用户身份'
},
user_email: {
type: 'string',
format: 'email',
description: '用户电子邮件地址,可用于通知和报告分发'
},
user_attributes: {
type: 'string',
description: '用户附加属性的JSON字符串存储用户相关的额外信息如订阅级别、账户创建日期等'
},
// Team information
team_id: {
type: 'string',
description: '团队ID用于标识链接归属的团队支持多团队使用场景'
},
team_name: {
type: 'string',
description: '团队名称,用于在报表和管理界面中显示更友好的团队标识'
},
team_attributes: {
type: 'string',
description: '团队附加属性的JSON字符串存储团队相关的额外信息如部门、地区等'
},
// Project information
project_id: {
type: 'string',
description: '项目ID用于将链接归类到特定项目下便于项目级别的分析'
},
project_name: {
type: 'string',
description: '项目名称,提供更具描述性的项目标识,用于报表和管理界面'
},
project_attributes: {
type: 'string',
description: '项目附加属性的JSON字符串存储项目相关的额外信息如目标、预算等'
},
// QR code information
qr_code_id: {
type: 'string',
description: '二维码ID标识与事件关联的二维码用于跟踪二维码的使用情况'
},
qr_code_name: {
type: 'string',
description: '二维码名称,提供更具描述性的二维码标识,便于管理和报表'
},
qr_code_attributes: {
type: 'string',
description: '二维码附加属性的JSON字符串存储与二维码相关的额外信息如尺寸、颜色、logo等'
},
// Visitor information
visitor_id: {
type: 'string',
format: 'uuid',
description: '访问者唯一标识符,用于跟踪和识别独立访问者,分析用户行为'
},
session_id: {
type: 'string',
description: '会话标识符,用于将同一访问者的多个事件分组到同一会话中'
},
ip_address: {
type: 'string',
description: '访问者的IP地址用于地理位置分析和安全监控'
},
country: {
type: 'string',
description: '访问者所在国家,用于地理分布分析'
},
city: {
type: 'string',
description: '访问者所在城市,提供更精细的地理位置分析'
},
device_type: {
type: 'string',
description: '访问者使用的设备类型如mobile、desktop、tablet等用于设备分布分析'
},
browser: {
type: 'string',
description: '访问者使用的浏览器如Chrome、Safari、Firefox等用于浏览器分布分析'
},
os: {
type: 'string',
description: '访问者使用的操作系统如iOS、Android、Windows等用于操作系统分布分析'
},
user_agent: {
type: 'string',
description: '访问者的User-Agent字符串包含有关浏览器、操作系统和设备的详细信息'
},
// Referrer information
referrer: {
type: 'string',
description: '引荐来源URL指示用户从哪个网站或页面访问短链接用于分析流量来源'
},
utm_source: {
type: 'string',
description: 'UTM来源参数标识流量的来源渠道如Google、Facebook、Newsletter等'
},
utm_medium: {
type: 'string',
description: 'UTM媒介参数标识营销媒介类型如cpc、email、social等'
},
utm_campaign: {
type: 'string',
description: 'UTM活动参数标识特定的营销活动名称用于跟踪不同活动的效果'
},
// Interaction information
time_spent_sec: {
type: 'number',
description: '用户停留时间(秒),表示用户在目标页面上花费的时间,用于分析用户参与度'
},
is_bounce: {
type: 'boolean',
description: '是否为跳出访问,表示用户是否在查看单个页面后离开,不与网站进一步交互'
},
is_qr_scan: {
type: 'boolean',
description: '是否来自二维码扫描,用于区分和分析二维码带来的流量'
},
conversion_type: {
type: 'string',
description: '转化类型,表示事件触发的转化类型,如注册、购买、下载等,用于细分不同类型的转化'
},
conversion_value: {
type: 'number',
description: '转化价值,表示转化事件的经济价值或重要性,如购买金额、潜在客户价值等'
}
}
},
Event: {
type: 'object',
required: ['event_id', 'event_type', 'event_time', 'visitor_id'],
properties: {
event_id: {
type: 'string',
format: 'uuid',
description: '事件唯一标识符,用于唯一标识事件记录',
},
event_type: {
type: 'string',
enum: ['click', 'conversion'],
description: '事件类型用于分类不同的用户交互行为。click表示点击事件conversion表示转化事件',
},
event_time: {
type: 'string',
format: 'date-time',
description: '事件发生的时间戳,记录事件发生的精确时间',
},
link_id: {
type: 'string',
description: '短链接的唯一标识符,用于关联事件与特定短链接',
},
link_slug: {
type: 'string',
description: '短链接的短码/slug部分即URL路径中的短字符串',
},
link_original_url: {
type: 'string',
format: 'uri',
description: '短链接对应的原始目标URL即用户访问短链接后将被重定向到的实际URL',
},
visitor_id: {
type: 'string',
format: 'uuid',
description: '访问者唯一标识符,用于跟踪和识别独立访问者,分析用户行为',
},
device_type: {
type: 'string',
description: '访问者使用的设备类型如mobile、desktop、tablet等用于设备分布分析',
},
browser: {
type: 'string',
description: '访问者使用的浏览器如Chrome、Safari、Firefox等用于浏览器分布分析',
},
os: {
type: 'string',
description: '访问者使用的操作系统如iOS、Android、Windows等用于操作系统分布分析',
},
country: {
type: 'string',
description: '访问者所在国家,用于地理分布分析',
},
region: {
type: 'string',
description: '访问者所在地区/省份,提供中等精细度的地理位置分析',
},
city: {
type: 'string',
description: '访问者所在城市,提供更精细的地理位置分析',
},
referrer: {
type: 'string',
description: '引荐来源URL指示用户从哪个网站或页面访问短链接用于分析流量来源',
},
conversion_type: {
type: 'string',
description: '转化类型表示事件触发的转化类型如注册、购买、下载等仅当event_type为conversion时有效',
},
conversion_value: {
type: 'number',
description: '转化价值表示转化事件的经济价值或重要性如购买金额、潜在客户价值等仅当event_type为conversion时有效',
},
},
},
EventsSummary: {
type: 'object',
required: ['totalEvents', 'uniqueVisitors'],
properties: {
totalEvents: {
type: 'integer',
description: '时间段内的事件总数,包括所有类型的事件总计',
},
uniqueVisitors: {
type: 'integer',
description: '时间段内的独立访问者数量基于唯一访问者ID计算',
},
totalConversions: {
type: 'integer',
description: '时间段内的转化事件总数,用于衡量营销效果',
},
averageTimeSpent: {
type: 'number',
description: '平均停留时间(秒),表示用户平均在目标页面上停留的时间,是用户参与度的重要指标',
},
},
},
TimeSeriesData: {
type: 'object',
properties: {
timestamp: {
type: 'string',
format: 'date-time',
description: '时间序列中的时间点,表示数据采集的精确时间',
},
events: {
type: 'number',
description: '该时间点的事件数量,显示事件随时间的分布趋势',
},
visitors: {
type: 'number',
description: '该时间点的独立访问者数量,显示访问者随时间的分布趋势',
},
conversions: {
type: 'number',
description: '该时间点的转化数量,显示转化随时间的分布趋势',
},
},
},
GeoData: {
type: 'object',
properties: {
location: {
type: 'string',
description: '位置标识符,可以是国家、地区或城市的组合标识',
},
country: {
type: 'string',
description: '国家名称,表示访问者所在的国家',
},
region: {
type: 'string',
description: '地区/省份名称,表示访问者所在的地区或省份',
},
city: {
type: 'string',
description: '城市名称,表示访问者所在的城市',
},
visits: {
type: 'number',
description: '来自该位置的访问次数,用于分析不同地区的流量分布',
},
visitors: {
type: 'number',
description: '来自该位置的独立访问者数量,用于分析不同地区的用户分布',
},
percentage: {
type: 'number',
description: '占总访问量的百分比,便于直观比较不同地区的流量占比',
},
},
},
DeviceAnalytics: {
type: 'object',
properties: {
deviceTypes: {
type: 'array',
items: {
type: 'object',
properties: {
type: {
type: 'string',
description: '设备类型如mobile、desktop、tablet等用于设备类型分析',
},
count: {
type: 'number',
description: '使用该设备类型的访问次数,用于统计各类设备的使用情况',
},
percentage: {
type: 'number',
description: '该设备类型占总访问量的百分比,便于比较不同设备类型的使用占比',
},
},
},
},
browsers: {
type: 'array',
items: {
type: 'object',
properties: {
name: {
type: 'string',
description: '浏览器名称如Chrome、Safari、Firefox等用于浏览器使用分析',
},
count: {
type: 'number',
description: '使用该浏览器的访问次数,用于统计各类浏览器的使用情况',
},
percentage: {
type: 'number',
description: '该浏览器占总访问量的百分比,便于比较不同浏览器的使用占比',
},
},
},
},
operatingSystems: {
type: 'array',
items: {
type: 'object',
properties: {
name: {
type: 'string',
description: '操作系统名称如iOS、Android、Windows等用于操作系统使用分析',
},
count: {
type: 'number',
description: '使用该操作系统的访问次数,用于统计各类操作系统的使用情况',
},
percentage: {
type: 'number',
description: '该操作系统占总访问量的百分比,便于比较不同操作系统的使用占比',
},
},
},
},
},
},
Pagination: {
type: 'object',
required: ['page', 'pageSize', 'totalItems', 'totalPages'],
properties: {
page: {
type: 'integer',
description: '当前页码,表示结果集中的当前页面位置',
},
pageSize: {
type: 'integer',
description: '每页项目数,表示每页显示的结果数量',
},
totalItems: {
type: 'integer',
description: '总项目数,表示符合查询条件的结果总数',
},
totalPages: {
type: 'integer',
description: '总页数,基于总项目数和每页项目数计算得出',
},
},
},
Error: {
type: 'object',
required: ['code', 'message'],
properties: {
code: {
type: 'string',
description: '错误代码,用于标识特定类型的错误,便于客户端处理不同错误情况',
},
message: {
type: 'string',
description: '错误消息,提供关于错误的人类可读描述,帮助理解错误原因',
},
},
},
},
},
};
return (
<div className="container mx-auto px-4 py-8">
<div className="mb-8">
<h1 className="text-3xl font-bold mb-2">API Documentation</h1>
<p className="text-gray-600">
Explore and test the ShortURL Analytics API endpoints using the interactive documentation below.
</p>
</div>
<SwaggerUI spec={swaggerConfig} />
</div>
);
}

View File

@@ -1,28 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getDeviceAnalysis } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId') || undefined;
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 获取设备分析详情
const analysisData = await getDeviceAnalysis(
startDate,
endDate,
linkId
);
// 返回数据
return NextResponse.json(analysisData);
} catch (error) {
console.error('Error in device-analysis API:', error);
return NextResponse.json(
{ error: 'Failed to fetch device analysis data' },
{ status: 500 }
);
}
}

View File

@@ -1,36 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getConversionFunnel } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId');
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 验证必要参数
if (!linkId) {
return NextResponse.json(
{ error: 'Missing required parameter: linkId' },
{ status: 400 }
);
}
// 获取转化漏斗数据
const funnelData = await getConversionFunnel(
linkId,
startDate || undefined,
endDate || undefined
);
// 返回数据
return NextResponse.json(funnelData);
} catch (error) {
console.error('Error in funnel API:', error);
return NextResponse.json(
{ error: 'Failed to fetch funnel data' },
{ status: 500 }
);
}
}

View File

@@ -1,36 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getLinkPerformance } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId');
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 验证必要参数
if (!linkId) {
return NextResponse.json(
{ error: 'Missing required parameter: linkId' },
{ status: 400 }
);
}
// 获取链接表现数据
const performanceData = await getLinkPerformance(
linkId,
startDate || undefined,
endDate || undefined
);
// 返回数据
return NextResponse.json(performanceData);
} catch (error) {
console.error('Error in link-performance API:', error);
return NextResponse.json(
{ error: 'Failed to fetch link performance data' },
{ status: 500 }
);
}
}

View File

@@ -1,28 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getLinkStatusDistribution } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const projectId = searchParams.get('projectId') || undefined;
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 获取链接状态分布数据
const distributionData = await getLinkStatusDistribution(
startDate,
endDate,
projectId
);
// 返回数据
return NextResponse.json(distributionData);
} catch (error) {
console.error('Error in link-status-distribution API:', error);
return NextResponse.json(
{ error: 'Failed to fetch link status distribution data' },
{ status: 500 }
);
}
}

View File

@@ -1,28 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getOverviewCards } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const projectId = searchParams.get('projectId') || undefined;
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 获取概览卡片数据
const cardsData = await getOverviewCards(
startDate,
endDate,
projectId
);
// 返回数据
return NextResponse.json(cardsData);
} catch (error) {
console.error('Error in overview-cards API:', error);
return NextResponse.json(
{ error: 'Failed to fetch overview cards data' },
{ status: 500 }
);
}
}

View File

@@ -1,36 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getLinkOverview } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId');
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 验证必要参数
if (!linkId) {
return NextResponse.json(
{ error: 'Missing required parameter: linkId' },
{ status: 400 }
);
}
// 获取链接概览数据
const overviewData = await getLinkOverview(
linkId,
startDate || undefined,
endDate || undefined
);
// 返回数据
return NextResponse.json(overviewData);
} catch (error) {
console.error('Error in overview API:', error);
return NextResponse.json(
{ error: 'Failed to fetch overview data' },
{ status: 500 }
);
}
}

View File

@@ -1,28 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getPlatformDistribution } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId') || undefined;
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 获取平台分布数据
const distributionData = await getPlatformDistribution(
startDate,
endDate,
linkId
);
// 返回数据
return NextResponse.json(distributionData);
} catch (error) {
console.error('Error in platform-distribution API:', error);
return NextResponse.json(
{ error: 'Failed to fetch platform distribution data' },
{ status: 500 }
);
}
}

View File

@@ -1,32 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getPopularLinks } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const projectId = searchParams.get('projectId') || undefined;
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
const sortBy = searchParams.get('sortBy') as 'visits' | 'uniqueVisitors' | 'conversionRate' || 'visits';
const limit = searchParams.get('limit') ? parseInt(searchParams.get('limit') as string, 10) : 10;
// 获取热门链接数据
const linksData = await getPopularLinks(
startDate,
endDate,
projectId,
sortBy,
limit
);
// 返回数据
return NextResponse.json(linksData);
} catch (error) {
console.error('Error in popular-links API:', error);
return NextResponse.json(
{ error: 'Failed to fetch popular links data' },
{ status: 500 }
);
}
}

View File

@@ -1,32 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getPopularReferrers } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId') || undefined;
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
const type = searchParams.get('type') as 'domain' | 'full' || 'domain';
const limit = searchParams.get('limit') ? parseInt(searchParams.get('limit') as string, 10) : 10;
// 获取热门引荐来源数据
const referrersData = await getPopularReferrers(
startDate,
endDate,
linkId,
type,
limit
);
// 返回数据
return NextResponse.json(referrersData);
} catch (error) {
console.error('Error in popular-referrers API:', error);
return NextResponse.json(
{ error: 'Failed to fetch popular referrers data' },
{ status: 500 }
);
}
}

View File

@@ -1,30 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getQrCodeAnalysis } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId') || undefined;
const qrCodeId = searchParams.get('qrCodeId') || undefined;
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
// 获取QR码分析数据
const analysisData = await getQrCodeAnalysis(
startDate,
endDate,
linkId,
qrCodeId
);
// 返回数据
return NextResponse.json(analysisData);
} catch (error) {
console.error('Error in qr-code-analysis API:', error);
return NextResponse.json(
{ error: 'Failed to fetch QR code analysis data' },
{ status: 500 }
);
}
}

View File

@@ -1,68 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { trackEvent, EventType, ConversionType } from '@/lib/analytics';
export async function POST(request: NextRequest) {
try {
// 解析请求体
const body = await request.json();
// 验证必要字段
if (!body.linkId) {
return NextResponse.json(
{ error: 'Missing required field: linkId' },
{ status: 400 }
);
}
if (!body.eventType || !Object.values(EventType).includes(body.eventType)) {
return NextResponse.json(
{
error: 'Invalid or missing eventType',
validValues: Object.values(EventType)
},
{ status: 400 }
);
}
// 验证转化类型(如果提供)
if (
body.conversionType &&
!Object.values(ConversionType).includes(body.conversionType)
) {
return NextResponse.json(
{
error: 'Invalid conversionType',
validValues: Object.values(ConversionType)
},
{ status: 400 }
);
}
// 添加客户端IP
const clientIp = request.headers.get('x-forwarded-for') ||
request.headers.get('x-real-ip') ||
'0.0.0.0';
// 添加用户代理
const userAgent = request.headers.get('user-agent') || '';
// 合并数据
const eventData = {
...body,
ipAddress: body.ipAddress || clientIp,
userAgent: body.userAgent || userAgent,
};
// 追踪事件
const result = await trackEvent(eventData);
// 返回结果
return NextResponse.json(result);
} catch (error) {
console.error('Error in track API:', error);
return NextResponse.json(
{ error: 'Failed to track event' },
{ status: 500 }
);
}
}

View File

@@ -1,50 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getVisitTrends, TimeGranularity } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
// 获取请求参数
const { searchParams } = new URL(request.url);
const linkId = searchParams.get('linkId');
const startDate = searchParams.get('startDate') || undefined;
const endDate = searchParams.get('endDate') || undefined;
const granularity = searchParams.get('granularity') as TimeGranularity || TimeGranularity.DAY;
// 验证必要参数
if (!linkId) {
return NextResponse.json(
{ error: 'Missing required parameter: linkId' },
{ status: 400 }
);
}
// 验证粒度参数
const validGranularities = Object.values(TimeGranularity);
if (granularity && !validGranularities.includes(granularity)) {
return NextResponse.json(
{
error: 'Invalid granularity value',
validValues: validGranularities
},
{ status: 400 }
);
}
// 获取访问趋势数据
const trendsData = await getVisitTrends(
linkId,
startDate || undefined,
endDate || undefined,
granularity
);
// 返回数据
return NextResponse.json(trendsData);
} catch (error) {
console.error('Error in trends API:', error);
return NextResponse.json(
{ error: 'Failed to fetch trends data' },
{ status: 500 }
);
}
}

View File

@@ -0,0 +1,28 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse } from '@/lib/types';
import { getDeviceAnalytics } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
const data = await getDeviceAnalytics({
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
linkId: searchParams.get('linkId') || undefined
});
const response: ApiResponse<typeof data> = {
success: true,
data
};
return NextResponse.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -0,0 +1,29 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse } from '@/lib/types';
import { getGeoAnalytics } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
const data = await getGeoAnalytics({
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
linkId: searchParams.get('linkId') || undefined,
groupBy: (searchParams.get('groupBy') || 'country') as 'country' | 'city'
});
const response: ApiResponse<typeof data> = {
success: true,
data
};
return NextResponse.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

51
app/api/events/route.ts Normal file
View File

@@ -0,0 +1,51 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse, EventsQueryParams, EventType } from '@/lib/types';
import {
getEvents,
getEventsSummary,
getTimeSeriesData,
getGeoAnalytics,
getDeviceAnalytics
} from '@/lib/analytics';
// 获取事件列表
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
const params: EventsQueryParams = {
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
eventType: searchParams.get('eventType') as EventType || undefined,
linkId: searchParams.get('linkId') || undefined,
linkSlug: searchParams.get('linkSlug') || undefined,
userId: searchParams.get('userId') || undefined,
teamId: searchParams.get('teamId') || undefined,
projectId: searchParams.get('projectId') || undefined,
page: searchParams.has('page') ? parseInt(searchParams.get('page')!, 10) : 1,
pageSize: searchParams.has('pageSize') ? parseInt(searchParams.get('pageSize')!, 10) : 20,
sortBy: searchParams.get('sortBy') || undefined,
sortOrder: (searchParams.get('sortOrder') as 'asc' | 'desc') || undefined
};
const { events, total } = await getEvents(params);
const response: ApiResponse<typeof events> = {
success: true,
data: events,
meta: {
total,
page: params.page,
pageSize: params.pageSize
}
};
return NextResponse.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -0,0 +1,28 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse } from '@/lib/types';
import { getEventsSummary } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
const summary = await getEventsSummary({
startTime: searchParams.get('startTime') || undefined,
endTime: searchParams.get('endTime') || undefined,
linkId: searchParams.get('linkId') || undefined
});
const response: ApiResponse<typeof summary> = {
success: true,
data: summary
};
return NextResponse.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -0,0 +1,38 @@
import { NextRequest, NextResponse } from 'next/server';
import type { ApiResponse } from '@/lib/types';
import { getTimeSeriesData } from '@/lib/analytics';
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
const startTime = searchParams.get('startTime');
const endTime = searchParams.get('endTime');
if (!startTime || !endTime) {
return NextResponse.json({
success: false,
error: 'startTime and endTime are required'
}, { status: 400 });
}
const data = await getTimeSeriesData({
startTime,
endTime,
linkId: searchParams.get('linkId') || undefined,
granularity: (searchParams.get('granularity') || 'day') as 'hour' | 'day' | 'week' | 'month'
});
const response: ApiResponse<typeof data> = {
success: true,
data
};
return NextResponse.json(response);
} catch (error) {
const response: ApiResponse<null> = {
success: false,
error: error instanceof Error ? error.message : 'Unknown error occurred'
};
return NextResponse.json(response, { status: 500 });
}
}

View File

@@ -0,0 +1,137 @@
import { NextRequest, NextResponse } from 'next/server';
import { Event } from '../../types';
import { v4 as uuid } from 'uuid';
import clickhouse from '@/lib/clickhouse';
// 将时间格式化为ClickHouse兼容的格式YYYY-MM-DD HH:MM:SS.SSS
const formatDateTime = (date: Date) => {
return date.toISOString().replace('T', ' ').replace('Z', '');
};
// Handler for POST request to track events
export async function POST(req: NextRequest) {
try {
// Parse request body
const eventData = await req.json();
// Validate required fields
if (!eventData.event_type) {
return NextResponse.json(
{ error: 'Missing required field: event_type' },
{ status: 400 }
);
}
// 获取当前时间并格式化
const currentTime = formatDateTime(new Date());
// Set default values for required fields if missing
const event: Event = {
// Core event fields
event_id: eventData.event_id || uuid(),
event_time: eventData.event_time ? formatDateTime(new Date(eventData.event_time)) : currentTime,
event_type: eventData.event_type,
event_attributes: eventData.event_attributes || '{}',
// Link information
link_id: eventData.link_id || '',
link_slug: eventData.link_slug || '',
link_label: eventData.link_label || '',
link_title: eventData.link_title || '',
link_original_url: eventData.link_original_url || '',
link_attributes: eventData.link_attributes || '{}',
link_created_at: eventData.link_created_at ? formatDateTime(new Date(eventData.link_created_at)) : currentTime,
link_expires_at: eventData.link_expires_at ? formatDateTime(new Date(eventData.link_expires_at)) : null,
link_tags: eventData.link_tags || '[]',
// User information
user_id: eventData.user_id || '',
user_name: eventData.user_name || '',
user_email: eventData.user_email || '',
user_attributes: eventData.user_attributes || '{}',
// Team information
team_id: eventData.team_id || '',
team_name: eventData.team_name || '',
team_attributes: eventData.team_attributes || '{}',
// Project information
project_id: eventData.project_id || '',
project_name: eventData.project_name || '',
project_attributes: eventData.project_attributes || '{}',
// QR code information
qr_code_id: eventData.qr_code_id || '',
qr_code_name: eventData.qr_code_name || '',
qr_code_attributes: eventData.qr_code_attributes || '{}',
// Visitor information
visitor_id: eventData.visitor_id || uuid(),
session_id: eventData.session_id || uuid(),
ip_address: eventData.ip_address || req.headers.get('x-forwarded-for')?.toString() || '',
country: eventData.country || '',
city: eventData.city || '',
device_type: eventData.device_type || '',
browser: eventData.browser || '',
os: eventData.os || '',
user_agent: eventData.user_agent || req.headers.get('user-agent')?.toString() || '',
// Referrer information
referrer: eventData.referrer || req.headers.get('referer')?.toString() || '',
utm_source: eventData.utm_source || '',
utm_medium: eventData.utm_medium || '',
utm_campaign: eventData.utm_campaign || '',
// Interaction information
time_spent_sec: eventData.time_spent_sec || 0,
is_bounce: eventData.is_bounce !== undefined ? eventData.is_bounce : true,
is_qr_scan: eventData.is_qr_scan !== undefined ? eventData.is_qr_scan : false,
conversion_type: eventData.conversion_type || '',
conversion_value: eventData.conversion_value || 0,
};
// 确保JSON字符串字段的正确处理
if (typeof event.event_attributes === 'object') {
event.event_attributes = JSON.stringify(event.event_attributes);
}
if (typeof event.link_attributes === 'object') {
event.link_attributes = JSON.stringify(event.link_attributes);
}
if (typeof event.user_attributes === 'object') {
event.user_attributes = JSON.stringify(event.user_attributes);
}
if (typeof event.team_attributes === 'object') {
event.team_attributes = JSON.stringify(event.team_attributes);
}
if (typeof event.project_attributes === 'object') {
event.project_attributes = JSON.stringify(event.project_attributes);
}
if (typeof event.qr_code_attributes === 'object') {
event.qr_code_attributes = JSON.stringify(event.qr_code_attributes);
}
if (typeof event.link_tags === 'object') {
event.link_tags = JSON.stringify(event.link_tags);
}
// Insert event into ClickHouse
await clickhouse.insert({
table: 'events',
values: [event],
format: 'JSONEachRow',
});
// Return success response
return NextResponse.json({
success: true,
message: 'Event tracked successfully',
event_id: event.event_id
}, { status: 201 });
} catch (error) {
console.error('Error tracking event:', error);
return NextResponse.json(
{ error: 'Failed to track event', details: (error as Error).message },
{ status: 500 }
);
}
}

View File

@@ -1,30 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getLinkDetailsById } from '@/app/api/links/service';
// 正确的Next.js 15 API路由处理函数参数类型定义
export async function GET(
request: NextRequest,
context: { params: Promise<any> }
) {
try {
// 获取参数,支持异步格式
const params = await context.params;
const linkId = params.linkId;
const link = await getLinkDetailsById(linkId);
if (!link) {
return NextResponse.json(
{ error: 'Link not found' },
{ status: 404 }
);
}
return NextResponse.json(link);
} catch (error) {
console.error('Failed to fetch link details:', error);
return NextResponse.json(
{ error: 'Failed to fetch link details', message: (error as Error).message },
{ status: 500 }
);
}
}

View File

@@ -1,29 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { getLinkById } from '../service';
export async function GET(
request: NextRequest,
context: { params: Promise<any> }
) {
try {
// 获取参数,支持异步格式
const params = await context.params;
const linkId = params.linkId;
const link = await getLinkById(linkId);
if (!link) {
return NextResponse.json(
{ error: 'Link not found' },
{ status: 404 }
);
}
return NextResponse.json(link);
} catch (error) {
console.error('Failed to fetch link:', error);
return NextResponse.json(
{ error: 'Failed to fetch link', message: (error as Error).message },
{ status: 500 }
);
}
}

View File

@@ -1,157 +0,0 @@
import { executeQuery, executeQuerySingle } from '@/lib/clickhouse';
import { Link, LinkQueryParams } from '../types';
/**
* Find links with filtering options
*/
export async function findLinks({
limit = 10,
offset = 0,
searchTerm = '',
tagFilter = '',
isActive = null,
}: LinkQueryParams) {
// Build WHERE conditions
const conditions = [];
if (searchTerm) {
conditions.push(`
(lower(title) LIKE lower('%${searchTerm}%') OR
lower(original_url) LIKE lower('%${searchTerm}%'))
`);
}
if (tagFilter) {
conditions.push(`hasAny(tags, ['${tagFilter}'])`);
}
if (isActive !== null) {
conditions.push(`is_active = ${isActive ? 'true' : 'false'}`);
}
const whereClause = conditions.length > 0
? `WHERE ${conditions.join(' AND ')}`
: '';
// Get total count
const countQuery = `
SELECT count() as total
FROM links
${whereClause}
`;
const countData = await executeQuery<{ total: number }>(countQuery);
const total = countData.length > 0 ? countData[0].total : 0;
// 使用左连接获取链接数据和统计信息
const linksQuery = `
SELECT
l.link_id,
l.original_url,
l.created_at,
l.created_by,
l.title,
l.description,
l.tags,
l.is_active,
l.expires_at,
l.team_id,
l.project_id,
count(le.event_id) as visits,
count(DISTINCT le.visitor_id) as unique_visits
FROM links l
LEFT JOIN link_events le ON l.link_id = le.link_id
${whereClause}
GROUP BY
l.link_id,
l.original_url,
l.created_at,
l.created_by,
l.title,
l.description,
l.tags,
l.is_active,
l.expires_at,
l.team_id,
l.project_id
ORDER BY l.created_at DESC
LIMIT ${limit}
OFFSET ${offset}
`;
const links = await executeQuery<Link>(linksQuery);
return {
links,
total,
limit,
offset,
page: Math.floor(offset / limit) + 1,
totalPages: Math.ceil(total / limit)
};
}
/**
* Find a single link by ID
*/
export async function findLinkById(linkId: string): Promise<Link | null> {
const query = `
SELECT
l.link_id,
l.original_url,
l.created_at,
l.created_by,
l.title,
l.description,
l.tags,
l.is_active,
l.expires_at,
l.team_id,
l.project_id,
count(le.event_id) as visits,
count(DISTINCT le.visitor_id) as unique_visits
FROM links l
LEFT JOIN link_events le ON l.link_id = le.link_id
WHERE l.link_id = '${linkId}'
GROUP BY
l.link_id,
l.original_url,
l.created_at,
l.created_by,
l.title,
l.description,
l.tags,
l.is_active,
l.expires_at,
l.team_id,
l.project_id
LIMIT 1
`;
return await executeQuerySingle<Link>(query);
}
/**
* Find a single link by ID - only basic info without statistics
*/
export async function findLinkDetailsById(linkId: string): Promise<Omit<Link, 'visits' | 'unique_visits'> | null> {
const query = `
SELECT
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active,
expires_at,
team_id,
project_id
FROM links
WHERE link_id = '${linkId}'
LIMIT 1
`;
return await executeQuerySingle<Omit<Link, 'visits' | 'unique_visits'>>(query);
}

View File

@@ -1,32 +0,0 @@
import { NextRequest, NextResponse } from 'next/server';
import { LinkQueryParams } from '../types';
import { getLinks } from './service';
export async function GET(request: NextRequest) {
try {
const searchParams = request.nextUrl.searchParams;
// Parse request parameters
const params: LinkQueryParams = {
limit: searchParams.has('limit') ? Number(searchParams.get('limit')) : 10,
page: searchParams.has('page') ? Number(searchParams.get('page')) : 1,
searchTerm: searchParams.get('search') || '',
tagFilter: searchParams.get('tag') || '',
};
// Handle active status filter
const activeFilter = searchParams.get('active');
if (activeFilter === 'true') params.isActive = true;
if (activeFilter === 'false') params.isActive = false;
// Get link data
const result = await getLinks(params);
return NextResponse.json(result);
} catch (error) {
console.error('Failed to fetch links:', error);
return NextResponse.json(
{ error: 'Failed to fetch links', message: (error as Error).message },
{ status: 500 }
);
}
}

View File

@@ -1,42 +0,0 @@
import { Link, LinkQueryParams, PaginatedResponse } from '../types';
import { findLinkById, findLinkDetailsById, findLinks } from './repository';
/**
* Get links with pagination information
*/
export async function getLinks(params: LinkQueryParams): Promise<PaginatedResponse<Link>> {
// Convert page number to offset
const { page, limit = 10, ...otherParams } = params;
const offset = page ? (page - 1) * limit : params.offset || 0;
const result = await findLinks({
...otherParams,
limit,
offset
});
return {
data: result.links,
pagination: {
total: result.total,
limit: result.limit,
offset: result.offset,
page: result.page,
totalPages: result.totalPages
}
};
}
/**
* Get a single link by ID with full details (including statistics)
*/
export async function getLinkById(linkId: string): Promise<Link | null> {
return await findLinkById(linkId);
}
/**
* Get a single link by ID - only basic info without statistics
*/
export async function getLinkDetailsById(linkId: string): Promise<Omit<Link, 'visits' | 'unique_visits'> | null> {
return await findLinkDetailsById(linkId);
}

View File

@@ -1,21 +0,0 @@
import { executeQuerySingle } from '@/lib/clickhouse';
import { StatsOverview } from '../types';
/**
* Get overview statistics for links
*/
export async function findStatsOverview(): Promise<StatsOverview | null> {
const query = `
WITH
toUInt64(count()) as total_links,
toUInt64(countIf(is_active = true)) as active_links
FROM links
SELECT
total_links as totalLinks,
active_links as activeLinks,
(SELECT count() FROM link_events) as totalVisits,
(SELECT count() FROM link_events) / NULLIF(total_links, 0) as conversionRate
`;
return await executeQuerySingle<StatsOverview>(query);
}

View File

@@ -1,15 +0,0 @@
import { NextResponse } from 'next/server';
import { getStatsOverview } from './service';
export async function GET() {
try {
const stats = await getStatsOverview();
return NextResponse.json(stats);
} catch (error) {
console.error('获取统计概览失败:', error);
return NextResponse.json(
{ error: '获取统计概览失败', message: (error as Error).message },
{ status: 500 }
);
}
}

View File

@@ -1,21 +0,0 @@
import { StatsOverview } from '../types';
import { findStatsOverview } from './repository';
/**
* Get link statistics overview
*/
export async function getStatsOverview(): Promise<StatsOverview> {
const stats = await findStatsOverview();
// Return default values if no data
if (!stats) {
return {
totalLinks: 0,
activeLinks: 0,
totalVisits: 0,
conversionRate: 0
};
}
return stats;
}

View File

@@ -1,19 +0,0 @@
import { executeQuery } from '@/lib/clickhouse';
import { Tag } from '../types';
/**
* Get all tags with usage counts
*/
export async function findAllTags(): Promise<Tag[]> {
const query = `
SELECT
tag,
count() as count
FROM links
ARRAY JOIN tags as tag
GROUP BY tag
ORDER BY count DESC
`;
return await executeQuery<Tag>(query);
}

View File

@@ -1,15 +0,0 @@
import { NextResponse } from 'next/server';
import { getAllTags } from './service';
export async function GET() {
try {
const tags = await getAllTags();
return NextResponse.json(tags);
} catch (error) {
console.error('Failed to fetch tags:', error);
return NextResponse.json(
{ error: 'Failed to fetch tags', message: (error as Error).message },
{ status: 500 }
);
}
}

View File

@@ -1,9 +0,0 @@
import { Tag } from '../types';
import { findAllTags } from './repository';
/**
* Get all available tags
*/
export async function getAllTags(): Promise<Tag[]> {
return await findAllTags();
}

View File

@@ -1,221 +1,169 @@
// 链接数据类型
export interface Link {
// Event Types
export interface Event {
// 核心事件信息
event_id: string;
event_time: string;
event_type: string;
event_attributes: string;
// 链接信息
link_id: string;
original_url: string;
created_at: string;
created_by: string;
title: string;
description: string;
tags: string[];
is_active: boolean;
expires_at: string | null;
link_slug: string;
link_label: string;
link_title: string;
link_original_url: string;
link_attributes: string;
link_created_at: string;
link_expires_at: string | null;
link_tags: string;
// 用户信息
user_id: string;
user_name: string;
user_email: string;
user_attributes: string;
// 团队信息
team_id: string;
team_name: string;
team_attributes: string;
// 项目信息
project_id: string;
project_name: string;
project_attributes: string;
// 二维码信息
qr_code_id: string;
qr_code_name: string;
qr_code_attributes: string;
// 访问者信息
visitor_id: string;
session_id: string;
ip_address: string;
country: string;
city: string;
device_type: string;
browser: string;
os: string;
user_agent: string;
// 来源信息
referrer: string;
utm_source: string;
utm_medium: string;
utm_campaign: string;
// 交互信息
time_spent_sec: number;
is_bounce: boolean;
is_qr_scan: boolean;
conversion_type: string;
conversion_value: number;
// 旧接口兼容字段
id?: string;
time?: string;
type?: string;
linkInfo?: {
id: string;
shortUrl: string;
originalUrl: string;
};
visitor?: {
id: string;
browser: string;
os: string;
device: string;
};
location?: {
country: string;
region: string;
city: string;
};
}
// Analytics Types
export interface TimeSeriesData {
timestamp: string;
events: number;
visitors: number;
conversions: number;
}
export interface GeoData {
location?: string;
country?: string;
region?: string;
city?: string;
visits: number;
unique_visits: number;
uniqueVisitors?: number;
visitors?: number;
percentage: number;
}
// 分页响应类型
export interface PaginatedResponse<T> {
data: T[];
pagination: {
total: number;
limit: number;
offset: number;
page: number;
totalPages: number;
}
export type DeviceType = 'mobile' | 'desktop' | 'tablet' | 'other';
export interface DeviceAnalytics {
deviceTypes: {
type: string;
count: number;
percentage: number;
}[];
browsers: {
name: string;
count: number;
percentage: number;
}[];
operatingSystems: {
name: string;
count: number;
percentage: number;
}[];
}
// 链接查询参数
export interface LinkQueryParams {
limit?: number;
offset?: number;
page?: number;
searchTerm?: string;
tagFilter?: string;
isActive?: boolean | null;
}
// 标签类型
export interface Tag {
tag: string;
count: number;
}
// 统计概览类型
export interface StatsOverview {
totalLinks: number;
activeLinks: number;
totalVisits: number;
conversionRate: number;
}
// Analytics数据类型
export interface LinkOverviewData {
totalVisits: number;
export interface EventsSummary {
totalEvents: number;
uniqueVisitors: number;
totalConversions: number;
averageTimeSpent: number;
bounceCount: number;
conversionCount: number;
uniqueReferrers: number;
deviceTypes: {
mobile: number;
tablet: number;
desktop: number;
tablet: number;
other: number;
};
qrScanCount: number;
totalConversionValue: number;
browsers: {
name: string;
count: number;
percentage: number;
}[];
operatingSystems: {
name: string;
count: number;
percentage: number;
}[];
}
export interface FunnelStep {
name: string;
value: number;
percent: number;
}
export interface ConversionFunnelData {
steps: FunnelStep[];
export interface ConversionStats {
totalConversions: number;
conversionRate: number;
averageValue: number;
byType: {
type: string;
count: number;
percentage: number;
value: number;
}[];
}
export interface TrendPoint {
timestamp: string;
visits: number;
uniqueVisitors: number;
}
export interface VisitTrendsData {
trends: TrendPoint[];
totals: {
visits: number;
uniqueVisitors: number;
};
}
export interface TrackEventRequest {
linkId: string;
eventType: string;
visitorId?: string;
sessionId?: string;
referrer?: string;
userAgent?: string;
ipAddress?: string;
timeSpent?: number;
conversionType?: string;
conversionValue?: number;
customData?: Record<string, unknown>;
isQrScan?: boolean;
qrCodeId?: string;
utmSource?: string;
utmMedium?: string;
utmCampaign?: string;
}
export interface TrackEventResponse {
success: boolean;
eventId: string;
timestamp: string;
}
// 链接表现数据
export interface LinkPerformanceData {
totalClicks: number;
uniqueVisitors: number;
averageTimeSpent: number;
bounceRate: number;
uniqueReferrers: number;
conversionRate: number;
activeDays: number;
lastClickTime: string | null;
deviceDistribution: {
mobile: number;
desktop: number;
};
}
// 平台分布数据
export interface PlatformItem {
name: string;
count: number;
percent: number;
}
export interface PlatformDistributionData {
totalVisits: number;
platforms: PlatformItem[];
browsers: PlatformItem[];
}
// 设备分析数据
export interface DeviceItem {
name: string;
count: number;
percent: number;
}
export interface DeviceModelItem {
type: string;
brand: string;
model: string;
count: number;
percent: number;
}
export interface DeviceAnalysisData {
totalVisits: number;
deviceTypes: DeviceItem[];
deviceBrands: DeviceItem[];
deviceModels: DeviceModelItem[];
}
// 热门引荐来源数据
export interface ReferrerItem {
source: string;
visitCount: number;
uniqueVisitors: number;
conversionCount: number;
conversionRate: number;
averageTimeSpent: number;
percent: number;
}
export interface PopularReferrersData {
referrers: ReferrerItem[];
totalVisits: number;
}
// QR码分析数据
export interface LocationItem {
city: string;
country: string;
scanCount: number;
percent: number;
}
export interface DeviceDistributionItem {
type: string;
count: number;
percent: number;
}
export interface HourlyDistributionItem {
hour: number;
scanCount: number;
percent: number;
}
export interface QrCodeAnalysisData {
overview: {
totalScans: number;
uniqueScanners: number;
conversionCount: number;
conversionRate: number;
averageTimeSpent: number;
};
locations: LocationItem[];
deviceDistribution: DeviceDistributionItem[];
hourlyDistribution: HourlyDistributionItem[];
export interface EventFilters {
startTime?: string;
endTime?: string;
eventType?: string;
linkId?: string;
linkSlug?: string;
page?: number;
pageSize?: number;
}

View File

@@ -0,0 +1,68 @@
"use client";
import { useEffect, useRef } from 'react';
import { DeviceAnalytics as DeviceAnalyticsType } from '@/app/api/types';
import { Chart, PieController, ArcElement, Tooltip, Legend } from 'chart.js';
interface DeviceAnalyticsProps {
data: DeviceAnalyticsType;
}
function StatCard({ title, items }: { title: string; items: { name: string; count: number; percentage: number }[] }) {
// 安全地格式化数字
const formatNumber = (value: number | string | undefined | null): string => {
if (value === undefined || value === null) return '0';
return typeof value === 'number' ? value.toLocaleString() : String(value);
};
// 安全地格式化百分比
const formatPercent = (value: number | undefined | null): string => {
if (value === undefined || value === null) return '0';
return value.toFixed(1);
};
return (
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
<h3 className="text-lg font-semibold text-gray-900 dark:text-gray-100 mb-4">{title}</h3>
<div className="space-y-4">
{items.map((item, index) => (
<div key={index}>
<div className="flex justify-between text-sm text-gray-600 dark:text-gray-400 mb-1">
<span>{item.name || 'Unknown'}</span>
<span>{formatNumber(item.count)} ({formatPercent(item.percentage)}%)</span>
</div>
<div className="w-full bg-gray-200 dark:bg-gray-700 rounded-full h-2">
<div
className="bg-blue-500 h-2 rounded-full"
style={{ width: `${item.percentage || 0}%` }}
/>
</div>
</div>
))}
</div>
</div>
);
}
export default function DeviceAnalytics({ data }: DeviceAnalyticsProps) {
return (
<div className="grid grid-cols-1 md:grid-cols-3 gap-6">
<StatCard
title="Device Types"
items={(data.deviceTypes || []).map(item => ({
name: item.type ? (item.type.charAt(0).toUpperCase() + item.type.slice(1)) : 'Unknown',
count: item.count,
percentage: item.percentage
}))}
/>
<StatCard
title="Browsers"
items={data.browsers || []}
/>
<StatCard
title="Operating Systems"
items={data.operatingSystems || []}
/>
</div>
);
}

View File

@@ -0,0 +1,72 @@
"use client";
import { useEffect, useRef } from 'react';
import { GeoData } from '@/app/api/types';
import { Chart, PieController, ArcElement, Tooltip, Legend } from 'chart.js';
interface GeoAnalyticsProps {
data: GeoData[];
}
export default function GeoAnalytics({ data }: GeoAnalyticsProps) {
// 安全地格式化数字
const formatNumber = (value: any): string => {
if (value === undefined || value === null) return '0';
return typeof value === 'number' ? value.toLocaleString() : String(value);
};
// 安全地格式化百分比
const formatPercent = (value: any): string => {
if (value === undefined || value === null) return '0';
return typeof value === 'number' ? value.toFixed(2) : String(value);
};
return (
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-gray-200 dark:divide-gray-700">
<thead className="bg-gray-50 dark:bg-gray-800">
<tr>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Location
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Visits
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Unique Visitors
</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Percentage
</th>
</tr>
</thead>
<tbody className="bg-white dark:bg-gray-900 divide-y divide-gray-200 dark:divide-gray-700">
{data.map((item, index) => (
<tr key={index} className={index % 2 === 0 ? 'bg-white dark:bg-gray-900' : 'bg-gray-50 dark:bg-gray-800'}>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
{item.city ? `${item.city}, ${item.region}, ${item.country}` : item.region ? `${item.region}, ${item.country}` : item.country || item.location || 'Unknown'}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
{formatNumber(item.visits)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
{formatNumber(item.uniqueVisitors || item.visitors)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900 dark:text-gray-100">
<div className="flex items-center">
<span className="mr-2">{formatPercent(item.percentage)}%</span>
<div className="w-24 bg-gray-200 dark:bg-gray-700 rounded-full h-2">
<div
className="bg-blue-500 h-2 rounded-full"
style={{ width: `${item.percentage || 0}%` }}
/>
</div>
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
);
}

View File

@@ -0,0 +1,184 @@
"use client";
import { useEffect, useRef } from 'react';
import {
Chart as ChartJS,
CategoryScale,
LinearScale,
PointElement,
LineElement,
LineController,
Title,
Tooltip,
Legend,
Filler,
ChartData,
ChartOptions,
TooltipItem
} from 'chart.js';
import { TimeSeriesData } from '@/app/api/types';
// 注册 Chart.js 组件
ChartJS.register(
CategoryScale,
LinearScale,
PointElement,
LineElement,
LineController,
Title,
Tooltip,
Legend,
Filler
);
interface TimeSeriesChartProps {
data: TimeSeriesData[];
}
export default function TimeSeriesChart({ data }: TimeSeriesChartProps) {
const chartRef = useRef<HTMLCanvasElement | null>(null);
const chartInstance = useRef<ChartJS | null>(null);
useEffect(() => {
if (!chartRef.current) return;
// 销毁旧的图表实例
if (chartInstance.current) {
chartInstance.current.destroy();
}
const ctx = chartRef.current.getContext('2d');
if (!ctx) return;
// 准备数据
const labels = data.map(item => {
if (!item || !item.timestamp) return '';
const date = new Date(item.timestamp);
return date.toLocaleDateString();
});
const eventsData = data.map(item => {
if (!item || item.events === undefined || item.events === null) return 0;
return Number(item.events);
});
const visitorsData = data.map(item => {
if (!item || item.visitors === undefined || item.visitors === null) return 0;
return Number(item.visitors);
});
const conversionsData = data.map(item => {
if (!item || item.conversions === undefined || item.conversions === null) return 0;
return Number(item.conversions);
});
// 创建新的图表实例
chartInstance.current = new ChartJS(ctx, {
type: 'line',
data: {
labels,
datasets: [
{
label: 'Events',
data: eventsData,
borderColor: 'rgb(59, 130, 246)', // blue-500
backgroundColor: 'rgba(59, 130, 246, 0.1)',
tension: 0.4,
fill: true
},
{
label: 'Visitors',
data: visitorsData,
borderColor: 'rgb(16, 185, 129)', // green-500
backgroundColor: 'rgba(16, 185, 129, 0.1)',
tension: 0.4,
fill: true
},
{
label: 'Conversions',
data: conversionsData,
borderColor: 'rgb(239, 68, 68)', // red-500
backgroundColor: 'rgba(239, 68, 68, 0.1)',
tension: 0.4,
fill: true
}
]
} as ChartData<'line'>,
options: {
responsive: true,
maintainAspectRatio: false,
interaction: {
mode: 'index',
intersect: false
},
plugins: {
legend: {
position: 'top',
labels: {
usePointStyle: true,
padding: 20,
color: 'rgb(156, 163, 175)' // gray-400
}
},
tooltip: {
mode: 'index',
intersect: false,
backgroundColor: 'rgb(31, 41, 55)', // gray-800
titleColor: 'rgb(229, 231, 235)', // gray-200
bodyColor: 'rgb(229, 231, 235)', // gray-200
borderColor: 'rgb(75, 85, 99)', // gray-600
borderWidth: 1,
padding: 12,
displayColors: true,
callbacks: {
title: (items: TooltipItem<'line'>[]) => {
if (items.length > 0) {
const date = new Date(data[items[0].dataIndex].timestamp);
return date.toLocaleDateString();
}
return '';
}
}
}
},
scales: {
x: {
grid: {
display: false
},
ticks: {
color: 'rgb(156, 163, 175)' // gray-400
}
},
y: {
beginAtZero: true,
grid: {
color: 'rgb(75, 85, 99, 0.1)' // gray-600 with opacity
},
ticks: {
color: 'rgb(156, 163, 175)', // gray-400
callback: (value: number) => {
if (!value && value !== 0) return '';
if (value >= 1000) {
return `${(value / 1000).toFixed(1)}k`;
}
return value;
}
}
}
}
} as ChartOptions<'line'>
});
// 清理函数
return () => {
if (chartInstance.current) {
chartInstance.current.destroy();
}
};
}, [data]);
return (
<canvas ref={chartRef} />
);
}

View File

@@ -0,0 +1,71 @@
"use client";
import { useState, useEffect } from 'react';
import { format } from 'date-fns';
interface DateRangePickerProps {
value: {
from: Date;
to: Date;
};
onChange: (range: { from: Date; to: Date }) => void;
}
export function DateRangePicker({ value, onChange }: DateRangePickerProps) {
const [from, setFrom] = useState(format(value.from, 'yyyy-MM-dd'));
const [to, setTo] = useState(format(value.to, 'yyyy-MM-dd'));
useEffect(() => {
setFrom(format(value.from, 'yyyy-MM-dd'));
setTo(format(value.to, 'yyyy-MM-dd'));
}, [value]);
const handleFromChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newFrom = e.target.value;
setFrom(newFrom);
onChange({
from: new Date(newFrom),
to: value.to
});
};
const handleToChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newTo = e.target.value;
setTo(newTo);
onChange({
from: value.from,
to: new Date(newTo)
});
};
return (
<div className="flex items-center space-x-4">
<div>
<label htmlFor="from" className="block text-sm font-medium text-gray-500 dark:text-gray-400 mb-1">
From
</label>
<input
type="date"
id="from"
value={from}
onChange={handleFromChange}
max={to}
className="block w-full px-3 py-2 bg-white dark:bg-gray-800 border border-gray-300 dark:border-gray-600 rounded-md text-sm text-gray-900 dark:text-gray-100 focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
/>
</div>
<div>
<label htmlFor="to" className="block text-sm font-medium text-gray-500 dark:text-gray-400 mb-1">
To
</label>
<input
type="date"
id="to"
value={to}
onChange={handleToChange}
min={from}
className="block w-full px-3 py-2 bg-white dark:bg-gray-800 border border-gray-300 dark:border-gray-600 rounded-md text-sm text-gray-900 dark:text-gray-100 focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
/>
</div>
</div>
);
}

View File

@@ -1,21 +1,9 @@
import './globals.css';
import type { Metadata } from 'next';
import { Geist, Geist_Mono } from "next/font/google";
import Navbar from "./components/layout/Navbar";
const geistSans = Geist({
variable: "--font-geist-sans",
subsets: ["latin"],
});
const geistMono = Geist_Mono({
variable: "--font-geist-mono",
subsets: ["latin"],
});
export const metadata: Metadata = {
title: 'ShortURL Analytics',
description: 'Analytics dashboard for short URL management',
title: 'Link Management & Analytics',
description: 'Track and analyze shortened links',
};
export default function RootLayout({
@@ -25,14 +13,9 @@ export default function RootLayout({
}>) {
return (
<html lang="en" suppressHydrationWarning>
<body
className={`${geistSans.variable} ${geistMono.variable} antialiased bg-background`}
>
<Navbar />
<main className="min-h-screen px-4 py-6">
{children}
</main>
<body>
{children}
</body>
</html>
);
}
}

View File

@@ -1,21 +0,0 @@
import './globals.css';
import type { Metadata } from 'next';
export const metadata: Metadata = {
title: 'Link Management & Analytics',
description: 'Track and analyze shortened links',
};
export default function RootLayout({
children,
}: Readonly<{
children: React.ReactNode;
}>) {
return (
<html lang="en" suppressHydrationWarning>
<body>
{children}
</body>
</html>
);
}

View File

@@ -1,5 +1,85 @@
import { redirect } from 'next/navigation';
import Link from 'next/link';
export default function Home() {
redirect('/links');
export default function HomePage() {
const sections = [
{
title: 'Dashboard',
description: 'Get an overview of your link performance with key metrics and trends.',
href: '/dashboard',
icon: (
<svg className="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 19v-6a2 2 0 00-2-2H5a2 2 0 00-2 2v6a2 2 0 002 2h2a2 2 0 002-2zm0 0V9a2 2 0 012-2h2a2 2 0 012 2v10m-6 0a2 2 0 002 2h2a2 2 0 002-2m0 0V5a2 2 0 012-2h2a2 2 0 012 2v14a2 2 0 01-2 2h-2a2 2 0 01-2-2z" />
</svg>
),
},
{
title: 'Events',
description: 'Track and analyze all events including clicks, conversions, and more.',
href: '/events',
icon: (
<svg className="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 8v4l3 3m6-3a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
),
},
{
title: 'Geographic',
description: 'See where your visitors are coming from with detailed location data.',
href: '/analytics/geo',
icon: (
<svg className="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M3.055 11H5a2 2 0 012 2v1a2 2 0 002 2 2 2 0 012 2v2.945M8 3.935V5.5A2.5 2.5 0 0010.5 8h.5a2 2 0 012 2 2 2 0 104 0 2 2 0 012-2h1.064M15 20.488V18a2 2 0 012-2h3.064M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
),
},
{
title: 'Devices',
description: 'Understand what devices, browsers, and operating systems your visitors use.',
href: '/analytics/devices',
icon: (
<svg className="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9.75 17L9 20l-1 1h8l-1-1-.75-3M3 13h18M5 17h14a2 2 0 002-2V5a2 2 0 00-2-2H5a2 2 0 00-2 2v10a2 2 0 002 2z" />
</svg>
),
},
];
return (
<div className="container mx-auto px-4">
<div className="max-w-4xl mx-auto">
<div className="text-center mb-12">
<h1 className="text-4xl font-bold text-gray-900 dark:text-gray-100 mb-4">
Welcome to ShortURL Analytics
</h1>
<p className="text-lg text-gray-600 dark:text-gray-400">
Get detailed insights into your link performance and visitor behavior
</p>
</div>
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
{sections.map((section) => (
<Link
key={section.title}
href={section.href}
className="group block p-6 bg-white dark:bg-gray-800 rounded-lg shadow-sm hover:shadow-md transition-shadow duration-200"
>
<div className="flex items-center mb-4">
<div className="p-2 bg-blue-100 dark:bg-blue-900 rounded-lg mr-4">
<div className="text-blue-600 dark:text-blue-300">
{section.icon}
</div>
</div>
<h2 className="text-xl font-semibold text-gray-900 dark:text-gray-100 group-hover:text-blue-600 dark:group-hover:text-blue-400 transition-colors duration-200">
{section.title}
</h2>
</div>
<p className="text-gray-600 dark:text-gray-400">
{section.description}
</p>
</Link>
))}
</div>
</div>
</div>
);
}

151
docs/swagger-setup.md Normal file
View File

@@ -0,0 +1,151 @@
# Setting up Swagger UI in Next.js
This guide explains how to set up Swagger UI in a Next.js application using route groups.
## Directory Structure
The recommended directory structure for Swagger documentation:
```
app/
(swagger)/ # Route group for swagger-related pages
swagger/ # Actual swagger route
page.tsx # Swagger UI component
```
## Installation
1. Add Swagger UI dependencies to your project:
```json
{
"dependencies": {
"swagger-ui-react": "^5.12.0",
"swagger-ui-dist": "^5.12.0"
},
"devDependencies": {
"@types/swagger-ui-react": "^4.18.3"
}
}
```
2. Install webpack style loaders for handling Swagger UI CSS:
```bash
pnpm add -D style-loader css-loader
```
## Next.js Configuration
Create or update `next.config.js` to handle Swagger UI CSS:
```javascript
/** @type {import('next').NextConfig} */
const nextConfig = {
transpilePackages: ['swagger-ui-react'],
webpack: (config) => {
config.module.rules.push({
test: /\.css$/,
use: ['style-loader', 'css-loader'],
});
return config;
},
};
module.exports = nextConfig;
```
## Swagger UI Component
Create `app/(swagger)/swagger/page.tsx`:
```typescript
"use client";
import { useEffect } from 'react';
import SwaggerUI from 'swagger-ui-react';
import 'swagger-ui-react/swagger-ui.css';
export default function SwaggerPage() {
useEffect(() => {
document.title = 'API Documentation - ShortURL Analytics';
}, []);
const swaggerConfig = {
openapi: '3.0.0',
info: {
title: 'Your API Title',
version: '1.0.0',
description: 'API documentation',
contact: {
name: 'API Support',
email: 'support@example.com',
},
license: {
name: 'MIT',
url: 'https://opensource.org/licenses/MIT',
},
},
// ... your API configuration
};
return (
<div className="container mx-auto px-4 py-8">
<div className="mb-8">
<h1 className="text-3xl font-bold mb-2">API Documentation</h1>
<p className="text-gray-600">
Explore and test the API endpoints using the interactive documentation below.
</p>
</div>
<SwaggerUI spec={swaggerConfig} />
</div>
);
}
```
## Best Practices
1. **Route Groups**: Use route groups `(groupname)` to organize related pages without affecting the URL structure.
2. **API Documentation**:
- Add detailed descriptions for all endpoints
- Include parameter descriptions and constraints
- Define response schemas
- Document error responses
- Use appropriate data formats (UUID, URI, etc.)
- Group related endpoints using tags
3. **Swagger Configuration**:
- Add contact information
- Include license details
- Set appropriate servers configuration
- Define required fields
- Add parameter validations (min/max values)
## Common Issues
1. **Route Conflicts**: Avoid parallel routes that resolve to the same path. For example, don't have both `app/swagger/page.tsx` and `app/(group)/swagger/page.tsx` as they would conflict.
2. **CSS Loading**: Make sure to:
- Import Swagger UI CSS
- Configure webpack in `next.config.js`
- Use the `"use client"` directive as Swagger UI is a client-side component
3. **React Version Compatibility**: Be aware of potential peer dependency warnings between Swagger UI React and your React version. You might need to use `--legacy-peer-deps` or adjust your React version accordingly.
## Accessing the Documentation
After setup, your Swagger documentation will be available at `/swagger` in your application. The UI provides:
- Interactive API documentation
- Request/response examples
- Try-it-out functionality
- Schema definitions
- Error responses
## Maintenance
Keep your Swagger documentation up-to-date by:
- Updating the OpenAPI specification when adding or modifying endpoints
- Maintaining accurate parameter descriptions
- Keeping example values relevant
- Updating response schemas when data structures change

File diff suppressed because it is too large Load Diff

View File

@@ -1,39 +1,109 @@
import { createClient } from '@clickhouse/client';
import type { EventsQueryParams } from './types';
// Create configuration object using the URL approach
const config = {
url: process.env.CLICKHOUSE_URL || 'http://localhost:8123',
username: process.env.CLICKHOUSE_USER || 'default',
password: process.env.CLICKHOUSE_PASSWORD || '',
database: process.env.CLICKHOUSE_DATABASE || 'limq'
};
// ClickHouse 客户端配置
const clickhouse = createClient({
url: process.env.CLICKHOUSE_URL,
username: process.env.CLICKHOUSE_USER ,
password: process.env.CLICKHOUSE_PASSWORD ,
database: process.env.CLICKHOUSE_DATABASE
});
// Create ClickHouse client with proper URL format
export const clickhouse = createClient(config);
// 构建日期过滤条件
function buildDateFilter(startTime?: string, endTime?: string): string {
const filters = [];
if (startTime) {
filters.push(`event_time >= parseDateTimeBestEffort('${startTime}')`);
}
if (endTime) {
filters.push(`event_time <= parseDateTimeBestEffort('${endTime}')`);
}
return filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
}
// 构建通用过滤条件
export function buildFilter(params: Partial<EventsQueryParams>): string {
const filters = [];
// 时间范围过滤
if (params.startTime || params.endTime) {
const dateFilter = buildDateFilter(params.startTime, params.endTime).replace('WHERE ', '');
if (dateFilter) {
filters.push(dateFilter);
}
}
// 事件类型过滤
if (params.eventType) {
filters.push(`event_type = '${params.eventType}'`);
}
// 链接ID过滤
if (params.linkId) {
filters.push(`link_id = '${params.linkId}'`);
}
// 链接短码过滤
if (params.linkSlug) {
filters.push(`link_slug = '${params.linkSlug}'`);
}
// 用户ID过滤
if (params.userId) {
filters.push(`user_id = '${params.userId}'`);
}
// 团队ID过滤
if (params.teamId) {
filters.push(`team_id = '${params.teamId}'`);
}
// 项目ID过滤
if (params.projectId) {
filters.push(`project_id = '${params.projectId}'`);
}
return filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
}
/**
* Execute ClickHouse query and return results
*/
// 构建分页
export function buildPagination(page?: number, pageSize?: number): string {
const limit = pageSize || 20;
const offset = ((page || 1) - 1) * limit;
return `LIMIT ${limit} OFFSET ${offset}`;
}
// 构建排序
export function buildOrderBy(sortBy?: string, sortOrder?: 'asc' | 'desc'): string {
if (!sortBy) {
return 'ORDER BY event_time DESC';
}
return `ORDER BY ${sortBy} ${sortOrder || 'desc'}`;
}
// 执行查询并处理错误
export async function executeQuery<T>(query: string): Promise<T[]> {
try {
const result = await clickhouse.query({
const resultSet = await clickhouse.query({
query,
format: 'JSONEachRow',
format: 'JSONEachRow'
});
const data = await result.json();
return data as T[];
const rows = await resultSet.json<T>();
return Array.isArray(rows) ? rows : [rows];
} catch (error) {
console.error('ClickHouse query error:', error);
throw error;
}
}
/**
* Execute ClickHouse query and return a single result
*/
// 执行查询并返回单个结果
export async function executeQuerySingle<T>(query: string): Promise<T | null> {
const results = await executeQuery<T>(query);
return results.length > 0 ? results[0] : null;
}
}
export default clickhouse;

171
lib/types.ts Normal file
View File

@@ -0,0 +1,171 @@
// 事件类型
export enum EventType {
CLICK = 'click',
REDIRECT = 'redirect',
CONVERSION = 'conversion',
ERROR = 'error'
}
// 转化类型
export enum ConversionType {
VISIT = 'visit',
STAY = 'stay',
INTERACT = 'interact',
SIGNUP = 'signup',
SUBSCRIPTION = 'subscription',
PURCHASE = 'purchase'
}
// 设备类型
export enum DeviceType {
MOBILE = 'mobile',
TABLET = 'tablet',
DESKTOP = 'desktop',
OTHER = 'other'
}
// API 响应基础接口
export interface ApiResponse<T> {
success: boolean;
data?: T;
error?: string;
meta?: {
total?: number;
page?: number;
pageSize?: number;
};
}
// 事件查询参数
export interface EventsQueryParams {
startTime?: string; // ISO 格式时间
endTime?: string; // ISO 格式时间
eventType?: EventType;
linkId?: string;
linkSlug?: string;
userId?: string;
teamId?: string;
projectId?: string;
page?: number;
pageSize?: number;
sortBy?: string;
sortOrder?: 'asc' | 'desc';
}
// 事件基础信息
export interface Event {
event_id: string;
event_time: string;
event_type: EventType;
event_attributes: Record<string, any>;
// 链接信息
link_id: string;
link_slug: string;
link_label: string;
link_title: string;
link_original_url: string;
link_attributes: Record<string, any>;
link_created_at: string;
link_expires_at: string | null;
link_tags: string[];
// 用户信息
user_id: string;
user_name: string;
user_email: string;
user_attributes: Record<string, any>;
// 团队信息
team_id: string;
team_name: string;
team_attributes: Record<string, any>;
// 项目信息
project_id: string;
project_name: string;
project_attributes: Record<string, any>;
// 访问者信息
visitor_id: string;
session_id: string;
ip_address: string;
country: string;
city: string;
device_type: DeviceType;
browser: string;
os: string;
user_agent: string;
// 来源信息
referrer: string;
utm_source: string;
utm_medium: string;
utm_campaign: string;
// 交互信息
time_spent_sec: number;
is_bounce: boolean;
is_qr_scan: boolean;
conversion_type: ConversionType;
conversion_value: number;
}
// 事件概览数据
export interface EventsSummary {
totalEvents: number;
uniqueVisitors: number;
totalConversions: number;
averageTimeSpent: number;
deviceTypes: {
mobile: number;
desktop: number;
tablet: number;
other: number;
};
browsers: Array<{
name: string;
count: number;
percentage: number;
}>;
operatingSystems: Array<{
name: string;
count: number;
percentage: number;
}>;
}
// 时间序列数据
export interface TimeSeriesData {
timestamp: string;
events: number;
visitors: number;
conversions: number;
}
// 地理位置数据
export interface GeoData {
location: string;
visits: number;
visitors: number;
percentage: number;
}
// 设备分析数据
export interface DeviceAnalytics {
deviceTypes: Array<{
type: DeviceType;
count: number;
percentage: number;
}>;
browsers: Array<{
name: string;
count: number;
percentage: number;
}>;
operatingSystems: Array<{
name: string;
count: number;
percentage: number;
}>;
}

13
next.config.js Normal file
View File

@@ -0,0 +1,13 @@
/** @type {import('next').NextConfig} */
const nextConfig = {
transpilePackages: ['swagger-ui-react'],
webpack: (config) => {
config.module.rules.push({
test: /\.css$/,
use: ['style-loader', 'css-loader'],
});
return config;
},
};
module.exports = nextConfig;

454
package-lock.json generated
View File

@@ -9,9 +9,16 @@
"version": "0.1.0",
"dependencies": {
"@clickhouse/client": "^1.11.0",
"@types/chart.js": "^2.9.41",
"@types/recharts": "^1.8.29",
"@types/uuid": "^10.0.0",
"chart.js": "^4.4.8",
"date-fns": "^4.1.0",
"next": "15.2.3",
"react": "^19.0.0",
"react-dom": "^19.0.0"
"react-dom": "^19.0.0",
"recharts": "^2.15.1",
"uuid": "^10.0.0"
},
"devDependencies": {
"@eslint/eslintrc": "^3",
@@ -38,6 +45,18 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@babel/runtime": {
"version": "7.27.0",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.0.tgz",
"integrity": "sha512-VtPOkrdPHZsKc/clNqyi9WUA8TINkZ4cGk63UUE3u4pmB2k+ZMQRDuIOagv8UVd6j7k0T3+RRIb7beKTebNbcw==",
"license": "MIT",
"dependencies": {
"regenerator-runtime": "^0.14.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@clickhouse/client": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.11.0.tgz",
@@ -654,6 +673,12 @@
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@kurkle/color": {
"version": "0.3.4",
"resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz",
"integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==",
"license": "MIT"
},
"node_modules/@napi-rs/wasm-runtime": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.7.tgz",
@@ -1136,6 +1161,78 @@
"tslib": "^2.4.0"
}
},
"node_modules/@types/chart.js": {
"version": "2.9.41",
"resolved": "https://registry.npmjs.org/@types/chart.js/-/chart.js-2.9.41.tgz",
"integrity": "sha512-3dvkDvueckY83UyUXtJMalYoH6faOLkWQoaTlJgB4Djde3oORmNP0Jw85HtzTuXyliUHcdp704s0mZFQKio/KQ==",
"license": "MIT",
"dependencies": {
"moment": "^2.10.2"
}
},
"node_modules/@types/d3-array": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz",
"integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==",
"license": "MIT"
},
"node_modules/@types/d3-color": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz",
"integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==",
"license": "MIT"
},
"node_modules/@types/d3-ease": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz",
"integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==",
"license": "MIT"
},
"node_modules/@types/d3-interpolate": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz",
"integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==",
"license": "MIT",
"dependencies": {
"@types/d3-color": "*"
}
},
"node_modules/@types/d3-path": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-1.0.11.tgz",
"integrity": "sha512-4pQMp8ldf7UaB/gR8Fvvy69psNHkTpD/pVw3vmEi8iZAB9EPMBruB1JvHO4BIq9QkUUd2lV1F5YXpMNj7JPBpw==",
"license": "MIT"
},
"node_modules/@types/d3-scale": {
"version": "4.0.9",
"resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz",
"integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==",
"license": "MIT",
"dependencies": {
"@types/d3-time": "*"
}
},
"node_modules/@types/d3-shape": {
"version": "1.3.12",
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-1.3.12.tgz",
"integrity": "sha512-8oMzcd4+poSLGgV0R1Q1rOlx/xdmozS4Xab7np0eamFFUYq71AU9pOCJEFnkXW2aI/oXdVYJzw6pssbSut7Z9Q==",
"license": "MIT",
"dependencies": {
"@types/d3-path": "^1"
}
},
"node_modules/@types/d3-time": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz",
"integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==",
"license": "MIT"
},
"node_modules/@types/d3-timer": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz",
"integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==",
"license": "MIT"
},
"node_modules/@types/estree": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@@ -1171,7 +1268,6 @@
"version": "19.0.12",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.12.tgz",
"integrity": "sha512-V6Ar115dBDrjbtXSrS+/Oruobc+qVbbUxDFC1RSbRqLt5SYvxxyIDrSC85RWml54g+jfNeEMZhEj7wW07ONQhA==",
"dev": true,
"license": "MIT",
"dependencies": {
"csstype": "^3.0.2"
@@ -1187,6 +1283,22 @@
"@types/react": "^19.0.0"
}
},
"node_modules/@types/recharts": {
"version": "1.8.29",
"resolved": "https://registry.npmjs.org/@types/recharts/-/recharts-1.8.29.tgz",
"integrity": "sha512-ulKklaVsnFIIhTQsQw226TnOibrddW1qUQNFVhoQEyY1Z7FRQrNecFCGt7msRuJseudzE9czVawZb17dK/aPXw==",
"license": "MIT",
"dependencies": {
"@types/d3-shape": "^1",
"@types/react": "*"
}
},
"node_modules/@types/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
"license": "MIT"
},
"node_modules/@typescript-eslint/eslint-plugin": {
"version": "8.27.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.27.0.tgz",
@@ -2003,12 +2115,33 @@
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/chart.js": {
"version": "4.4.8",
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.8.tgz",
"integrity": "sha512-IkGZlVpXP+83QpMm4uxEiGqSI7jFizwVtF3+n5Pc3k7sMO+tkd0qxh2OzLhenM0K80xtmAONWGBn082EiBQSDA==",
"license": "MIT",
"dependencies": {
"@kurkle/color": "^0.3.0"
},
"engines": {
"pnpm": ">=8"
}
},
"node_modules/client-only": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
"integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==",
"license": "MIT"
},
"node_modules/clsx": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/color": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
@@ -2080,9 +2213,129 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
"dev": true,
"license": "MIT"
},
"node_modules/d3-array": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
"integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
"license": "ISC",
"dependencies": {
"internmap": "1 - 2"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-color": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
"integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-ease": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz",
"integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-format": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz",
"integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-interpolate": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
"integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
"license": "ISC",
"dependencies": {
"d3-color": "1 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
"integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/d3-scale": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
"integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
"license": "ISC",
"dependencies": {
"d3-array": "2.10.0 - 3",
"d3-format": "1 - 3",
"d3-interpolate": "1.2.0 - 3",
"d3-time": "2.1.1 - 3",
"d3-time-format": "2 - 4"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-shape": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
"integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
"license": "ISC",
"dependencies": {
"d3-path": "^3.1.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-time": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz",
"integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==",
"license": "ISC",
"dependencies": {
"d3-array": "2 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-time-format": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz",
"integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==",
"license": "ISC",
"dependencies": {
"d3-time": "1 - 3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/d3-timer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz",
"integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/damerau-levenshtein": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",
@@ -2144,6 +2397,16 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/date-fns": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz",
"integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/kossnocorp"
}
},
"node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
@@ -2162,6 +2425,12 @@
}
}
},
"node_modules/decimal.js-light": {
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz",
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==",
"license": "MIT"
},
"node_modules/deep-is": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
@@ -2228,6 +2497,16 @@
"node": ">=0.10.0"
}
},
"node_modules/dom-helpers": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz",
"integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.8.7",
"csstype": "^3.0.2"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -2877,6 +3156,12 @@
"node": ">=0.10.0"
}
},
"node_modules/eventemitter3": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
"integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
"license": "MIT"
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -2884,6 +3169,15 @@
"dev": true,
"license": "MIT"
},
"node_modules/fast-equals": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.2.2.tgz",
"integrity": "sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==",
"license": "MIT",
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/fast-glob": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz",
@@ -3345,6 +3639,15 @@
"node": ">= 0.4"
}
},
"node_modules/internmap": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
"integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==",
"license": "ISC",
"engines": {
"node": ">=12"
}
},
"node_modules/is-array-buffer": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz",
@@ -3786,7 +4089,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true,
"license": "MIT"
},
"node_modules/js-yaml": {
@@ -4151,6 +4453,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
"license": "MIT"
},
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -4162,7 +4470,6 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
"integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"js-tokens": "^3.0.0 || ^4.0.0"
@@ -4228,6 +4535,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/moment": {
"version": "2.30.1",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
"integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
"license": "MIT",
"engines": {
"node": "*"
}
},
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -4346,7 +4662,6 @@
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
@@ -4645,7 +4960,6 @@
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
"dev": true,
"license": "MIT",
"dependencies": {
"loose-envify": "^1.4.0",
@@ -4709,7 +5023,75 @@
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
"dev": true,
"license": "MIT"
},
"node_modules/react-smooth": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz",
"integrity": "sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==",
"license": "MIT",
"dependencies": {
"fast-equals": "^5.0.1",
"prop-types": "^15.8.1",
"react-transition-group": "^4.4.5"
},
"peerDependencies": {
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/react-transition-group": {
"version": "4.4.5",
"resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
"integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==",
"license": "BSD-3-Clause",
"dependencies": {
"@babel/runtime": "^7.5.5",
"dom-helpers": "^5.0.1",
"loose-envify": "^1.4.0",
"prop-types": "^15.6.2"
},
"peerDependencies": {
"react": ">=16.6.0",
"react-dom": ">=16.6.0"
}
},
"node_modules/recharts": {
"version": "2.15.1",
"resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.1.tgz",
"integrity": "sha512-v8PUTUlyiDe56qUj82w/EDVuzEFXwEHp9/xOowGAZwfLjB9uAy3GllQVIYMWF6nU+qibx85WF75zD7AjqoT54Q==",
"license": "MIT",
"dependencies": {
"clsx": "^2.0.0",
"eventemitter3": "^4.0.1",
"lodash": "^4.17.21",
"react-is": "^18.3.1",
"react-smooth": "^4.0.4",
"recharts-scale": "^0.4.4",
"tiny-invariant": "^1.3.1",
"victory-vendor": "^36.6.8"
},
"engines": {
"node": ">=14"
},
"peerDependencies": {
"react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/recharts-scale": {
"version": "0.4.5",
"resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz",
"integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==",
"license": "MIT",
"dependencies": {
"decimal.js-light": "^2.4.1"
}
},
"node_modules/recharts/node_modules/react-is": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
"integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
"license": "MIT"
},
"node_modules/reflect.getprototypeof": {
@@ -4735,6 +5117,12 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/regenerator-runtime": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
"integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==",
"license": "MIT"
},
"node_modules/regexp.prototype.flags": {
"version": "1.5.4",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz",
@@ -5353,6 +5741,12 @@
"node": ">=6"
}
},
"node_modules/tiny-invariant": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
"integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==",
"license": "MIT"
},
"node_modules/tinyglobby": {
"version": "0.2.12",
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.12.tgz",
@@ -5584,6 +5978,50 @@
"punycode": "^2.1.0"
}
},
"node_modules/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
"integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/victory-vendor": {
"version": "36.9.2",
"resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz",
"integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==",
"license": "MIT AND ISC",
"dependencies": {
"@types/d3-array": "^3.0.3",
"@types/d3-ease": "^3.0.0",
"@types/d3-interpolate": "^3.0.1",
"@types/d3-scale": "^4.0.2",
"@types/d3-shape": "^3.1.0",
"@types/d3-time": "^3.0.0",
"@types/d3-timer": "^3.0.0",
"d3-array": "^3.1.6",
"d3-ease": "^3.0.1",
"d3-interpolate": "^3.0.1",
"d3-scale": "^4.0.2",
"d3-shape": "^3.1.0",
"d3-time": "^3.0.0",
"d3-timer": "^3.0.1"
}
},
"node_modules/victory-vendor/node_modules/@types/d3-shape": {
"version": "3.1.7",
"resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.7.tgz",
"integrity": "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg==",
"license": "MIT",
"dependencies": {
"@types/d3-path": "*"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

View File

@@ -24,10 +24,17 @@
},
"dependencies": {
"@clickhouse/client": "^1.11.0",
"@types/chart.js": "^2.9.41",
"@types/recharts": "^1.8.29",
"@types/uuid": "^10.0.0",
"chart.js": "^4.4.8",
"date-fns": "^4.1.0",
"next": "15.2.3",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"recharts": "^2.15.1",
"swagger-ui-dist": "^5.12.0",
"swagger-ui-react": "^5.12.0",
"uuid": "^10.0.0"
},
"devDependencies": {
@@ -36,8 +43,11 @@
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",
"@types/swagger-ui-react": "^4.18.3",
"css-loader": "^7.1.2",
"eslint": "^9",
"eslint-config-next": "15.2.3",
"style-loader": "^4.0.0",
"tailwindcss": "^4",
"typescript": "^5"
},

2263
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,76 @@
-- 创建数据库(如果不存在)
CREATE DATABASE IF NOT EXISTS shorturl_analytics;
-- 切换到shorturl_analytics数据库
USE shorturl_analytics;
-- 删除已存在的表
DROP TABLE IF EXISTS shorturl_analytics.events;
-- 创建新表
CREATE TABLE IF NOT EXISTS shorturl_analytics.events (
-- 事件基础信息
event_id String,
event_time DateTime64(3),
-- 精确到毫秒的时间戳
event_type String,
-- click, redirect, conversion, error
event_attributes String DEFAULT '{}',
-- 链接基本信息
link_id String,
link_slug String,
-- 新增slug
link_label String,
-- 新增label
link_title String,
link_original_url String,
link_attributes String DEFAULT '{}',
link_created_at DateTime64(3),
-- 精确到毫秒的时间戳
link_expires_at Nullable(DateTime64(3)),
-- 精确到毫秒的时间戳
link_tags String DEFAULT '[]',
-- Array of {id, name, attributes}
-- 用户信息
user_id String,
user_name String,
user_email String,
user_attributes String DEFAULT '{}',
-- 团队信息
team_id String,
team_name String,
team_attributes String DEFAULT '{}',
-- 项目信息
project_id String,
project_name String,
project_attributes String DEFAULT '{}',
-- QR码信息
qr_code_id String,
qr_code_name String,
qr_code_attributes String DEFAULT '{}',
-- 访问者信息
visitor_id String,
session_id String,
ip_address String,
country String,
city String,
device_type String,
-- 改为String类型
browser String,
os String,
user_agent String,
-- 来源信息
referrer String,
utm_source String,
utm_medium String,
utm_campaign String,
-- 交互信息
time_spent_sec UInt32 DEFAULT 0,
is_bounce Boolean DEFAULT true,
is_qr_scan Boolean DEFAULT false,
conversion_type String,
-- 改为String类型
conversion_value Float64 DEFAULT 0
) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) -- 直接使用DateTime64进行分区
ORDER BY
(event_time, link_id, event_id) SETTINGS index_granularity = 8192;

View File

@@ -1,146 +0,0 @@
-- 添加team、project和qrcode表到limq数据库
USE limq;
-- 团队表
CREATE TABLE IF NOT EXISTS limq.teams (
team_id String,
name String,
created_at DateTime,
created_by String,
description String DEFAULT '',
avatar_url String DEFAULT '',
is_active Boolean DEFAULT true,
plan_type Enum8(
'free' = 1,
'pro' = 2,
'enterprise' = 3
),
members_count UInt32 DEFAULT 1,
PRIMARY KEY (team_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
team_id SETTINGS index_granularity = 8192;
-- 项目表
CREATE TABLE IF NOT EXISTS limq.projects (
project_id String,
team_id String,
name String,
created_at DateTime,
created_by String,
description String DEFAULT '',
is_archived Boolean DEFAULT false,
links_count UInt32 DEFAULT 0,
total_clicks UInt64 DEFAULT 0,
last_updated DateTime DEFAULT now(),
PRIMARY KEY (project_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(project_id, team_id) SETTINGS index_granularity = 8192;
-- QR码表 (扩展现有的qr_scans表)
CREATE TABLE IF NOT EXISTS limq.qrcodes (
qr_code_id String,
link_id String,
team_id String,
project_id String DEFAULT '',
name String,
description String DEFAULT '',
created_at DateTime,
created_by String,
updated_at DateTime DEFAULT now(),
qr_type Enum8(
'standard' = 1,
'custom' = 2,
'dynamic' = 3
) DEFAULT 'standard',
image_url String DEFAULT '',
design_config String DEFAULT '{}',
is_active Boolean DEFAULT true,
total_scans UInt64 DEFAULT 0,
unique_scanners UInt32 DEFAULT 0,
PRIMARY KEY (qr_code_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(qr_code_id, link_id) SETTINGS index_granularity = 8192;
-- 团队成员表
CREATE TABLE IF NOT EXISTS limq.team_members (
team_id String,
user_id String,
role Enum8(
'owner' = 1,
'admin' = 2,
'editor' = 3,
'viewer' = 4
),
joined_at DateTime DEFAULT now(),
invited_by String,
is_active Boolean DEFAULT true,
last_active DateTime DEFAULT now(),
PRIMARY KEY (team_id, user_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(team_id, user_id) SETTINGS index_granularity = 8192;
-- 团队每日统计视图
CREATE MATERIALIZED VIEW limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, team_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.team_id AS team_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.team_id != ''
GROUP BY
date,
l.team_id;
-- 项目每日统计视图
CREATE MATERIALIZED VIEW limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, project_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.project_id AS project_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.project_id != ''
GROUP BY
date,
l.project_id;
-- QR码每日统计视图
CREATE MATERIALIZED VIEW limq.qrcode_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, qr_code_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(scan_time) AS date,
qr_code_id,
count() AS total_scans,
uniqExact(visitor_id) AS unique_scanners,
countIf(led_to_conversion) AS conversions,
countIf(device_type = 'mobile') AS mobile_scans,
countIf(device_type = 'tablet') AS tablet_scans,
countIf(device_type = 'desktop') AS desktop_scans,
uniqExact(location) AS unique_locations
FROM
limq.qr_scans
GROUP BY
date,
qr_code_id;

View File

@@ -1,29 +0,0 @@
#!/bin/bash
# 脚本名称: load-clickhouse-testdata.sh
# 用途: 将测试数据加载到ClickHouse数据库中
# 设置脚本目录路径
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# 设置SQL文件路径
SQL_FILE="$SCRIPT_DIR/sql/clickhouse/seed-clickhouse-analytics.sql"
# 检查SQL文件是否存在
if [ ! -f "$SQL_FILE" ]; then
echo "错误: SQL文件 '$SQL_FILE' 不存在"
exit 1
fi
# 执行CH查询脚本
echo "开始加载测试数据到ClickHouse数据库..."
bash "$SCRIPT_DIR/sql/clickhouse/ch-query.sh" -f "$SQL_FILE"
# 检查执行结果
if [ $? -eq 0 ]; then
echo "测试数据已成功加载到ClickHouse数据库"
else
echo "错误: 加载测试数据失败"
exit 1
fi
exit 0

View File

@@ -1,997 +0,0 @@
-- 移动端点击访问事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 10:25:30',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-123',
's-456',
'click',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
45,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 11:32:21',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-124',
's-457',
'click',
'43.78.123.45',
'Japan',
'Tokyo',
'https://twitter.com',
'twitter',
'social',
'spring_promo',
'Mozilla/5.0 (Android 10)',
'mobile',
'Chrome',
'Android',
15,
true,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 14:15:45',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-125',
's-458',
'click',
'72.34.67.81',
'US',
'New York',
'https://www.facebook.com',
'facebook',
'social',
'crypto_ad',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
120,
false,
false,
'interact',
0
);
-- 桌面设备点击事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 08:45:12',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-126',
's-459',
'click',
'89.67.43.21',
'Germany',
'Berlin',
'https://www.reddit.com',
'reddit',
'referral',
'none',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Chrome',
'Windows',
300,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 16:20:33',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-127',
's-460',
'click',
'178.65.43.12',
'UK',
'London',
'https://www.linkedin.com',
'linkedin',
'social',
'biz_campaign',
'Mozilla/5.0 (Macintosh)',
'desktop',
'Safari',
'MacOS',
250,
false,
false,
'stay',
0
);
-- 平板设备点击事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 13:10:55',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-128',
's-461',
'click',
'156.78.34.12',
'Canada',
'Toronto',
'https://www.youtube.com',
'youtube',
'video',
'tutorial',
'Mozilla/5.0 (iPad)',
'tablet',
'Safari',
'iOS',
180,
false,
false,
'visit',
0
);
-- QR扫描访问事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 09:30:22',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_qr',
'v-129',
's-462',
'click',
'101.56.78.90',
'China',
'Beijing',
'direct',
'qr',
'print',
'offline_event',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
75,
false,
true,
'visit',
0
);
-- 转化事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 10:27:45',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-123',
's-456',
'conversion',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
120,
false,
false,
'signup',
50
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 08:52:18',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-126',
's-459',
'conversion',
'89.67.43.21',
'Germany',
'Berlin',
'https://www.reddit.com',
'reddit',
'referral',
'none',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Chrome',
'Windows',
450,
false,
false,
'purchase',
150.75
);
-- 第二天的数据 (3/16)
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-16 11:15:30',
'2025-03-16',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-130',
's-463',
'click',
'178.91.45.67',
'France',
'Paris',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (Android 11)',
'mobile',
'Chrome',
'Android',
60,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-16 14:22:45',
'2025-03-16',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-131',
's-464',
'click',
'89.123.45.78',
'Spain',
'Madrid',
'https://www.instagram.com',
'instagram',
'social',
'influencer',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
90,
false,
false,
'interact',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-16 16:40:12',
'2025-03-16',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-131',
's-464',
'conversion',
'89.123.45.78',
'Spain',
'Madrid',
'https://www.instagram.com',
'instagram',
'social',
'influencer',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
200,
false,
false,
'subscription',
75.50
);
-- 第三天数据 (3/17)
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-17 09:10:22',
'2025-03-17',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-132',
's-465',
'click',
'45.67.89.123',
'US',
'Los Angeles',
'https://www.google.com',
'google',
'cpc',
'spring_sale',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Edge',
'Windows',
150,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-17 12:30:45',
'2025-03-17',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-133',
's-466',
'click',
'67.89.123.45',
'Brazil',
'Sao Paulo',
'https://www.yahoo.com',
'yahoo',
'organic',
'none',
'Mozilla/5.0 (iPad)',
'tablet',
'Safari',
'iOS',
120,
false,
false,
'stay',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-17 15:45:33',
'2025-03-17',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-132',
's-465',
'conversion',
'45.67.89.123',
'US',
'Los Angeles',
'https://www.google.com',
'google',
'cpc',
'spring_sale',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Edge',
'Windows',
300,
false,
false,
'purchase',
225.50
);
-- 添加一周前的数据 (对比期)
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-08 10:25:30',
'2025-03-08',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-140',
's-470',
'click',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
30,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-08 11:32:21',
'2025-03-08',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-141',
's-471',
'click',
'89.67.43.21',
'Germany',
'Berlin',
'https://www.reddit.com',
'reddit',
'referral',
'none',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Chrome',
'Windows',
200,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-08 13:10:55',
'2025-03-08',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-140',
's-470',
'conversion',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
100,
false,
false,
'purchase',
100.00
);

View File

@@ -1,122 +0,0 @@
-- 修改设备类型字段从枚举类型更改为字符串类型
-- 先删除依赖于link_events表的物化视图
DROP TABLE IF EXISTS limq.platform_distribution;
DROP TABLE IF EXISTS limq.link_hourly_patterns;
DROP TABLE IF EXISTS limq.link_daily_stats;
DROP TABLE IF EXISTS limq.team_daily_stats;
DROP TABLE IF EXISTS limq.project_daily_stats;
-- 修改link_events表的device_type字段
ALTER TABLE
limq.link_events
MODIFY
COLUMN device_type String;
-- 重新创建物化视图
-- 每日链接汇总视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
link_id,
count() AS total_clicks,
uniqExact(visitor_id) AS unique_visitors,
uniqExact(session_id) AS unique_sessions,
sum(time_spent_sec) AS total_time_spent,
avg(time_spent_sec) AS avg_time_spent,
countIf(is_bounce) AS bounce_count,
countIf(event_type = 'conversion') AS conversion_count,
uniqExact(referrer) AS unique_referrers,
countIf(device_type = 'mobile') AS mobile_count,
countIf(device_type = 'tablet') AS tablet_count,
countIf(device_type = 'desktop') AS desktop_count,
countIf(is_qr_scan) AS qr_scan_count,
sum(conversion_value) AS total_conversion_value
FROM
limq.link_events
GROUP BY
date,
link_id;
-- 每小时访问模式视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_hourly_patterns ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, hour, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
toHour(event_time) AS hour,
link_id,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
GROUP BY
date,
hour,
link_id;
-- 平台分布视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.platform_distribution ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, utm_source, device_type) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
utm_source,
device_type,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
WHERE
utm_source != ''
GROUP BY
date,
utm_source,
device_type;
-- 团队每日统计视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, team_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.team_id AS team_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.team_id != ''
GROUP BY
date,
l.team_id;
-- 项目每日统计视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, project_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.project_id AS project_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.project_id != ''
GROUP BY
date,
l.project_id;

View File

@@ -1,379 +0,0 @@
-- 删除所有物化视图(需要先删除视图,因为它们依赖于表)
DROP TABLE IF EXISTS limq.platform_distribution;
DROP TABLE IF EXISTS limq.link_hourly_patterns;
DROP TABLE IF EXISTS limq.link_daily_stats;
DROP TABLE IF EXISTS limq.team_daily_stats;
DROP TABLE IF EXISTS limq.project_daily_stats;
DROP TABLE IF EXISTS limq.qrcode_daily_stats;
-- 删除所有表
DROP TABLE IF EXISTS limq.qr_scans;
DROP TABLE IF EXISTS limq.sessions;
DROP TABLE IF EXISTS limq.link_events;
DROP TABLE IF EXISTS limq.links;
DROP TABLE IF EXISTS limq.teams;
DROP TABLE IF EXISTS limq.projects;
DROP TABLE IF EXISTS limq.qrcodes;
DROP TABLE IF EXISTS limq.team_members;
DROP TABLE IF EXISTS limq.users;
-- 创建数据库(如果不存在)
CREATE DATABASE IF NOT EXISTS limq;
-- 切换到limq数据库
USE limq;
-- 创建短链接访问事件表
CREATE TABLE IF NOT EXISTS limq.link_events (
event_id UUID DEFAULT generateUUIDv4(),
event_time DateTime64(3) DEFAULT now64(),
date Date DEFAULT toDate(event_time),
link_id String,
channel_id String,
visitor_id String,
session_id String,
event_type Enum8(
'click' = 1,
'redirect' = 2,
'conversion' = 3,
'error' = 4
),
-- 访问者信息
ip_address String,
country String,
city String,
-- 来源信息
referrer String,
utm_source String,
utm_medium String,
utm_campaign String,
-- 设备信息
user_agent String,
device_type Enum8(
'mobile' = 1,
'tablet' = 2,
'desktop' = 3,
'other' = 4
),
browser String,
os String,
-- 交互信息
time_spent_sec UInt32 DEFAULT 0,
is_bounce Boolean DEFAULT true,
-- QR码相关
is_qr_scan Boolean DEFAULT false,
qr_code_id String DEFAULT '',
-- 转化数据
conversion_type Enum8(
'visit' = 1,
'stay' = 2,
'interact' = 3,
'signup' = 4,
'subscription' = 5,
'purchase' = 6
) DEFAULT 'visit',
conversion_value Float64 DEFAULT 0,
-- 其他属性
custom_data String DEFAULT '{}'
) ENGINE = MergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, link_id, event_time) SETTINGS index_granularity = 8192;
-- 短链接维度表
CREATE TABLE IF NOT EXISTS limq.links (
link_id String,
original_url String,
created_at DateTime64(3),
created_by String,
title String,
description String,
tags Array(String),
is_active Boolean DEFAULT true,
expires_at Nullable(DateTime64(3)),
team_id String DEFAULT '',
project_id String DEFAULT '',
PRIMARY KEY (link_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
link_id SETTINGS index_granularity = 8192;
-- 会话跟踪表
CREATE TABLE IF NOT EXISTS limq.sessions (
session_id String,
visitor_id String,
link_id String,
started_at DateTime64(3),
last_activity DateTime64(3),
ended_at Nullable(DateTime64(3)),
duration_sec UInt32 DEFAULT 0,
session_pages UInt8 DEFAULT 1,
is_completed Boolean DEFAULT false,
PRIMARY KEY (session_id)
) ENGINE = ReplacingMergeTree(last_activity)
ORDER BY
(session_id, link_id, visitor_id) SETTINGS index_granularity = 8192;
-- QR码统计表
CREATE TABLE IF NOT EXISTS limq.qr_scans (
scan_id UUID DEFAULT generateUUIDv4(),
qr_code_id String,
link_id String,
scan_time DateTime64(3),
visitor_id String,
location String,
device_type Enum8(
'mobile' = 1,
'tablet' = 2,
'desktop' = 3,
'other' = 4
),
led_to_conversion Boolean DEFAULT false,
PRIMARY KEY (scan_id)
) ENGINE = MergeTree() PARTITION BY toYYYYMM(scan_time)
ORDER BY
scan_id SETTINGS index_granularity = 8192;
-- 团队表
CREATE TABLE IF NOT EXISTS limq.teams (
team_id String,
name String,
created_at DateTime,
created_by String,
description String DEFAULT '',
avatar_url String DEFAULT '',
is_active Boolean DEFAULT true,
plan_type Enum8(
'free' = 1,
'pro' = 2,
'enterprise' = 3
),
members_count UInt32 DEFAULT 1,
PRIMARY KEY (team_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
team_id SETTINGS index_granularity = 8192;
-- 项目表
CREATE TABLE IF NOT EXISTS limq.projects (
project_id String,
team_id String,
name String,
created_at DateTime,
created_by String,
description String DEFAULT '',
is_archived Boolean DEFAULT false,
links_count UInt32 DEFAULT 0,
total_clicks UInt64 DEFAULT 0,
last_updated DateTime DEFAULT now(),
PRIMARY KEY (project_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(project_id, team_id) SETTINGS index_granularity = 8192;
-- QR码表
CREATE TABLE IF NOT EXISTS limq.qrcodes (
qr_code_id String,
link_id String,
team_id String,
project_id String DEFAULT '',
name String,
description String DEFAULT '',
created_at DateTime,
created_by String,
updated_at DateTime DEFAULT now(),
qr_type Enum8(
'standard' = 1,
'custom' = 2,
'dynamic' = 3
) DEFAULT 'standard',
image_url String DEFAULT '',
design_config String DEFAULT '{}',
is_active Boolean DEFAULT true,
total_scans UInt64 DEFAULT 0,
unique_scanners UInt32 DEFAULT 0,
PRIMARY KEY (qr_code_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(qr_code_id, link_id) SETTINGS index_granularity = 8192;
-- 团队成员表
CREATE TABLE IF NOT EXISTS limq.team_members (
team_id String,
user_id String,
role Enum8(
'owner' = 1,
'admin' = 2,
'editor' = 3,
'viewer' = 4
),
joined_at DateTime DEFAULT now(),
invited_by String,
is_active Boolean DEFAULT true,
last_active DateTime DEFAULT now(),
PRIMARY KEY (team_id, user_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(team_id, user_id) SETTINGS index_granularity = 8192;
-- 用户表
CREATE TABLE IF NOT EXISTS limq.users (
user_id String,
username String,
email String,
full_name String,
avatar_url String DEFAULT '',
created_at DateTime,
last_login DateTime DEFAULT now(),
is_active Boolean DEFAULT true,
is_verified Boolean DEFAULT false,
auth_provider Enum8(
'email' = 1,
'google' = 2,
'github' = 3,
'microsoft' = 4
) DEFAULT 'email',
roles Array(String) DEFAULT [ 'user' ],
preferences String DEFAULT '{}',
teams_count UInt32 DEFAULT 0,
links_created UInt32 DEFAULT 0,
PRIMARY KEY (user_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
user_id SETTINGS index_granularity = 8192;
-- 每日链接汇总视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
link_id,
count() AS total_clicks,
uniqExact(visitor_id) AS unique_visitors,
uniqExact(session_id) AS unique_sessions,
sum(time_spent_sec) AS total_time_spent,
avg(time_spent_sec) AS avg_time_spent,
countIf(is_bounce) AS bounce_count,
countIf(event_type = 'conversion') AS conversion_count,
uniqExact(referrer) AS unique_referrers,
countIf(device_type = 'mobile') AS mobile_count,
countIf(device_type = 'tablet') AS tablet_count,
countIf(device_type = 'desktop') AS desktop_count,
countIf(is_qr_scan) AS qr_scan_count,
sum(conversion_value) AS total_conversion_value
FROM
limq.link_events
GROUP BY
date,
link_id;
-- 每小时访问模式视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_hourly_patterns ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, hour, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
toHour(event_time) AS hour,
link_id,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
GROUP BY
date,
hour,
link_id;
-- 平台分布视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.platform_distribution ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, utm_source, device_type) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
utm_source,
device_type,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
WHERE
utm_source != ''
GROUP BY
date,
utm_source,
device_type;
-- 团队每日统计视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, team_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.team_id AS team_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.team_id != ''
GROUP BY
date,
l.team_id;
-- 项目每日统计视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, project_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.project_id AS project_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.project_id != ''
GROUP BY
date,
l.project_id;
-- QR码每日统计视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.qrcode_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, qr_code_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(scan_time) AS date,
qr_code_id,
count() AS total_scans,
uniqExact(visitor_id) AS unique_scanners,
countIf(led_to_conversion) AS conversions,
countIf(device_type = 'mobile') AS mobile_scans,
countIf(device_type = 'tablet') AS tablet_scans,
countIf(device_type = 'desktop') AS desktop_scans,
uniqExact(location) AS unique_locations
FROM
limq.qr_scans
GROUP BY
date,
qr_code_id;

View File

@@ -1,828 +0,0 @@
-- 清空现有数据(可选)
TRUNCATE TABLE IF EXISTS limq.link_events;
TRUNCATE TABLE IF EXISTS limq.link_daily_stats;
TRUNCATE TABLE IF EXISTS limq.link_hourly_patterns;
TRUNCATE TABLE IF EXISTS limq.links;
-- 使用固定的UUID值插入链接
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'11111111-1111-1111-1111-111111111111',
'https://example.com/page1',
now(),
'user-1',
'产品页面',
'我们的主要产品页面',
[ '产品',
'营销' ],
true
);
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'22222222-2222-2222-2222-222222222222',
'https://example.com/promo',
now(),
'user-1',
'促销活动',
'夏季特别促销活动',
[ '促销',
'活动' ],
true
);
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'33333333-3333-3333-3333-333333333333',
'https://example.com/blog',
now(),
'user-2',
'公司博客',
'公司新闻和更新',
[ '博客',
'内容' ],
true
);
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'44444444-4444-4444-4444-444444444444',
'https://example.com/signup',
now(),
'user-2',
'注册页面',
'新用户注册页面',
[ '转化',
'注册' ],
true
);
-- 为第一个链接创建500条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'11111111-1111-1111-1111-111111111111' AS link_id,
'channel-1' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 50 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 300 AS time_spent_sec,
rand() % 100 < 25 AS is_bounce,
rand() % 100 < 20 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 1.5 AS conversion_value,
'{}' AS custom_data
FROM
numbers(500);
-- 为第二个链接创建300条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'22222222-2222-2222-2222-222222222222' AS link_id,
'channel-1' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 40 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 300 AS time_spent_sec,
rand() % 100 < 25 AS is_bounce,
rand() % 100 < 15 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 2.5 AS conversion_value,
'{}' AS custom_data
FROM
numbers(300);
-- 为第三个链接创建200条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'33333333-3333-3333-3333-333333333333' AS link_id,
'channel-2' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 30 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 600 AS time_spent_sec,
rand() % 100 < 15 AS is_bounce,
rand() % 100 < 10 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 1.2 AS conversion_value,
'{}' AS custom_data
FROM
numbers(200);
-- 为第四个链接创建400条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'44444444-4444-4444-4444-444444444444' AS link_id,
'channel-2' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 60 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 400 AS time_spent_sec,
rand() % 100 < 20 AS is_bounce,
rand() % 100 < 25 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 3.5 AS conversion_value,
'{}' AS custom_data
FROM
numbers(400);
-- 插入link_daily_stats表数据
INSERT INTO
limq.link_daily_stats (
date,
link_id,
total_clicks,
unique_visitors,
unique_sessions,
total_time_spent,
avg_time_spent,
bounce_count,
conversion_count,
unique_referrers,
mobile_count,
tablet_count,
desktop_count,
qr_scan_count,
total_conversion_value
)
SELECT
subtractDays(today(), number) AS date,
multiIf(
number % 4 = 0,
'11111111-1111-1111-1111-111111111111',
number % 4 = 1,
'22222222-2222-2222-2222-222222222222',
number % 4 = 2,
'33333333-3333-3333-3333-333333333333',
'44444444-4444-4444-4444-444444444444'
) AS link_id,
50 + rand() % 100 AS total_clicks,
30 + rand() % 50 AS unique_visitors,
20 + rand() % 40 AS unique_sessions,
(500 + rand() % 1000) * 60 AS total_time_spent,
(rand() % 10) * 60 + rand() % 60 AS avg_time_spent,
5 + rand() % 20 AS bounce_count,
rand() % 30 AS conversion_count,
3 + rand() % 8 AS unique_referrers,
20 + rand() % 40 AS mobile_count,
5 + rand() % 15 AS tablet_count,
15 + rand() % 30 AS desktop_count,
rand() % 10 AS qr_scan_count,
rand() % 1000 * 2.5 AS total_conversion_value
FROM
numbers(30)
WHERE
number < 30;
-- 插入link_hourly_patterns表数据
INSERT INTO
limq.link_hourly_patterns (date, hour, link_id, visits, unique_visitors)
SELECT
subtractDays(today(), number % 7) AS date,
number % 24 AS hour,
multiIf(
intDiv(number, 24) % 4 = 0,
'11111111-1111-1111-1111-111111111111',
intDiv(number, 24) % 4 = 1,
'22222222-2222-2222-2222-222222222222',
intDiv(number, 24) % 4 = 2,
'33333333-3333-3333-3333-333333333333',
'44444444-4444-4444-4444-444444444444'
) AS link_id,
5 + rand() % 20 AS visits,
3 + rand() % 10 AS unique_visitors
FROM
numbers(672) -- 7天 x 24小时 x 4个链接
WHERE
number < 672;
-- 显示数据行数,验证插入成功
SELECT
'link_events 表行数:' AS metric,
count() AS value
FROM
limq.link_events
UNION
ALL
SELECT
'link_daily_stats 表行数:',
count()
FROM
limq.link_daily_stats
UNION
ALL
SELECT
'link_hourly_patterns 表行数:',
count()
FROM
limq.link_hourly_patterns;

View File

@@ -0,0 +1,364 @@
// Sync data from MongoDB trace table to ClickHouse events table
import { getVariable } from "npm:windmill-client@1";
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
interface MongoConfig {
host: string;
port: string;
db: string;
username: string;
password: string;
}
interface ClickHouseConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_database: string;
clickhouse_url: string;
}
interface TraceRecord {
_id: ObjectId;
slugId: ObjectId;
label: string | null;
ip: string;
type: number;
platform: string;
platformOS: string;
browser: string;
browserVersion: string;
url: string;
createTime: number;
}
export async function main(
batch_size = 1000,
max_records = 9999999,
timeout_minutes = 60,
skip_clickhouse_check = false,
force_insert = false
) {
const logWithTimestamp = (message: string) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
// Set timeout
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
return true;
}
return false;
};
// Get MongoDB and ClickHouse connection info
let mongoConfig: MongoConfig;
let clickhouseConfig: ClickHouseConfig;
try {
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
} catch (error) {
console.error("Failed to get config:", error);
throw error;
}
// Build MongoDB connection URL
let mongoUrl = "mongodb://";
if (mongoConfig.username && mongoConfig.password) {
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
}
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
// Connect to MongoDB
const client = new MongoClient();
try {
await client.connect(mongoUrl);
console.log("MongoDB connected successfully");
const db = client.database(mongoConfig.db);
const traceCollection = db.collection<TraceRecord>("trace");
// Build query conditions
const query: Record<string, unknown> = {
type: 1 // Only sync records with type 1
};
// Count total records
const totalRecords = await traceCollection.countDocuments(query);
console.log(`Found ${totalRecords} records to sync`);
const recordsToProcess = Math.min(totalRecords, max_records);
console.log(`Will process ${recordsToProcess} records`);
if (totalRecords === 0) {
console.log("No records to sync, task completed");
return {
success: true,
records_synced: 0,
message: "No records to sync"
};
}
// Check ClickHouse connection
const checkClickHouseConnection = async (): Promise<boolean> => {
if (skip_clickhouse_check) {
logWithTimestamp("Skipping ClickHouse connection check");
return true;
}
try {
logWithTimestamp("Testing ClickHouse connection...");
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
},
body: "SELECT 1",
signal: AbortSignal.timeout(5000)
});
if (response.ok) {
logWithTimestamp("ClickHouse connection test successful");
return true;
} else {
const errorText = await response.text();
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
return false;
}
} catch (err) {
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
return false;
}
};
// Check if records exist in ClickHouse
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
if (records.length === 0) return [];
if (skip_clickhouse_check || force_insert) {
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
return records;
}
try {
const recordIds = records.map(record => record._id.toString());
const query = `
SELECT event_id
FROM ${clickhouseConfig.clickhouse_database}.events
WHERE event_attributes LIKE '%"mongo_id":"%'
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
FORMAT JSON
`;
const response = await fetch(clickhouseConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: query,
signal: AbortSignal.timeout(10000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
}
const result = await response.json();
const existingIds = new Set(result.data.map((row: any) => {
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
return matches ? matches[1] : null;
}).filter(Boolean));
return records.filter(record => !existingIds.has(record._id.toString()));
} catch (err) {
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
return skip_clickhouse_check ? records : [];
}
};
// Process records function
const processRecords = async (records: TraceRecord[]) => {
if (records.length === 0) return 0;
const newRecords = await checkExistingRecords(records);
if (newRecords.length === 0) {
logWithTimestamp("All records already exist, skipping");
return 0;
}
// Prepare ClickHouse insert data
const clickhouseData = newRecords.map(record => {
const eventTime = new Date(record.createTime).toISOString();
return {
event_time: eventTime,
event_type: "click",
event_attributes: JSON.stringify({
mongo_id: record._id.toString(),
original_type: record.type
}),
// Link information
link_id: record.slugId.toString(),
link_slug: "",
link_label: record.label || "",
link_title: "",
link_original_url: record.url || "",
link_attributes: "{}",
link_created_at: eventTime,
link_expires_at: null,
link_tags: "[]",
// User information (empty as not available in trace)
user_id: "",
user_name: "",
user_email: "",
user_attributes: "{}",
// Team information (empty as not available in trace)
team_id: "",
team_name: "",
team_attributes: "{}",
// Project information (empty as not available in trace)
project_id: "",
project_name: "",
project_attributes: "{}",
// QR code information (empty as not available in trace)
qr_code_id: "",
qr_code_name: "",
qr_code_attributes: "{}",
// Visitor information
visitor_id: record._id.toString(),
session_id: `${record._id.toString()}-${record.createTime}`,
ip_address: record.ip || "",
country: "",
city: "",
device_type: record.platform || "unknown",
browser: record.browser || "",
os: record.platformOS || "",
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
// Source information
referrer: record.url || "",
utm_source: "",
utm_medium: "",
utm_campaign: "",
// Interaction information
time_spent_sec: 0,
is_bounce: true,
is_qr_scan: false,
conversion_type: "visit",
conversion_value: 0
};
});
// Generate ClickHouse insert SQL
const insertSQL = `
INSERT INTO ${clickhouseConfig.clickhouse_database}.events
FORMAT JSONEachRow
${JSON.stringify(clickhouseData)}
`;
try {
const response = await fetch(clickhouseConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: insertSQL,
signal: AbortSignal.timeout(20000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
}
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
return newRecords.length;
} catch (err) {
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
throw err;
}
};
// Check ClickHouse connection before processing
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
throw new Error("ClickHouse connection failed, cannot continue sync");
}
// Process records in batches
let processedRecords = 0;
let totalBatchRecords = 0;
for (let page = 0; processedRecords < recordsToProcess; page++) {
if (checkTimeout()) {
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
break;
}
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
const records = await traceCollection.find(
query,
{
allowDiskUse: true,
sort: { createTime: 1 },
skip: page * batch_size,
limit: batch_size
}
).toArray();
if (records.length === 0) {
logWithTimestamp("No more records found, sync complete");
break;
}
const batchSize = await processRecords(records);
processedRecords += records.length;
totalBatchRecords += batchSize;
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
}
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
message: "Data sync completed"
};
} catch (err) {
console.error("Error during sync:", err);
return {
success: false,
error: err instanceof Error ? err.message : String(err),
stack: err instanceof Error ? err.stack : undefined
};
} finally {
await client.close();
console.log("MongoDB connection closed");
}
}

View File

@@ -0,0 +1,409 @@
// Sync data from MongoDB trace table to ClickHouse events table
import { getVariable } from "npm:windmill-client@1";
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
interface MongoConfig {
host: string;
port: string;
db: string;
username: string;
password: string;
}
interface ClickHouseConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_url: string;
}
interface TraceRecord {
_id: ObjectId;
slugId: ObjectId;
label: string | null;
ip: string;
type: number;
platform: string;
platformOS: string;
browser: string;
browserVersion: string;
url: string;
createTime: number;
}
interface ShortRecord {
_id: ObjectId;
slug: string; // 短链接的slug部分
origin: string; // 原始URL
domain?: string; // 域名
createTime: number; // 创建时间戳
user?: string; // 创建用户
title?: string; // 标题
description?: string; // 描述
tags?: string[]; // 标签
active?: boolean; // 是否活跃
expiresAt?: number; // 过期时间戳
teamId?: string; // 团队ID
projectId?: string; // 项目ID
}
interface ClickHouseRow {
event_id: string;
event_attributes: string;
}
export async function main(
batch_size = 1000,
max_records = 9999999,
timeout_minutes = 60,
skip_clickhouse_check = false,
force_insert = false
) {
const logWithTimestamp = (message: string) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("Starting sync from MongoDB to ClickHouse events table");
logWithTimestamp(`Batch size: ${batch_size}, Max records: ${max_records}, Timeout: ${timeout_minutes} minutes`);
// Set timeout
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
console.log(`Execution time exceeded ${timeout_minutes} minutes, stopping`);
return true;
}
return false;
};
// Get MongoDB and ClickHouse connection info
let mongoConfig: MongoConfig;
let clickhouseConfig: ClickHouseConfig;
try {
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
mongoConfig = typeof rawMongoConfig === "string" ? JSON.parse(rawMongoConfig) : rawMongoConfig;
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
clickhouseConfig = typeof rawClickhouseConfig === "string" ? JSON.parse(rawClickhouseConfig) : rawClickhouseConfig;
} catch (error) {
console.error("Failed to get config:", error);
throw error;
}
// Build MongoDB connection URL
let mongoUrl = "mongodb://";
if (mongoConfig.username && mongoConfig.password) {
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
}
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
// Connect to MongoDB
const client = new MongoClient();
try {
await client.connect(mongoUrl);
console.log("MongoDB connected successfully");
const db = client.database(mongoConfig.db);
const traceCollection = db.collection<TraceRecord>("trace");
const shortCollection = db.collection<ShortRecord>("short");
// Build query conditions
const query: Record<string, unknown> = {
type: 1 // Only sync records with type 1
};
// Count total records
const totalRecords = await traceCollection.countDocuments(query);
console.log(`Found ${totalRecords} records to sync`);
const recordsToProcess = Math.min(totalRecords, max_records);
console.log(`Will process ${recordsToProcess} records`);
if (totalRecords === 0) {
console.log("No records to sync, task completed");
return {
success: true,
records_synced: 0,
message: "No records to sync"
};
}
// Check ClickHouse connection
const checkClickHouseConnection = async (): Promise<boolean> => {
if (skip_clickhouse_check) {
logWithTimestamp("Skipping ClickHouse connection check");
return true;
}
try {
logWithTimestamp("Testing ClickHouse connection...");
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
},
body: "SELECT 1",
signal: AbortSignal.timeout(5000)
});
if (response.ok) {
logWithTimestamp("ClickHouse connection test successful");
return true;
} else {
const errorText = await response.text();
logWithTimestamp(`ClickHouse connection test failed: ${response.status} ${errorText}`);
return false;
}
} catch (err) {
logWithTimestamp(`ClickHouse connection test failed: ${(err as Error).message}`);
return false;
}
};
// Check if records exist in ClickHouse
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
if (records.length === 0) return [];
if (skip_clickhouse_check || force_insert) {
logWithTimestamp(`Skipping ClickHouse duplicate check, will process all ${records.length} records`);
return records;
}
try {
const recordIds = records.map(record => record._id.toString());
const query = `
SELECT event_id
FROM shorturl_analytics.events
WHERE event_attributes LIKE '%"mongo_id":"%'
AND event_attributes LIKE ANY ('%${recordIds.join("%' OR '%")}%')
FORMAT JSON
`;
const response = await fetch(clickhouseConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: query,
signal: AbortSignal.timeout(10000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse query error: ${response.status} ${errorText}`);
}
const result = await response.json();
const existingIds = new Set(result.data.map((row: ClickHouseRow) => {
const matches = row.event_attributes.match(/"mongo_id":"([^"]+)"/);
return matches ? matches[1] : null;
}).filter(Boolean));
return records.filter(record => !existingIds.has(record._id.toString()));
} catch (err) {
logWithTimestamp(`Error checking existing records: ${(err as Error).message}`);
return skip_clickhouse_check ? records : [];
}
};
// Process records function
const processRecords = async (records: TraceRecord[]) => {
if (records.length === 0) return 0;
const newRecords = await checkExistingRecords(records);
if (newRecords.length === 0) {
logWithTimestamp("All records already exist, skipping");
return 0;
}
// Get link information for all records
const slugIds = newRecords.map(record => record.slugId);
const shortLinks = await shortCollection.find({
_id: { $in: slugIds }
}).toArray();
// Create a map for quick lookup
const shortLinksMap = new Map(shortLinks.map(link => [link._id.toString(), link]));
// Prepare ClickHouse insert data
const clickhouseData = newRecords.map(record => {
const shortLink = shortLinksMap.get(record.slugId.toString());
// 将毫秒时间戳转换为 DateTime64(3) 格式
const formatDateTime = (timestamp: number) => {
return new Date(timestamp).toISOString().replace('T', ' ').replace('Z', '');
};
return {
// Event base information
event_id: record._id.toString(),
event_time: formatDateTime(record.createTime),
event_type: "click",
event_attributes: JSON.stringify({
original_type: record.type
}),
// Link information from short collection
link_id: record.slugId.toString(),
link_slug: shortLink?.slug || "",
link_label: record.label || "",
link_title: "",
link_original_url: shortLink?.origin || "",
link_attributes: JSON.stringify({
domain: shortLink?.domain || null
}),
link_created_at: shortLink?.createTime ? formatDateTime(shortLink.createTime) : formatDateTime(record.createTime),
link_expires_at: shortLink?.expiresAt ? formatDateTime(shortLink.expiresAt) : null,
link_tags: "[]", // Empty array as default
// User information
user_id: shortLink?.user || "",
user_name: "",
user_email: "",
user_attributes: "{}",
// Team information
team_id: shortLink?.teamId || "",
team_name: "",
team_attributes: "{}",
// Project information
project_id: shortLink?.projectId || "",
project_name: "",
project_attributes: "{}",
// QR code information
qr_code_id: "",
qr_code_name: "",
qr_code_attributes: "{}",
// Visitor information
visitor_id: "", // Empty string as default
session_id: `${record.slugId.toString()}-${record.createTime}`,
ip_address: record.ip || "",
country: "",
city: "",
device_type: record.platform || "",
browser: record.browser || "",
os: record.platformOS || "",
user_agent: `${record.browser || ""} ${record.browserVersion || ""}`.trim(),
// Source information
referrer: record.url || "",
utm_source: "",
utm_medium: "",
utm_campaign: "",
// Interaction information
time_spent_sec: 0,
is_bounce: true,
is_qr_scan: false,
conversion_type: "visit",
conversion_value: 0
};
});
// Generate ClickHouse insert SQL
const rows = clickhouseData.map(row => {
// 只需要处理JSON字符串的转义
const formattedRow = {
...row,
event_attributes: row.event_attributes.replace(/\\/g, '\\\\'),
link_attributes: row.link_attributes.replace(/\\/g, '\\\\')
};
return JSON.stringify(formattedRow);
}).join('\n');
const insertSQL = `INSERT INTO shorturl_analytics.events FORMAT JSONEachRow\n${rows}`;
try {
const response = await fetch(clickhouseConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: insertSQL,
signal: AbortSignal.timeout(20000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse insert error: ${response.status} ${errorText}`);
}
logWithTimestamp(`Successfully inserted ${newRecords.length} records to ClickHouse`);
return newRecords.length;
} catch (err) {
logWithTimestamp(`Failed to insert data to ClickHouse: ${(err as Error).message}`);
throw err;
}
};
// Check ClickHouse connection before processing
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
throw new Error("ClickHouse connection failed, cannot continue sync");
}
// Process records in batches
let processedRecords = 0;
let totalBatchRecords = 0;
for (let page = 0; processedRecords < recordsToProcess; page++) {
if (checkTimeout()) {
logWithTimestamp(`Processed ${processedRecords}/${recordsToProcess} records, stopping due to timeout`);
break;
}
logWithTimestamp(`Processing batch ${page+1}, completed ${processedRecords}/${recordsToProcess} records (${Math.round(processedRecords/recordsToProcess*100)}%)`);
const records = await traceCollection.find(
query,
{
allowDiskUse: true,
sort: { createTime: 1 },
skip: page * batch_size,
limit: batch_size
}
).toArray();
if (records.length === 0) {
logWithTimestamp("No more records found, sync complete");
break;
}
const batchSize = await processRecords(records);
processedRecords += records.length;
totalBatchRecords += batchSize;
logWithTimestamp(`Batch ${page+1} complete. Processed ${processedRecords}/${recordsToProcess} records, inserted ${totalBatchRecords} (${Math.round(processedRecords/recordsToProcess*100)}%)`);
}
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
message: "Data sync completed"
};
} catch (err) {
console.error("Error during sync:", err);
return {
success: false,
error: err instanceof Error ? err.message : String(err),
stack: err instanceof Error ? err.stack : undefined
};
} finally {
await client.close();
console.log("MongoDB connection closed");
}
}

View File

@@ -1,474 +0,0 @@
// 从MongoDB的trace表同步数据到ClickHouse的link_events表
import { getVariable, setVariable } from "npm:windmill-client@1";
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
interface MongoConfig {
host: string;
port: string;
db: string;
username: string;
password: string;
}
interface ClickHouseConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_database: string;
clickhouse_url: string;
}
interface TraceRecord {
_id: ObjectId;
slugId: ObjectId;
label: string | null;
ip: string;
type: number;
platform: string;
platformOS: string;
browser: string;
browserVersion: string;
url: string;
createTime: number;
}
interface SyncState {
last_sync_time: number;
records_synced: number;
last_sync_id?: string;
}
export async function main(
batch_size = 1000,
max_records = 9999999,
timeout_minutes = 60,
skip_clickhouse_check = false,
force_insert = false
) {
const logWithTimestamp = (message: string) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("开始执行MongoDB到ClickHouse的同步任务");
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
if (skip_clickhouse_check) {
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式不会检查记录是否已存在");
}
if (force_insert) {
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
}
// 设置超时
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
// 检查是否超时
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
console.log(`运行时间超过${timeout_minutes}分钟,暂停执行`);
return true;
}
return false;
};
// 获取MongoDB和ClickHouse的连接信息
let mongoConfig: MongoConfig;
let clickhouseConfig: ClickHouseConfig;
try {
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
console.log("原始MongoDB配置:", JSON.stringify(rawMongoConfig));
// 尝试解析配置,如果是字符串形式
if (typeof rawMongoConfig === "string") {
try {
mongoConfig = JSON.parse(rawMongoConfig);
} catch (e) {
console.error("MongoDB配置解析失败:", e);
throw e;
}
} else {
mongoConfig = rawMongoConfig as MongoConfig;
}
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
console.log("原始ClickHouse配置:", JSON.stringify(rawClickhouseConfig));
// 尝试解析配置,如果是字符串形式
if (typeof rawClickhouseConfig === "string") {
try {
clickhouseConfig = JSON.parse(rawClickhouseConfig);
} catch (e) {
console.error("ClickHouse配置解析失败:", e);
throw e;
}
} else {
clickhouseConfig = rawClickhouseConfig as ClickHouseConfig;
}
console.log("MongoDB配置解析为:", JSON.stringify(mongoConfig));
console.log("ClickHouse配置解析为:", JSON.stringify(clickhouseConfig));
} catch (error) {
console.error("获取配置失败:", error);
throw error;
}
// 构建MongoDB连接URL
let mongoUrl = "mongodb://";
if (mongoConfig.username && mongoConfig.password) {
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
}
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
// 连接MongoDB
const client = new MongoClient();
try {
await client.connect(mongoUrl);
console.log("MongoDB连接成功");
const db = client.database(mongoConfig.db);
const traceCollection = db.collection<TraceRecord>("trace");
// 构建查询条件,获取所有记录
const query: Record<string, unknown> = {
type: 1 // 只同步type为1的记录
};
// 计算总记录数
const totalRecords = await traceCollection.countDocuments(query);
console.log(`找到 ${totalRecords} 条记录需要同步`);
// 限制此次处理的记录数量
const recordsToProcess = Math.min(totalRecords, max_records);
console.log(`本次将处理 ${recordsToProcess} 条记录`);
if (totalRecords === 0) {
console.log("没有记录需要同步,任务完成");
return {
success: true,
records_synced: 0,
message: "没有记录需要同步"
};
}
// 检查ClickHouse连接状态
const checkClickHouseConnection = async (): Promise<boolean> => {
if (skip_clickhouse_check) {
logWithTimestamp("已启用跳过ClickHouse检查不测试连接");
return true;
}
try {
logWithTimestamp("测试ClickHouse连接...");
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
},
body: "SELECT 1",
// 设置5秒超时
signal: AbortSignal.timeout(5000)
});
if (response.ok) {
logWithTimestamp("ClickHouse连接测试成功");
return true;
} else {
const errorText = await response.text();
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
return false;
}
} catch (err) {
const error = err as Error;
logWithTimestamp(`ClickHouse连接测试失败: ${error.message}`);
return false;
}
};
// 检查记录是否已经存在于ClickHouse中
const checkExistingRecords = async (records: TraceRecord[]): Promise<TraceRecord[]> => {
if (records.length === 0) return [];
// 如果跳过ClickHouse检查或强制插入则直接返回所有记录
if (skip_clickhouse_check || force_insert) {
logWithTimestamp(`已跳过ClickHouse重复检查准备处理所有 ${records.length} 条记录`);
return records;
}
logWithTimestamp(`正在检查 ${records.length} 条记录是否已存在于ClickHouse中...`);
try {
// 提取所有记录的ID
const recordIds = records.map(record => record.slugId.toString()); // 使用slugId作为link_id查询
logWithTimestamp(`待检查的记录ID: ${recordIds.join(', ')}`);
// 构建查询SQL检查记录是否已存在确保添加FORMAT JSON来获取正确的JSON格式响应
const query = `
SELECT link_id
FROM ${clickhouseConfig.clickhouse_database}.link_events
WHERE link_id IN ('${recordIds.join("','")}')
FORMAT JSON
`;
logWithTimestamp(`执行ClickHouse查询: ${query.replace(/\n\s*/g, ' ')}`);
// 发送请求到ClickHouse添加10秒超时
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: query,
signal: AbortSignal.timeout(10000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse查询错误: ${response.status} ${errorText}`);
}
// 获取响应文本以便记录
const responseText = await response.text();
logWithTimestamp(`ClickHouse查询响应: ${responseText.slice(0, 200)}${responseText.length > 200 ? '...' : ''}`);
if (!responseText.trim()) {
logWithTimestamp("ClickHouse返回空响应假定没有记录存在");
return records; // 如果响应为空,假设没有记录
}
// 解析结果
let result;
try {
result = JSON.parse(responseText);
} catch (err) {
logWithTimestamp(`ClickHouse响应不是有效的JSON: ${responseText}`);
throw new Error(`解析ClickHouse响应失败: ${(err as Error).message}`);
}
// 确保result有正确的结构
if (!result.data) {
logWithTimestamp(`ClickHouse响应缺少data字段: ${JSON.stringify(result)}`);
return records; // 如果没有data字段假设没有记录
}
// 提取已存在的记录ID
const existingIds = new Set(result.data.map((row: { link_id: string }) => row.link_id));
logWithTimestamp(`检测到 ${existingIds.size} 条记录已存在于ClickHouse中`);
if (existingIds.size > 0) {
logWithTimestamp(`已存在的记录ID: ${Array.from(existingIds).join(', ')}`);
}
// 过滤出不存在的记录
const newRecords = records.filter(record => !existingIds.has(record.slugId.toString())); // 使用slugId匹配link_id
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
return newRecords;
} catch (err) {
const error = err as Error;
logWithTimestamp(`ClickHouse查询出错: ${error.message}`);
if (skip_clickhouse_check) {
logWithTimestamp("已启用跳过ClickHouse检查将继续处理所有记录");
return records;
} else {
throw error; // 如果没有启用跳过检查,则抛出错误
}
}
};
// 在处理记录前先检查ClickHouse连接
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
logWithTimestamp("⚠️ ClickHouse连接测试失败请启用skip_clickhouse_check=true参数来跳过连接检查");
throw new Error("ClickHouse连接失败无法继续同步");
}
// 处理记录的函数
const processRecords = async (records: TraceRecord[]) => {
if (records.length === 0) return 0;
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条记录...`);
// 检查记录是否已存在
let newRecords;
try {
newRecords = await checkExistingRecords(records);
} catch (err) {
const error = err as Error;
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
if (!skip_clickhouse_check && !force_insert) {
throw error;
}
// 如果跳过检查或强制插入,则使用所有记录
logWithTimestamp("将使用所有记录进行处理");
newRecords = records;
}
if (newRecords.length === 0) {
logWithTimestamp("所有记录都已存在,跳过处理");
return 0;
}
logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
// 准备ClickHouse插入数据
const clickhouseData = newRecords.map(record => {
// 转换MongoDB记录为ClickHouse格式匹配ClickHouse表结构
return {
// UUID将由ClickHouse自动生成 (event_id)
link_id: record.slugId.toString(),
channel_id: record.label || "",
visitor_id: record._id.toString(), // 使用MongoDB ID作为访客ID
session_id: record._id.toString() + "-" + record.createTime, // 创建一个唯一会话ID
event_type: record.type <= 4 ? record.type : 1, // 确保event_type在枚举范围内
ip_address: record.ip,
country: "", // 这些字段在MongoDB中不存在使用默认值
city: "",
referrer: record.url || "",
utm_source: "",
utm_medium: "",
utm_campaign: "",
user_agent: record.browser + " " + record.browserVersion,
device_type: record.platform || "unknown",
browser: record.browser || "",
os: record.platformOS || "",
time_spent_sec: 0,
is_bounce: true,
is_qr_scan: false,
qr_code_id: "",
conversion_type: 1, // 默认为'visit'
conversion_value: 0,
custom_data: `{"mongo_id":"${record._id.toString()}"}`
};
});
// 生成ClickHouse插入SQL
const insertSQL = `
INSERT INTO ${clickhouseConfig.clickhouse_database}.link_events
(link_id, channel_id, visitor_id, session_id, event_type, ip_address, country, city,
referrer, utm_source, utm_medium, utm_campaign, user_agent, device_type, browser, os,
time_spent_sec, is_bounce, is_qr_scan, qr_code_id, conversion_type, conversion_value, custom_data)
VALUES ${clickhouseData.map(record => {
// 确保所有字符串值都是字符串类型,并安全处理替换
const safeReplace = (val: any): string => {
// 确保值是字符串如果是null或undefined则使用空字符串
const str = val === null || val === undefined ? "" : String(val);
// 安全替换单引号
return str.replace(/'/g, "''");
};
return `('${record.link_id}', '${safeReplace(record.channel_id)}', '${record.visitor_id}', '${record.session_id}',
${record.event_type}, '${safeReplace(record.ip_address)}', '', '',
'${safeReplace(record.referrer)}', '', '', '', '${safeReplace(record.user_agent)}', '${safeReplace(record.device_type)}',
'${safeReplace(record.browser)}', '${safeReplace(record.os)}',
0, true, false, '', 1, 0, '${safeReplace(record.custom_data)}')`;
}).join(", ")}
`;
if (insertSQL.length === 0) {
console.log("没有新记录需要插入");
return 0;
}
// 发送请求到ClickHouse添加20秒超时
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
try {
logWithTimestamp("发送插入请求到ClickHouse...");
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: insertSQL,
signal: AbortSignal.timeout(20000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
}
logWithTimestamp(`成功插入 ${newRecords.length} 条记录到ClickHouse`);
return newRecords.length;
} catch (err) {
const error = err as Error;
logWithTimestamp(`向ClickHouse插入数据失败: ${error.message}`);
throw error;
}
};
// 批量处理记录
let processedRecords = 0;
let totalBatchRecords = 0;
for (let page = 0; processedRecords < recordsToProcess; page++) {
// 检查超时
if (checkTimeout()) {
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
break;
}
// 每批次都输出进度
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
const records = await traceCollection.find(
query,
{
allowDiskUse: true,
sort: { createTime: 1 },
skip: page * batch_size,
limit: batch_size
}
).toArray();
if (records.length === 0) {
logWithTimestamp("没有找到更多数据,同步结束");
break;
}
// 找到数据,开始处理
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
// 输出当前批次的部分数据信息
if (records.length > 0) {
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, 时间=${new Date(records[0].createTime).toISOString()}`);
if (records.length > 1) {
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
}
}
const batchSize = await processRecords(records);
processedRecords += records.length;
totalBatchRecords += batchSize;
logWithTimestamp(`${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
}
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
message: "数据同步完成"
};
} catch (err) {
console.error("同步过程中发生错误:", err);
return {
success: false,
error: err instanceof Error ? err.message : String(err),
stack: err instanceof Error ? err.stack : undefined
};
} finally {
// 关闭MongoDB连接
await client.close();
console.log("MongoDB连接已关闭");
}
}

View File

@@ -1,532 +0,0 @@
// 从MongoDB的short表同步数据到ClickHouse的links表
import { getVariable, setVariable } from "npm:windmill-client@1";
import { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.32.0/mod.ts";
interface MongoConfig {
host: string;
port: string;
db: string;
username: string;
password: string;
}
interface ClickHouseConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_database: string;
clickhouse_url: string;
}
interface ShortRecord {
_id: ObjectId;
slug: string; // 短链接的slug部分
origin: string; // 原始URL
domain?: string; // 域名
createTime: number; // 创建时间戳
user?: string; // 创建用户
title?: string; // 标题
description?: string; // 描述
tags?: string[]; // 标签
active?: boolean; // 是否活跃
expiresAt?: number; // 过期时间戳
teamId?: string; // 团队ID
projectId?: string; // 项目ID
}
interface SyncState {
last_sync_time: number;
records_synced: number;
last_sync_id?: string;
}
export async function main(
batch_size = 100,
initial_sync = false,
max_records = 999999,
timeout_minutes = 30,
skip_clickhouse_check = false,
force_insert = false
) {
const logWithTimestamp = (message: string) => {
const now = new Date();
console.log(`[${now.toISOString()}] ${message}`);
};
logWithTimestamp("开始执行MongoDB到ClickHouse的短链接同步任务...");
logWithTimestamp(`批处理大小: ${batch_size}, 最大记录数: ${max_records}, 超时时间: ${timeout_minutes}分钟`);
if (skip_clickhouse_check) {
logWithTimestamp("⚠️ 警告: 已启用跳过ClickHouse检查模式不会检查记录是否已存在");
}
if (force_insert) {
logWithTimestamp("⚠️ 警告: 已启用强制插入模式,将尝试插入所有记录");
}
// 设置超时
const startTime = Date.now();
const timeoutMs = timeout_minutes * 60 * 1000;
// 检查是否超时
const checkTimeout = () => {
if (Date.now() - startTime > timeoutMs) {
console.log(`运行时间超过${timeout_minutes}分钟,暂停执行`);
return true;
}
return false;
};
// 获取MongoDB和ClickHouse的连接信息
let mongoConfig: MongoConfig;
let clickhouseConfig: ClickHouseConfig;
try {
const rawMongoConfig = await getVariable("f/shorturl_analytics/mongodb");
console.log("原始MongoDB配置:", typeof rawMongoConfig === "string" ? rawMongoConfig : JSON.stringify(rawMongoConfig));
// 尝试解析配置,如果是字符串形式
if (typeof rawMongoConfig === "string") {
try {
mongoConfig = JSON.parse(rawMongoConfig);
} catch (e) {
console.error("MongoDB配置解析失败:", e);
throw e;
}
} else {
mongoConfig = rawMongoConfig as MongoConfig;
}
const rawClickhouseConfig = await getVariable("f/shorturl_analytics/clickhouse");
console.log("原始ClickHouse配置:", typeof rawClickhouseConfig === "string" ? rawClickhouseConfig : JSON.stringify(rawClickhouseConfig));
// 尝试解析配置,如果是字符串形式
if (typeof rawClickhouseConfig === "string") {
try {
clickhouseConfig = JSON.parse(rawClickhouseConfig);
} catch (e) {
console.error("ClickHouse配置解析失败:", e);
throw e;
}
} else {
clickhouseConfig = rawClickhouseConfig as ClickHouseConfig;
}
console.log("MongoDB配置解析为:", JSON.stringify(mongoConfig));
console.log("ClickHouse配置解析为:", JSON.stringify(clickhouseConfig));
} catch (error) {
console.error("获取配置失败:", error);
throw error;
}
// 构建MongoDB连接URL
let mongoUrl = "mongodb://";
if (mongoConfig.username && mongoConfig.password) {
mongoUrl += `${mongoConfig.username}:${mongoConfig.password}@`;
}
mongoUrl += `${mongoConfig.host}:${mongoConfig.port}/${mongoConfig.db}`;
console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
// 获取上次同步的状态
let syncState: SyncState;
try {
const rawSyncState = await getVariable<string>("f/shorturl_analytics/clickhouse/shorturl_links_sync_state");
try {
syncState = JSON.parse(rawSyncState);
console.log(`获取同步状态成功: 上次同步时间 ${new Date(syncState.last_sync_time).toISOString()}`);
} catch (parseError) {
console.error("解析同步状态失败:", parseError);
throw parseError;
}
} catch (_unused_error) {
console.log("未找到同步状态,创建初始同步状态");
syncState = {
last_sync_time: 0,
records_synced: 0,
};
}
// 如果强制从头开始同步
if (initial_sync) {
console.log("强制从头开始同步");
syncState = {
last_sync_time: 0,
records_synced: 0,
};
}
// 连接MongoDB
const client = new MongoClient();
try {
await client.connect(mongoUrl);
console.log("MongoDB连接成功");
const db = client.database(mongoConfig.db);
const shortCollection = db.collection<ShortRecord>("short");
// 构建查询条件,只查询新的记录
const query: Record<string, unknown> = {};
if (syncState.last_sync_time > 0) {
query.createTime = { $gt: syncState.last_sync_time };
}
if (syncState.last_sync_id) {
// 如果有上次同步的ID则从该ID之后开始查询
query._id = { $gt: new ObjectId(syncState.last_sync_id) };
}
// 计算总记录数
const totalRecords = await shortCollection.countDocuments(query);
console.log(`找到 ${totalRecords} 条新短链接记录需要同步`);
// 限制此次处理的记录数量
const recordsToProcess = Math.min(totalRecords, max_records);
console.log(`本次将处理 ${recordsToProcess} 条记录`);
if (totalRecords === 0) {
console.log("没有新记录需要同步,任务完成");
return {
success: true,
records_synced: 0,
total_synced: syncState.records_synced,
message: "没有新记录需要同步"
};
}
// 分批处理记录
let processedRecords = 0;
let lastId: string | undefined;
let lastCreateTime = syncState.last_sync_time;
let totalBatchRecords = 0;
// 检查ClickHouse连接状态
const checkClickHouseConnection = async (): Promise<boolean> => {
if (skip_clickhouse_check) {
logWithTimestamp("已启用跳过ClickHouse检查不测试连接");
return true;
}
try {
logWithTimestamp("测试ClickHouse连接...");
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`,
},
body: "SELECT 1 FORMAT JSON",
signal: AbortSignal.timeout(5000)
});
if (response.ok) {
logWithTimestamp("ClickHouse连接测试成功");
return true;
} else {
const errorText = await response.text();
logWithTimestamp(`ClickHouse连接测试失败: ${response.status} ${errorText}`);
return false;
}
} catch (err) {
const error = err as Error;
logWithTimestamp(`ClickHouse连接测试失败: ${error.message}`);
return false;
}
};
// 检查记录是否已经存在于ClickHouse中
const checkExistingRecords = async (records: ShortRecord[]): Promise<ShortRecord[]> => {
if (records.length === 0) return [];
// 如果跳过ClickHouse检查或强制插入则直接返回所有记录
if (skip_clickhouse_check || force_insert) {
logWithTimestamp(`已跳过ClickHouse重复检查准备处理所有 ${records.length} 条记录`);
return records;
}
logWithTimestamp(`正在检查 ${records.length} 条短链接记录是否已存在于ClickHouse中...`);
try {
// 提取所有记录的ID
const recordIds = records.map(record => record._id.toString());
logWithTimestamp(`待检查的短链接ID: ${recordIds.join(', ')}`);
// 构建查询SQL检查记录是否已存在
const query = `
SELECT link_id
FROM ${clickhouseConfig.clickhouse_database}.links
WHERE link_id IN ('${recordIds.join("','")}')
FORMAT JSON
`;
logWithTimestamp(`执行ClickHouse查询: ${query.replace(/\n\s*/g, ' ')}`);
// 发送请求到ClickHouse
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: query,
signal: AbortSignal.timeout(10000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse查询错误: ${response.status} ${errorText}`);
}
// 获取响应文本
const responseText = await response.text();
logWithTimestamp(`ClickHouse查询响应: ${responseText.slice(0, 200)}${responseText.length > 200 ? '...' : ''}`);
if (!responseText.trim()) {
logWithTimestamp("ClickHouse返回空响应假定没有记录存在");
return records;
}
// 解析结果
let result;
try {
result = JSON.parse(responseText);
} catch (err) {
logWithTimestamp(`ClickHouse响应不是有效的JSON: ${responseText}`);
throw new Error(`解析ClickHouse响应失败: ${(err as Error).message}`);
}
// 确保result有正确的结构
if (!result.data) {
logWithTimestamp(`ClickHouse响应缺少data字段: ${JSON.stringify(result)}`);
return records;
}
// 提取已存在的记录ID
const existingIds = new Set(result.data.map((row: { link_id: string }) => row.link_id));
logWithTimestamp(`检测到 ${existingIds.size} 条记录已存在于ClickHouse中`);
if (existingIds.size > 0) {
logWithTimestamp(`已存在的记录ID: ${Array.from(existingIds).join(', ')}`);
}
// 过滤出不存在的记录
const newRecords = records.filter(record => !existingIds.has(record._id.toString()));
logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
return newRecords;
} catch (err) {
const error = err as Error;
logWithTimestamp(`ClickHouse查询出错: ${error.message}`);
if (skip_clickhouse_check) {
logWithTimestamp("已启用跳过ClickHouse检查将继续处理所有记录");
return records;
} else {
throw error;
}
}
};
// 在处理记录前先检查ClickHouse连接
const clickhouseConnected = await checkClickHouseConnection();
if (!clickhouseConnected && !skip_clickhouse_check) {
logWithTimestamp("⚠️ ClickHouse连接测试失败请启用skip_clickhouse_check=true参数来跳过连接检查");
throw new Error("ClickHouse连接失败无法继续同步");
}
// 处理记录的函数
const processRecords = async (records: ShortRecord[]) => {
if (records.length === 0) return 0;
logWithTimestamp(`开始处理批次数据,共 ${records.length} 条短链接记录...`);
// 检查记录是否已存在
let newRecords;
try {
newRecords = await checkExistingRecords(records);
} catch (err) {
const error = err as Error;
logWithTimestamp(`检查记录是否存在时出错: ${error.message}`);
if (!skip_clickhouse_check && !force_insert) {
throw error;
}
// 如果跳过检查或强制插入,则使用所有记录
logWithTimestamp("将使用所有记录进行处理");
newRecords = records;
}
if (newRecords.length === 0) {
logWithTimestamp("所有记录都已存在,跳过处理");
// 更新同步状态,即使没有新增记录
const lastRecord = records[records.length - 1];
lastId = lastRecord._id.toString();
lastCreateTime = lastRecord.createTime;
return 0;
}
logWithTimestamp(`准备处理 ${newRecords.length} 条新短链接记录...`);
// 准备ClickHouse插入数据
const clickhouseData = newRecords.map(record => {
// 转换MongoDB记录为ClickHouse格式匹配ClickHouse表结构
// 处理日期时间移除ISO格式中的Z以使ClickHouse正确解析
const createdAtStr = new Date(record.createTime).toISOString().replace('Z', '');
const expiresAtStr = record.expiresAt ? new Date(record.expiresAt).toISOString().replace('Z', '') : null;
return {
link_id: record._id.toString(), // 使用MongoDB的_id作为link_id
original_url: record.origin || "",
created_at: createdAtStr,
created_by: record.user || "unknown",
title: record.slug, // 使用slug作为title
description: record.description || "",
tags: record.tags || [],
is_active: record.active !== undefined ? record.active : true,
expires_at: expiresAtStr,
team_id: record.teamId || "",
project_id: record.projectId || ""
};
});
// 更新同步状态使用原始records的最后一条以确保进度正确
const lastRecord = records[records.length - 1];
lastId = lastRecord._id.toString();
lastCreateTime = lastRecord.createTime;
logWithTimestamp(`更新同步位置到: ID=${lastId}, 时间=${new Date(lastCreateTime).toISOString()}`);
// 生成ClickHouse插入SQL
// 注意Array类型需要特殊处理这里将tags作为JSON字符串处理
const insertSQL = `
INSERT INTO ${clickhouseConfig.clickhouse_database}.links
(link_id, original_url, created_at, created_by, title, description, tags, is_active, expires_at, team_id, project_id)
VALUES
${clickhouseData.map(record => {
// 处理tags数组
const tagsStr = JSON.stringify(record.tags || []);
// 处理expires_at可能为null的情况
const expiresAt = record.expires_at ? `'${record.expires_at}'` : "NULL";
// 确保所有字段在使用replace前都有默认值
const safeOriginalUrl = (record.original_url || "").replace(/'/g, "''");
const safeCreatedBy = (record.created_by || "unknown").replace(/'/g, "''");
const safeTitle = (record.title || "无标题").replace(/'/g, "''");
const safeDescription = (record.description || "").replace(/'/g, "''");
const safeTeamId = record.team_id || "";
const safeProjectId = record.project_id || "";
return `('${record.link_id}', '${safeOriginalUrl}', '${record.created_at}', '${safeCreatedBy}', '${safeTitle}', '${safeDescription}', ${tagsStr}, ${record.is_active}, ${expiresAt}, '${safeTeamId}', '${safeProjectId}')`;
}).join(", ")}
`;
if (clickhouseData.length === 0) {
console.log("没有新记录需要插入");
return 0;
}
// 发送请求到ClickHouse
const clickhouseUrl = `${clickhouseConfig.clickhouse_url}`;
try {
logWithTimestamp("发送插入请求到ClickHouse...");
const response = await fetch(clickhouseUrl, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${clickhouseConfig.clickhouse_user}:${clickhouseConfig.clickhouse_password}`)}`
},
body: insertSQL,
signal: AbortSignal.timeout(20000)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse插入错误: ${response.status} ${errorText}`);
}
logWithTimestamp(`成功插入 ${newRecords.length} 条短链接记录到ClickHouse`);
return newRecords.length;
} catch (err) {
const error = err as Error;
logWithTimestamp(`向ClickHouse插入数据失败: ${error.message}`);
throw error;
}
};
// 批量处理记录
for (let page = 0; processedRecords < recordsToProcess; page++) {
// 检查超时
if (checkTimeout()) {
logWithTimestamp(`已处理 ${processedRecords}/${recordsToProcess} 条记录,因超时暂停执行`);
break;
}
// 每批次都输出进度
logWithTimestamp(`开始处理第 ${page+1} 批次,已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
const records = await shortCollection.find(query)
.sort({ createTime: 1, _id: 1 })
.skip(page * batch_size)
.limit(batch_size)
.toArray();
if (records.length === 0) {
logWithTimestamp(`${page+1} 批次没有找到数据,结束处理`);
break;
}
logWithTimestamp(`获取到 ${records.length} 条记录,开始处理...`);
// 输出当前批次的部分数据信息
if (records.length > 0) {
logWithTimestamp(`批次 ${page+1} 第一条记录: ID=${records[0]._id}, Slug=${records[0].slug}, 时间=${new Date(records[0].createTime).toISOString()}`);
if (records.length > 1) {
logWithTimestamp(`批次 ${page+1} 最后一条记录: ID=${records[records.length-1]._id}, Slug=${records[records.length-1].slug}, 时间=${new Date(records[records.length-1].createTime).toISOString()}`);
}
}
const batchSize = await processRecords(records);
processedRecords += records.length; // 总是增加处理的记录数,即使有些记录已存在
totalBatchRecords += batchSize; // 只增加实际插入的记录数
logWithTimestamp(`${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录,实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
// 更新查询条件,以便下一批次查询
query.createTime = { $gt: lastCreateTime };
if (lastId) {
query._id = { $gt: new ObjectId(lastId) };
}
logWithTimestamp(`更新查询条件: 创建时间 > ${new Date(lastCreateTime).toISOString()}, ID > ${lastId || 'none'}`);
}
// 更新同步状态
const newSyncState: SyncState = {
last_sync_time: lastCreateTime,
records_synced: syncState.records_synced + totalBatchRecords,
last_sync_id: lastId
};
await setVariable("f/shorturl_analytics/clickhouse/shorturl_links_sync_state", JSON.stringify(newSyncState));
console.log(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 总同步记录数 ${newSyncState.records_synced}`);
return {
success: true,
records_processed: processedRecords,
records_synced: totalBatchRecords,
total_synced: newSyncState.records_synced,
last_sync_time: new Date(newSyncState.last_sync_time).toISOString(),
message: "短链接数据同步完成"
};
} catch (err) {
console.error("同步过程中发生错误:", err);
return {
success: false,
error: err instanceof Error ? err.message : String(err),
stack: err instanceof Error ? err.stack : undefined
};
} finally {
// 关闭MongoDB连接
await client.close();
console.log("MongoDB连接已关闭");
}
}