sync event data

app/links/[id]/page.tsx

@@ -16,6 +16,11 @@ import {
   ReferrerItem,
 } from "@/app/api/types";
 import { TimeGranularity } from "@/lib/analytics";
+import { 
+  PieChart, Pie, ResponsiveContainer, Tooltip, Cell, Legend,
+  BarChart, Bar, XAxis, YAxis, CartesianGrid, LineChart, Line,
+  LabelList
+} from "recharts";
 
 interface LinkDetails {
   id: string;
@@ -206,7 +211,28 @@ export default function LinkDetailsPage({
         }
         
         const details = await response.json();
-        setLinkDetails(details);
+        console.log("Link details:", details); // 添加日志以确认 API 响应
+        
+        // 将 API 返回的数据映射到组件需要的格式
+        setLinkDetails({
+          id: details.link_id || linkId,
+          name: details.title || "Untitled Link",
+          shortUrl: details.short_url || window.location.hostname + "/" + details.link_id,
+          originalUrl: details.original_url || "",
+          creator: details.created_by || "Unknown",
+          createdAt: details.created_at || new Date().toISOString(),
+          visits: details.visits || 0,
+          visitChange: details.visit_change || 0,
+          uniqueVisitors: details.unique_visitors || 0,
+          uniqueVisitorsChange: details.unique_visitors_change || 0,
+          avgTime: formatTime(details.avg_time_spent || 0),
+          avgTimeChange: details.avg_time_change || 0,
+          conversionRate: details.conversion_rate || 0,
+          conversionChange: details.conversion_change || 0,
+          status: details.is_active ? "active" : "inactive",
+          tags: details.tags || [],
+        });
+        
         setLoading(false);
         
         // 获取分析数据
@@ -689,77 +715,78 @@ export default function LinkDetailsPage({
                     <h3 className="mb-4 font-medium text-md text-foreground">
                       Device Types
                     </h3>
-                    <div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-4">
+                    
-                      <div className="flex flex-col items-center">
+                    {/* 饼图显示 */}
-                        <div className="mb-2 text-sm font-medium text-text-secondary">
+                    <div className="flex flex-col items-center">
-                          Mobile
+                      <div className="w-full h-64">
-                        </div>
+                        <ResponsiveContainer width="100%" height="100%">
-                        <div className="text-2xl font-bold text-foreground">
+                          <PieChart>
-                          {overviewData.deviceTypes.mobile}
+                            <Pie
-                        </div>
+                              data={[
-                        <div className="mt-1 text-xs text-text-secondary">
+                                { name: 'Mobile', value: overviewData.deviceTypes.mobile, fill: "#3498db" },
-                          {overviewData.totalVisits
+                                { name: 'Desktop', value: overviewData.deviceTypes.desktop, fill: "#2ecc71" },
-                            ? Math.round(
+                                { name: 'Tablet', value: overviewData.deviceTypes.tablet, fill: "#f39c12" },
-                                (overviewData.deviceTypes.mobile /
+                                { name: 'Other', value: overviewData.deviceTypes.other, fill: "#e74c3c" }
-                                  overviewData.totalVisits) *
+                              ].filter(item => item.value > 0)}
-                                  100
+                              cx="50%"
-                              )
+                              cy="50%"
-                            : 0}
+                              innerRadius={60}
-                          %
+                              outerRadius={90}
-                        </div>
+                              fill="#8884d8"
+                              dataKey="value"
+                              nameKey="name"
+                              label={({ name, percent }) => `${name}: ${(percent * 100).toFixed(0)}%`}
+                              labelLine={true}
+                            >
+                              {/* 为每种设备类型设置不同的颜色 */}
+                              {[
+                                { key: "mobile", name: 'Mobile', value: overviewData.deviceTypes.mobile, fill: "#3498db" },
+                                { key: "desktop", name: 'Desktop', value: overviewData.deviceTypes.desktop, fill: "#2ecc71" },
+                                { key: "tablet", name: 'Tablet', value: overviewData.deviceTypes.tablet, fill: "#f39c12" },
+                                { key: "other", name: 'Other', value: overviewData.deviceTypes.other, fill: "#e74c3c" }
+                              ]
+                                .filter(item => item.value > 0)
+                                .map(item => (
+                                  <Cell key={item.key} fill={item.fill} />
+                                ))
+                              }
+                            </Pie>
+                            <Tooltip 
+                              formatter={(value) => [`${value} 访问`, '数量']} 
+                              separator=": "
+                            />
+                            <Legend />
+                          </PieChart>
+                        </ResponsiveContainer>
                       </div>
-                      <div className="flex flex-col items-center">
+                      <div className="grid grid-cols-4 gap-8 mt-4 text-center">
-                        <div className="mb-2 text-sm font-medium text-text-secondary">
+                        <div>
-                          Desktop
+                          <div className="text-sm font-medium text-text-secondary">Mobile</div>
+                          <div className="text-lg font-bold text-foreground">{overviewData.deviceTypes.mobile}</div>
+                          <div className="text-xs text-text-secondary">
+                            {overviewData.totalVisits ? Math.round((overviewData.deviceTypes.mobile / overviewData.totalVisits) * 100) : 0}%
+                          </div>
                         </div>
-                        <div className="text-2xl font-bold text-foreground">
+                        <div>
-                          {overviewData.deviceTypes.desktop}
+                          <div className="text-sm font-medium text-text-secondary">Desktop</div>
+                          <div className="text-lg font-bold text-foreground">{overviewData.deviceTypes.desktop}</div>
+                          <div className="text-xs text-text-secondary">
+                            {overviewData.totalVisits ? Math.round((overviewData.deviceTypes.desktop / overviewData.totalVisits) * 100) : 0}%
+                          </div>
                         </div>
-                        <div className="mt-1 text-xs text-text-secondary">
+                        <div>
-                          {overviewData.totalVisits
+                          <div className="text-sm font-medium text-text-secondary">Tablet</div>
-                            ? Math.round(
+                          <div className="text-lg font-bold text-foreground">{overviewData.deviceTypes.tablet}</div>
-                                (overviewData.deviceTypes.desktop /
+                          <div className="text-xs text-text-secondary">
-                                  overviewData.totalVisits) *
+                            {overviewData.totalVisits ? Math.round((overviewData.deviceTypes.tablet / overviewData.totalVisits) * 100) : 0}%
-                                  100
+                          </div>
-                              )
-                            : 0}
-                          %
                         </div>
-                      </div>
+                        <div>
-                      <div className="flex flex-col items-center">
+                          <div className="text-sm font-medium text-text-secondary">Other</div>
-                        <div className="mb-2 text-sm font-medium text-text-secondary">
+                          <div className="text-lg font-bold text-foreground">{overviewData.deviceTypes.other}</div>
-                          Tablet
+                          <div className="text-xs text-text-secondary">
-                        </div>
+                            {overviewData.totalVisits ? Math.round((overviewData.deviceTypes.other / overviewData.totalVisits) * 100) : 0}%
-                        <div className="text-2xl font-bold text-foreground">
+                          </div>
-                          {overviewData.deviceTypes.tablet}
-                        </div>
-                        <div className="mt-1 text-xs text-text-secondary">
-                          {overviewData.totalVisits
-                            ? Math.round(
-                                (overviewData.deviceTypes.tablet /
-                                  overviewData.totalVisits) *
-                                  100
-                              )
-                            : 0}
-                          %
-                        </div>
-                      </div>
-                      <div className="flex flex-col items-center">
-                        <div className="mb-2 text-sm font-medium text-text-secondary">
-                          Other
-                        </div>
-                        <div className="text-2xl font-bold text-foreground">
-                          {overviewData.deviceTypes.other}
-                        </div>
-                        <div className="mt-1 text-xs text-text-secondary">
-                          {overviewData.totalVisits
-                            ? Math.round(
-                                (overviewData.deviceTypes.other /
-                                  overviewData.totalVisits) *
-                                  100
-                              )
-                            : 0}
-                          %
                         </div>
                       </div>
                     </div>
@@ -781,26 +808,41 @@ export default function LinkDetailsPage({
                       </div>
                     </div>
 
-                    <div className="relative pt-2">
+                    <div className="h-80">
-                      {funnelData.steps.map((step) => (
+                      <ResponsiveContainer width="100%" height="100%">
-                        <div key={step.name} className="mb-4">
+                        <BarChart
-                          <div className="flex justify-between items-center mb-1">
+                          layout="vertical"
-                            <span className="text-sm font-medium">
+                          data={funnelData.steps}
-                              {step.name}
+                          margin={{ top: 20, right: 30, left: 40, bottom: 5 }}
-                            </span>
+                        >
-                            <span className="text-sm text-text-secondary">
+                          <CartesianGrid strokeDasharray="3 3" />
-                              {step.value} ({(step.percent || 0).toFixed(1)}
+                          <XAxis type="number" />
-                              %)
+                          <YAxis 
-                            </span>
+                            dataKey="name" 
-                          </div>
+                            type="category" 
-                          <div className="w-full bg-card-border rounded-full h-2.5">
+                            width={100}
-                            <div
+                            tick={{ fontSize: 14 }}
-                              className="bg-accent-blue h-2.5 rounded-full"
+                          />
-                              style={{ width: `${step.percent || 0}%` }}
+                          <Tooltip
-                            ></div>
+                            formatter={(value: number, name: string, props: any) => [
-                          </div>
+                              `${value} (${props.payload.percent.toFixed(1)}%)`,
-                        </div>
+                              "Value"
-                      ))}
+                            ]}
+                          />
+                          <Bar 
+                            dataKey="value" 
+                            fill="#3498db" 
+                            radius={[0, 4, 4, 0]}
+                          >
+                            <LabelList 
+                              dataKey="percent" 
+                              position="right" 
+                              formatter={(value: number) => `${value.toFixed(1)}%`} 
+                              style={{ fill: "#666", fontSize: 12 }}
+                            />
+                          </Bar>
+                        </BarChart>
+                      </ResponsiveContainer>
                     </div>
                   </div>
                 )}
@@ -820,9 +862,9 @@ export default function LinkDetailsPage({
                               onClick={() =>
                                 updateTimeGranularity(granularity)
                               }
-                              className={`px-3 py-1 text-xs rounded-md ${
+                              className={`px-4 py-2 text-sm font-medium rounded-md ${
                                 timeGranularity === granularity
-                                  ? "bg-accent-blue text-white"
+                                  ? "bg-accent-blue text-black shadow-sm"
                                   : "bg-card-bg text-text-secondary border border-card-border hover:bg-gray-100"
                               }`}
                             >
@@ -850,38 +892,51 @@ export default function LinkDetailsPage({
                         </div>
                       </div>
 
-                      {/* 简单趋势表格 */}
+                      {/* 图表展示访问趋势 */}
-                      <div className="overflow-x-auto">
+                      <div className="h-80">
-                        <table className="min-w-full divide-y divide-card-border">
+                        <ResponsiveContainer width="100%" height="100%">
-                          <thead>
+                          <LineChart
-                            <tr>
+                            data={trendsData.trends}
-                              <th className="px-4 py-3 text-xs font-medium tracking-wider text-left uppercase text-text-secondary">
+                            margin={{ top: 20, right: 30, left: 20, bottom: 10 }}
-                                Time
+                          >
-                              </th>
+                            <CartesianGrid strokeDasharray="3 3" />
-                              <th className="px-4 py-3 text-xs font-medium tracking-wider text-left uppercase text-text-secondary">
+                            <XAxis 
-                                Visits
+                              dataKey="timestamp" 
-                              </th>
+                              tick={{ fontSize: 12 }}
-                              <th className="px-4 py-3 text-xs font-medium tracking-wider text-left uppercase text-text-secondary">
+                              interval="preserveStartEnd"
-                                Unique Visitors
+                            />
-                              </th>
+                            <YAxis 
-                            </tr>
+                              tickFormatter={(value: number) => value.toLocaleString()}
-                          </thead>
+                            />
-                          <tbody className="divide-y bg-card-bg divide-card-border">
+                            <Tooltip 
-                            {trendsData.trends.map((trend, i) => (
+                              formatter={(value: number, name: string) => [
-                              <tr key={i}>
+                                value.toLocaleString(),
-                                <td className="px-4 py-2 text-sm whitespace-nowrap text-foreground">
+                                name === "visits" ? "访问量" : "唯一访客"
-                                  {trend.timestamp}
+                              ]}
-                                </td>
+                            />
-                                <td className="px-4 py-2 text-sm whitespace-nowrap text-foreground">
+                            <Legend 
-                                  {trend.visits}
+                              verticalAlign="top" 
-                                </td>
+                              height={36}
-                                <td className="px-4 py-2 text-sm whitespace-nowrap text-foreground">
+                              formatter={(value: string) => value === "visits" ? "访问量" : "唯一访客"}
-                                  {trend.uniqueVisitors}
+                            />
-                                </td>
+                            <Line
-                              </tr>
+                              type="monotone"
-                            ))}
+                              dataKey="visits"
-                          </tbody>
+                              stroke="#3498db"
-                        </table>
+                              strokeWidth={2}
+                              activeDot={{ r: 6 }}
+                              name="visits"
+                            />
+                            <Line
+                              type="monotone"
+                              dataKey="uniqueVisitors"
+                              stroke="#2ecc71"
+                              strokeWidth={2}
+                              dot={{ r: 4 }}
+                              name="uniqueVisitors"
+                            />
+                          </LineChart>
+                        </ResponsiveContainer>
                       </div>
                     </div>
                   </div>
@@ -895,6 +950,63 @@ export default function LinkDetailsPage({
                   <h3 className="mb-4 font-medium text-md text-foreground">
                     Link Performance
                   </h3>
+                  
+                  {/* 添加图表展示 */}
+                  <div className="mb-8">
+                    <div className="flex flex-col md:flex-row gap-6">
+                      {/* 柱状图展示总点击量和独立访客 */}
+                      <div className="h-80 md:w-1/2">
+                        <ResponsiveContainer width="100%" height="100%">
+                          <BarChart
+                            data={[
+                              { name: 'Total Clicks', value: performanceData.totalClicks },
+                              { name: 'Unique Visitors', value: performanceData.uniqueVisitors },
+                              { name: 'Active Days', value: performanceData.activeDays },
+                              { name: 'Unique Referrers', value: performanceData.uniqueReferrers }
+                            ]}
+                            margin={{ top: 20, right: 30, left: 20, bottom: 5 }}
+                          >
+                            <CartesianGrid strokeDasharray="3 3" />
+                            <XAxis dataKey="name" />
+                            <YAxis />
+                            <Tooltip formatter={(value: number) => [value.toLocaleString(), 'Count']} />
+                            <Legend />
+                            <Bar dataKey="value" name="Value" fill="#3498db" radius={[4, 4, 0, 0]}>
+                              <LabelList dataKey="value" position="top" formatter={(value: number) => value.toLocaleString()} />
+                            </Bar>
+                          </BarChart>
+                        </ResponsiveContainer>
+                      </div>
+                      
+                      {/* 饼图展示跳出率、转化率等比例数据 */}
+                      <div className="h-80 md:w-1/2">
+                        <ResponsiveContainer width="100%" height="100%">
+                          <PieChart>
+                            <Pie
+                              data={[
+                                { name: 'Bounce Rate', value: performanceData.bounceRate, fill: '#e74c3c' },
+                                { name: 'Conversion Rate', value: performanceData.conversionRate, fill: '#2ecc71' },
+                                { name: 'Non-converting, Non-bounce Visits', 
+                                  value: 100 - performanceData.bounceRate - performanceData.conversionRate,
+                                  fill: '#3498db' }
+                              ]}
+                              cx="50%"
+                              cy="50%"
+                              innerRadius={60}
+                              outerRadius={90}
+                              dataKey="value"
+                              nameKey="name"
+                              label={({name, value}: {name: string, value: number}) => `${name}: ${value}%`}
+                              labelLine={true}
+                            />
+                            <Tooltip formatter={(value: number) => [`${value}%`, 'Percentage']} />
+                            <Legend />
+                          </PieChart>
+                        </ResponsiveContainer>
+                      </div>
+                    </div>
+                  </div>
+                  
                   <div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-4">
                     <div className="flex flex-col">
                       <div className="mb-1 text-sm font-medium text-text-secondary">
@@ -975,6 +1087,89 @@ export default function LinkDetailsPage({
                   <h3 className="mb-4 font-medium text-md text-foreground">
                     Popular Referrers
                   </h3>
+                  
+                  {/* 添加图表展示 */}
+                  <div className="mb-8">
+                    <div className="flex flex-col md:flex-row gap-4">
+                      {/* 条形图展示访问量和独立访客 */}
+                      <div className="h-80 md:w-2/3">
+                        <ResponsiveContainer width="100%" height="100%">
+                          <BarChart
+                            layout="vertical"
+                            data={referrersData.referrers.slice(0, 10)} // 只显示前10个引荐来源
+                            margin={{ top: 20, right: 30, left: 120, bottom: 5 }}
+                          >
+                            <CartesianGrid strokeDasharray="3 3" horizontal={false} />
+                            <XAxis type="number" />
+                            <YAxis 
+                              dataKey="source" 
+                              type="category" 
+                              tick={{ fontSize: 12 }}
+                              width={120}
+                            />
+                            <Tooltip
+                              formatter={(value: number, name: string) => [
+                                value,
+                                name === "visitCount" ? "访问量" : "独立访客"
+                              ]}
+                            />
+                            <Legend verticalAlign="top" height={36} />
+                            <Bar 
+                              name="访问量"
+                              dataKey="visitCount" 
+                              fill="#3498db" 
+                              radius={[0, 4, 4, 0]}
+                            >
+                              <LabelList 
+                                dataKey="percent" 
+                                position="right" 
+                                formatter={(value: number) => `${value}%`} 
+                                style={{ fill: "#666", fontSize: 12 }}
+                              />
+                            </Bar>
+                            <Bar 
+                              name="独立访客"
+                              dataKey="uniqueVisitors" 
+                              fill="#2ecc71" 
+                              radius={[0, 4, 4, 0]}
+                            />
+                          </BarChart>
+                        </ResponsiveContainer>
+                      </div>
+                      
+                      {/* 饼图展示来源占比 */}
+                      <div className="h-80 md:w-1/3">
+                        <ResponsiveContainer width="100%" height="100%">
+                          <PieChart>
+                            <Pie
+                              data={referrersData.referrers.slice(0, 6)} // 只显示前6个引荐来源
+                              dataKey="visitCount"
+                              nameKey="source"
+                              cx="50%"
+                              cy="50%"
+                              innerRadius={60}
+                              outerRadius={90}
+                              paddingAngle={2}
+                              fill="#8884d8"
+                              label={({source, percent}: {source: string, percent: number}) => 
+                                `${source.substring(0, 10)}...: ${(percent * 100).toFixed(0)}%`
+                              }
+                              labelLine={false}
+                            >
+                              {referrersData.referrers.slice(0, 6).map((entry, index) => (
+                                <Cell key={`cell-${index}`} fill={['#3498db', '#2ecc71', '#9b59b6', '#f39c12', '#e74c3c', '#1abc9c'][index % 6]} />
+                              ))}
+                            </Pie>
+                            <Tooltip />
+                            <Legend 
+                              formatter={(value: string) => value.length > 20 ? value.substring(0, 20) + '...' : value}
+                            />
+                          </PieChart>
+                        </ResponsiveContainer>
+                      </div>
+                    </div>
+                  </div>
+                  
                   <div className="overflow-x-auto">
                     <table className="min-w-full divide-y divide-card-border">
                       <thead>
@@ -1031,6 +1226,36 @@ export default function LinkDetailsPage({
                   <h3 className="mb-4 font-medium text-md text-foreground">
                     Device Types
                   </h3>
+                  
+                  {/* 添加设备类型饼图 */}
+                  <div className="mb-8">
+                    <div className="h-80">
+                      <ResponsiveContainer width="100%" height="100%">
+                        <PieChart>
+                          <Pie
+                            data={deviceData.deviceTypes}
+                            dataKey="count"
+                            nameKey="name"
+                            cx="50%"
+                            cy="50%"
+                            innerRadius={60}
+                            outerRadius={90}
+                            fill="#8884d8"
+                            label={({name, percent}: {name: string, percent: number}) => 
+                              `${name}: ${(percent * 100).toFixed(0)}%`
+                            }
+                          >
+                            {deviceData.deviceTypes.map((entry, index) => (
+                              <Cell key={`device-type-cell-${index}`} fill={['#3498db', '#2ecc71', '#9b59b6', '#f39c12', '#e74c3c'][index % 5]} />
+                            ))}
+                          </Pie>
+                          <Tooltip formatter={(value: number) => [`${value} 次访问`, '访问量']} />
+                          <Legend />
+                        </PieChart>
+                      </ResponsiveContainer>
+                    </div>
+                  </div>
+                  
                   <div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
                     {deviceData.deviceTypes.map(
                       (device: DeviceItem, i: number) => (
@@ -1054,6 +1279,44 @@ export default function LinkDetailsPage({
                   <h3 className="mb-4 font-medium text-md text-foreground">
                     Device Brands
                   </h3>
+                  
+                  {/* 添加设备品牌横向条形图 */}
+                  <div className="mb-8">
+                    <div className="h-80">
+                      <ResponsiveContainer width="100%" height="100%">
+                        <BarChart
+                          layout="vertical"
+                          data={deviceData.deviceBrands.slice(0, 10)}
+                          margin={{ top: 5, right: 30, left: 100, bottom: 5 }}
+                        >
+                          <CartesianGrid strokeDasharray="3 3" horizontal={false} />
+                          <XAxis type="number" />
+                          <YAxis 
+                            dataKey="name" 
+                            type="category" 
+                            tick={{ fontSize: 12 }}
+                            width={100}
+                          />
+                          <Tooltip 
+                            formatter={(value: number) => [`${value} 次访问`, '访问量']}
+                          />
+                          <Bar 
+                            dataKey="count" 
+                            name="访问量" 
+                            fill="#3498db"
+                            radius={[0, 4, 4, 0]}
+                          >
+                            <LabelList 
+                              dataKey="percent" 
+                              position="right" 
+                              formatter={(value: number) => `${value}%`}
+                            />
+                          </Bar>
+                        </BarChart>
+                      </ResponsiveContainer>
+                    </div>
+                  </div>
+                  
                   <div className="overflow-x-auto">
                     <table className="min-w-full divide-y divide-card-border">
                       <thead>
@@ -1098,6 +1361,81 @@ export default function LinkDetailsPage({
                   <h3 className="mb-4 font-medium text-md text-foreground">
                     Platform Distribution
                   </h3>
+                  
+                  {/* 添加图表展示 */}
+                  <div className="mb-8">
+                    <div className="flex flex-col md:flex-row gap-6">
+                      {/* 操作系统分布饼图 */}
+                      <div className="h-80 md:w-1/2">
+                        <h4 className="mb-3 text-sm font-medium text-text-secondary text-center">
+                          Operating Systems
+                        </h4>
+                        <ResponsiveContainer width="100%" height="90%">
+                          <PieChart>
+                            <Pie
+                              data={platformData.platforms.slice(0, 6)}
+                              dataKey="count"
+                              nameKey="name"
+                              cx="50%"
+                              cy="50%"
+                              innerRadius={60}
+                              outerRadius={90}
+                              paddingAngle={2}
+                              fill="#8884d8"
+                              label={({name, percent}: {name: string, percent: number}) => 
+                                `${name}: ${(percent * 100).toFixed(0)}%`
+                              }
+                              labelLine={true}
+                            >
+                              {platformData.platforms.slice(0, 6).map((entry, index) => (
+                                <Cell key={`os-cell-${index}`} fill={['#3498db', '#2ecc71', '#9b59b6', '#f39c12', '#e74c3c', '#1abc9c'][index % 6]} />
+                              ))}
+                            </Pie>
+                            <Tooltip formatter={(value: number, name: string) => [`${value} 次访问`, name]} />
+                            <Legend formatter={(value: string) => value.length > 25 ? value.substring(0, 25) + '...' : value} />
+                          </PieChart>
+                        </ResponsiveContainer>
+                      </div>
+                      
+                      {/* 浏览器分布条形图 */}
+                      <div className="h-80 md:w-1/2">
+                        <h4 className="mb-3 text-sm font-medium text-text-secondary text-center">
+                          Browsers
+                        </h4>
+                        <ResponsiveContainer width="100%" height="90%">
+                          <BarChart
+                            data={platformData.browsers.slice(0, 8)}
+                            layout="vertical"
+                            margin={{ top: 5, right: 30, left: 80, bottom: 5 }}
+                          >
+                            <CartesianGrid strokeDasharray="3 3" horizontal={false} />
+                            <XAxis type="number" />
+                            <YAxis 
+                              type="category" 
+                              dataKey="name" 
+                              tick={{ fontSize: 12 }}
+                              width={80}
+                            />
+                            <Tooltip formatter={(value: number) => [`${value} 次访问`, '数量']} />
+                            <Legend />
+                            <Bar 
+                              dataKey="count" 
+                              name="访问量" 
+                              fill="#3498db" 
+                              radius={[0, 4, 4, 0]}
+                            >
+                              <LabelList 
+                                dataKey="percent" 
+                                position="right" 
+                                formatter={(value: number) => `${value}%`} 
+                              />
+                            </Bar>
+                          </BarChart>
+                        </ResponsiveContainer>
+                      </div>
+                    </div>
+                  </div>
+                  
                   <div className="grid grid-cols-1 gap-6 sm:grid-cols-2">
                     <div>
                       <h4 className="mb-3 text-sm font-medium text-text-secondary">
@@ -1225,6 +1563,42 @@ export default function LinkDetailsPage({
                   <h3 className="mb-4 font-medium text-md text-foreground">
                     Scan Locations
                   </h3>
+                  
+                  {/* 添加扫描位置饼图 */}
+                  <div className="mb-8">
+                    <div className="h-80">
+                      <ResponsiveContainer width="100%" height="100%">
+                        <PieChart>
+                          <Pie
+                            data={qrCodeData.locations.slice(0, 8)}
+                            dataKey="scanCount"
+                            nameKey="city"
+                            cx="50%"
+                            cy="50%"
+                            innerRadius={60}
+                            outerRadius={90}
+                            paddingAngle={2}
+                            fill="#8884d8"
+                            label={({city, country, percent}: {city: string, country: string, percent: number}) => 
+                              `${city}: ${(percent * 100).toFixed(0)}%`
+                            }
+                          >
+                            {qrCodeData.locations.slice(0, 8).map((entry, index) => (
+                              <Cell key={`location-cell-${index}`} fill={['#3498db', '#2ecc71', '#9b59b6', '#f39c12', '#e74c3c', '#1abc9c', '#34495e', '#16a085'][index % 8]} />
+                            ))}
+                          </Pie>
+                          <Tooltip formatter={(value: number, name: string) => [`${value} 次扫描`, name]} />
+                          <Legend 
+                            formatter={(value: string) => {
+                              const location = qrCodeData.locations.find(loc => loc.city === value);
+                              return location ? `${location.city}, ${location.country}` : value;
+                            }}
+                          />
+                        </PieChart>
+                      </ResponsiveContainer>
+                    </div>
+                  </div>
+                  
                   <div className="overflow-x-auto">
                     <table className="min-w-full divide-y divide-card-border">
                       <thead>
@@ -1263,40 +1637,46 @@ export default function LinkDetailsPage({
                   <h3 className="mb-4 font-medium text-md text-foreground">
                     Scan Time Distribution
                   </h3>
-                  <div className="grid grid-cols-1 gap-6">
+                  
-                    <div className="overflow-x-auto">
+                  {/* 添加扫描时间分布柱状图 */}
-                      <table className="min-w-full divide-y divide-card-border">
+                  <div className="mb-8">
-                        <thead>
+                    <div className="h-80">
-                          <tr>
+                      <ResponsiveContainer width="100%" height="100%">
-                            <th className="px-4 py-3 text-xs font-medium tracking-wider text-left uppercase text-text-secondary">
+                        <BarChart
-                              Hour
+                          data={qrCodeData.hourlyDistribution}
-                            </th>
+                          margin={{ top: 20, right: 30, left: 20, bottom: 5 }}
-                            <th className="px-4 py-3 text-xs font-medium tracking-wider text-left uppercase text-text-secondary">
+                        >
-                              Scans
+                          <CartesianGrid strokeDasharray="3 3" />
-                            </th>
+                          <XAxis 
-                            <th className="px-4 py-3 text-xs font-medium tracking-wider text-left uppercase text-text-secondary">
+                            dataKey="hour" 
-                              Percentage
+                            label={{ value: 'Hour of Day', position: 'insideBottom', offset: -5 }}
-                            </th>
+                          />
-                          </tr>
+                          <YAxis 
-                        </thead>
+                            label={{ value: 'Scan Count', angle: -90, position: 'insideLeft' }}
-                        <tbody className="divide-y bg-card-bg divide-card-border">
+                          />
-                          {qrCodeData.hourlyDistribution.map((hour) => (
+                          <Tooltip 
-                            <tr key={hour.hour}>
+                            formatter={(value: number) => [`${value} 次扫描`, '扫描次数']}
-                              <td className="px-4 py-2 text-sm whitespace-nowrap text-foreground">
+                            labelFormatter={(hour) => `${hour}:00 - ${hour}:59`}
-                                {hour.hour}:00 - {hour.hour}:59
+                          />
-                              </td>
+                          <Bar 
-                              <td className="px-4 py-2 text-sm whitespace-nowrap text-foreground">
+                            dataKey="scanCount" 
-                                {hour.scanCount}
+                            name="扫描次数" 
-                              </td>
+                            fill="#3498db"
-                              <td className="px-4 py-2 text-sm whitespace-nowrap text-foreground">
+                            radius={[4, 4, 0, 0]}
-                                {hour.percent.toFixed(1)}%
+                          >
-                              </td>
+                            <LabelList 
-                            </tr>
+                              dataKey="percent" 
-                          ))}
+                              position="top" 
-                        </tbody>
+                              formatter={(value: number) => `${value.toFixed(1)}%`}
-                      </table>
+                            />
+                          </Bar>
+                        </BarChart>
+                      </ResponsiveContainer>
                     </div>
                   </div>
+                  
+                  <div className="grid grid-cols-1 gap-6">
+                  </div>
                 </div>
               </div>
             )}

app/links/page.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useState, useEffect } from 'react';
+import { useState, useEffect, useCallback, useRef } from 'react';
 import CreateLinkModal from '../components/ui/CreateLinkModal';
 import { Link, StatsOverview, Tag } from '../api/types';
 
@@ -48,6 +48,13 @@ export default function LinksPage() {
   const [isLoading, setIsLoading] = useState(true);
   const [error, setError] = useState<string | null>(null);
   
+  // 无限加载相关状态
+  const [page, setPage] = useState(1);
+  const [hasMore, setHasMore] = useState(true);
+  const [isLoadingMore, setIsLoadingMore] = useState(false);
+  const observer = useRef<IntersectionObserver | null>(null);
+  const lastLinkElementRef = useRef<HTMLTableRowElement | null>(null);
+  
   // 映射API数据到UI所需格式的函数
   const mapApiLinkToUiLink = (apiLink: Link): UILink => {
     // 生成短URL显示 - 因为数据库中没有short_url字段
@@ -91,49 +98,116 @@ export default function LinksPage() {
   };
   
   // 获取链接数据
-  useEffect(() => {
+  const fetchLinks = useCallback(async (pageNum: number, isInitialLoad: boolean = false) => {
-    const fetchLinks = async () => {
+    try {
-      try {
+      if (isInitialLoad) {
         setIsLoading(true);
-        setError(null);
+      } else {
+        setIsLoadingMore(true);
+      }
+      setError(null);
       
-        // 获取链接列表
+      // 获取链接列表
-        const linksResponse = await fetch('/api/links');
+      const linksResponse = await fetch(`/api/links?page=${pageNum}&limit=20${searchQuery ? `&search=${encodeURIComponent(searchQuery)}` : ''}`);
-        if (!linksResponse.ok) {
+      if (!linksResponse.ok) {
-          throw new Error(`Failed to fetch links: ${linksResponse.statusText}`);
+        throw new Error(`Failed to fetch links: ${linksResponse.statusText}`);
-        }
+      }
-        const linksData = await linksResponse.json();
+      const linksData = await linksResponse.json();
       
-        // 获取标签列表
+      const uiLinks = linksData.data.map(mapApiLinkToUiLink);
+      
+      if (isInitialLoad) {
+        setLinks(uiLinks);
+      } else {
+        setLinks(prevLinks => [...prevLinks, ...uiLinks]);
+      }
+      
+      // 检查是否还有更多数据可加载
+      const { pagination } = linksData;
+      setHasMore(pagination.page < pagination.totalPages);
+      
+      if (isInitialLoad) {
+        // 只在初始加载时获取标签和统计数据
         const tagsResponse = await fetch('/api/tags');
         if (!tagsResponse.ok) {
           throw new Error(`Failed to fetch tags: ${tagsResponse.statusText}`);
         }
         const tagsData = await tagsResponse.json();
         
-        // 获取统计数据
         const statsResponse = await fetch('/api/stats');
         if (!statsResponse.ok) {
           throw new Error(`Failed to fetch stats: ${statsResponse.statusText}`);
         }
         const statsData = await statsResponse.json();
         
-        // 处理并设置数据
-        const uiLinks = linksData.data.map(mapApiLinkToUiLink);
-        setLinks(uiLinks);
         setAllTags(tagsData);
         setStats(statsData);
+      }
       
-      } catch (err) {
+    } catch (err) {
-        console.error('Data loading failed:', err);
+      console.error('Data loading failed:', err);
-        setError(err instanceof Error ? err.message : 'Unknown error');
+      setError(err instanceof Error ? err.message : 'Unknown error');
-      } finally {
+    } finally {
+      if (isInitialLoad) {
         setIsLoading(false);
+      } else {
+        setIsLoadingMore(false);
+      }
+    }
+  }, [searchQuery]);
+  
+  // 初始加载
+  useEffect(() => {
+    setPage(1);
+    fetchLinks(1, true);
+  }, [fetchLinks]);
+  
+  // 搜索过滤变化时重新加载数据
+  useEffect(() => {
+    // 当搜索关键词变化时，重置页码和链接列表，然后重新获取数据
+    setLinks([]);
+    setPage(1);
+    fetchLinks(1, true);
+  }, [searchQuery, fetchLinks]);
+  
+  // 设置Intersection Observer来检测滚动并加载更多数据
+  useEffect(() => {
+    // 如果正在加载或没有更多数据，则不设置observer
+    if (isLoading || isLoadingMore || !hasMore) return;
+    
+    // 断开之前的observer连接
+    if (observer.current) {
+      observer.current.disconnect();
+    }
+    
+    observer.current = new IntersectionObserver(entries => {
+      if (entries[0].isIntersecting && hasMore) {
+        // 当最后一个元素可见且有更多数据时，加载下一页
+        setPage(prevPage => prevPage + 1);
+      }
+    }, {
+      root: null,
+      rootMargin: '0px',
+      threshold: 0.5
+    });
+    
+    if (lastLinkElementRef.current) {
+      observer.current.observe(lastLinkElementRef.current);
+    }
+    
+    return () => {
+      if (observer.current) {
+        observer.current.disconnect();
       }
     };
+  }, [isLoading, isLoadingMore, hasMore, links]);
   
-    fetchLinks();
+  // 当页码变化时加载更多数据
-  }, []);
+  useEffect(() => {
+    if (page > 1) {
+      fetchLinks(page, false);
+    }
+  }, [page, fetchLinks]);
   
   const filteredLinks = links.filter(link => 
     link.name.toLowerCase().includes(searchQuery.toLowerCase()) ||
@@ -152,14 +226,8 @@ export default function LinksPage() {
       console.log('创建链接:', linkData);
       
       // 刷新链接列表
-      const response = await fetch('/api/links');
+      setPage(1);
-      if (!response.ok) {
+      fetchLinks(1, true);
-        throw new Error(`刷新链接列表失败: ${response.statusText}`);
-      }
-      
-      const newData = await response.json();
-      const uiLinks = newData.data.map(mapApiLinkToUiLink);
-      setLinks(uiLinks);
       
       setShowCreateModal(false);
     } catch (err) {
@@ -309,8 +377,8 @@ export default function LinksPage() {
         {/* Links Table */}
         <div className="overflow-hidden border rounded-lg shadow bg-card-bg border-card-border">
           <div className="overflow-x-auto">
-            <table className="w-full text-sm text-left text-text-secondary">
+            <table className="min-w-full divide-y divide-card-border">
-              <thead className="text-xs uppercase border-b bg-card-bg/60 text-text-secondary border-card-border">
+              <thead className="bg-card-bg-secondary">
                 <tr>
                   <th scope="col" className="px-6 py-3">Link Info</th>
                   <th scope="col" className="px-6 py-3">Visits</th>
@@ -323,28 +391,20 @@ export default function LinksPage() {
                   </th>
                 </tr>
               </thead>
-              <tbody>
+              <tbody className="divide-y divide-card-border">
-                {isLoading && links.length === 0 ? (
+                {filteredLinks.length === 0 ? (
-                  <tr className="border-b bg-card-bg border-card-border">
+                  <tr>
-                    <td colSpan={7} className="px-6 py-4 text-center text-text-secondary">
+                    <td colSpan={7} className="px-6 py-12 text-center text-text-secondary">
-                      <div className="flex items-center justify-center">
+                      No links found. Create one to get started.
-                        <div className="w-6 h-6 border-2 rounded-full border-accent-blue border-t-transparent animate-spin"></div>
-                        <span className="ml-2">Loading...</span>
-                      </div>
-                    </td>
-                  </tr>
-                ) : filteredLinks.length === 0 ? (
-                  <tr className="border-b bg-card-bg border-card-border">
-                    <td colSpan={7} className="px-6 py-4 text-center text-text-secondary">
-                      No links found matching your search criteria
                     </td>
                   </tr>
                 ) : (
-                  filteredLinks.map((link) => (
+                  filteredLinks.map((link, index) => (
                     <tr 
                       key={link.id} 
-                      className="border-b cursor-pointer bg-card-bg border-card-border hover:bg-card-bg/80"
                       onClick={() => handleOpenLinkDetails(link.id)}
+                      className="transition-colors cursor-pointer hover:bg-card-bg-secondary"
+                      ref={index === filteredLinks.length - 1 ? lastLinkElementRef : null}
                     >
                       <td className="px-6 py-4">
                         <div className="font-medium text-foreground">{link.name}</div>
@@ -436,6 +496,21 @@ export default function LinksPage() {
               </tbody>
             </table>
           </div>
+          
+          {/* Loading more indicator */}
+          {isLoadingMore && (
+            <div className="p-4 text-center">
+              <div className="inline-block w-6 h-6 border-2 rounded-full border-accent-blue border-t-transparent animate-spin"></div>
+              <p className="mt-2 text-sm text-text-secondary">Loading more links...</p>
+            </div>
+          )}
+          
+          {/* End of results message */}
+          {!hasMore && links.length > 0 && (
+            <div className="p-4 text-center text-sm text-text-secondary">
+              No more links to load.
+            </div>
+          )}
         </div>
         
         {/* Tags Section */}

app/page.tsx

@@ -1,50 +1,5 @@
-import Link from 'next/link';
+import { redirect } from 'next/navigation';
 
 export default function Home() {
-  return (
+  redirect('/links');
-    <main className="flex min-h-screen flex-col items-center justify-center p-24 relative overflow-hidden">
-      {/* Colorful background elements */}
-      <div className="absolute top-0 left-0 w-full h-full overflow-hidden z-0">
-        <div className="absolute top-10 left-1/4 w-64 h-64 rounded-full bg-accent-blue opacity-10 blur-3xl"></div>
-        <div className="absolute bottom-10 right-1/4 w-96 h-96 rounded-full bg-accent-purple opacity-10 blur-3xl"></div>
-        <div className="absolute top-1/3 right-1/3 w-48 h-48 rounded-full bg-accent-green opacity-10 blur-3xl"></div>
-      </div>
-      
-      <div className="text-center max-w-xl z-10 relative">
-        <div className="flex items-center justify-center mb-6">
-          <div className="h-10 w-10 rounded-lg bg-gradient-blue flex items-center justify-center shadow-lg">
-            <svg className="h-6 w-6 text-white" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-              <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M13 7h8m0 0v8m0-8l-8 8-4-4-6 6" />
-            </svg>
-          </div>
-          <h1 className="text-4xl font-bold ml-3 text-foreground">ShortURL <span className="text-accent-blue">Analytics</span></h1>
-        </div>
-        
-        <p className="text-text-secondary text-xl mb-10">Your complete analytics suite for tracking and optimizing short URL performance</p>
-        
-        <div className="flex flex-col md:flex-row items-center justify-center space-y-4 md:space-y-0 md:space-x-4">
-          <Link 
-            href="/dashboard" 
-            className="bg-gradient-blue hover:opacity-90 text-white font-medium py-2.5 px-6 rounded-md text-lg transition-colors inline-flex items-center shadow-lg"
-          >
-            Go to Dashboard
-            <svg className="ml-2 h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-              <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M14 5l7 7m0 0l-7 7m7-7H3" />
-            </svg>
-          </Link>
-          
-          <Link
-            href="/links"
-            className="bg-card-bg border border-card-border hover:border-accent-purple text-foreground font-medium py-2.5 px-6 rounded-md text-lg transition-all inline-flex items-center"
-          >
-            View Links
-            <svg className="ml-2 h-5 w-5 text-accent-purple" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-              <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M13.828 10.172a4 4 0 00-5.656 0l-4 4a4 4 0 105.656 5.656l1.102-1.101" />
-              <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M14.828 14.828a4 4 0 005.656 0l4-4a4 4 0 00-5.656-5.656l-1.1 1.1" />
-            </svg>
-          </Link>
-        </div>
-      </div>
-    </main>
-  )
 }

scripts/db/sql/clickhouse/modify_device_type.sql

@@ -0,0 +1,122 @@
+-- 修改设备类型字段从枚举类型更改为字符串类型
+-- 先删除依赖于link_events表的物化视图
+DROP TABLE IF EXISTS limq.platform_distribution;
+
+DROP TABLE IF EXISTS limq.link_hourly_patterns;
+
+DROP TABLE IF EXISTS limq.link_daily_stats;
+
+DROP TABLE IF EXISTS limq.team_daily_stats;
+
+DROP TABLE IF EXISTS limq.project_daily_stats;
+
+-- 修改link_events表的device_type字段
+ALTER TABLE
+  limq.link_events
+MODIFY
+  COLUMN device_type String;
+
+-- 重新创建物化视图
+-- 每日链接汇总视图
+CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
+ORDER BY
+  (date, link_id) SETTINGS index_granularity = 8192 AS
+SELECT
+  toDate(event_time) AS date,
+  link_id,
+  count() AS total_clicks,
+  uniqExact(visitor_id) AS unique_visitors,
+  uniqExact(session_id) AS unique_sessions,
+  sum(time_spent_sec) AS total_time_spent,
+  avg(time_spent_sec) AS avg_time_spent,
+  countIf(is_bounce) AS bounce_count,
+  countIf(event_type = 'conversion') AS conversion_count,
+  uniqExact(referrer) AS unique_referrers,
+  countIf(device_type = 'mobile') AS mobile_count,
+  countIf(device_type = 'tablet') AS tablet_count,
+  countIf(device_type = 'desktop') AS desktop_count,
+  countIf(is_qr_scan) AS qr_scan_count,
+  sum(conversion_value) AS total_conversion_value
+FROM
+  limq.link_events
+GROUP BY
+  date,
+  link_id;
+
+-- 每小时访问模式视图
+CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_hourly_patterns ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
+ORDER BY
+  (date, hour, link_id) SETTINGS index_granularity = 8192 AS
+SELECT
+  toDate(event_time) AS date,
+  toHour(event_time) AS hour,
+  link_id,
+  count() AS visits,
+  uniqExact(visitor_id) AS unique_visitors
+FROM
+  limq.link_events
+GROUP BY
+  date,
+  hour,
+  link_id;
+
+-- 平台分布视图
+CREATE MATERIALIZED VIEW IF NOT EXISTS limq.platform_distribution ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
+ORDER BY
+  (date, utm_source, device_type) SETTINGS index_granularity = 8192 AS
+SELECT
+  toDate(event_time) AS date,
+  utm_source,
+  device_type,
+  count() AS visits,
+  uniqExact(visitor_id) AS unique_visitors
+FROM
+  limq.link_events
+WHERE
+  utm_source != ''
+GROUP BY
+  date,
+  utm_source,
+  device_type;
+
+-- 团队每日统计视图
+CREATE MATERIALIZED VIEW IF NOT EXISTS limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
+ORDER BY
+  (date, team_id) SETTINGS index_granularity = 8192 AS
+SELECT
+  toDate(event_time) AS date,
+  l.team_id AS team_id,
+  count() AS total_clicks,
+  uniqExact(e.visitor_id) AS unique_visitors,
+  countIf(e.event_type = 'conversion') AS conversion_count,
+  uniqExact(e.link_id) AS links_used,
+  countIf(e.is_qr_scan) AS qr_scan_count
+FROM
+  limq.link_events e
+  JOIN limq.links l ON e.link_id = l.link_id
+WHERE
+  l.team_id != ''
+GROUP BY
+  date,
+  l.team_id;
+
+-- 项目每日统计视图
+CREATE MATERIALIZED VIEW IF NOT EXISTS limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
+ORDER BY
+  (date, project_id) SETTINGS index_granularity = 8192 AS
+SELECT
+  toDate(event_time) AS date,
+  l.project_id AS project_id,
+  count() AS total_clicks,
+  uniqExact(e.visitor_id) AS unique_visitors,
+  countIf(e.event_type = 'conversion') AS conversion_count,
+  uniqExact(e.link_id) AS links_used,
+  countIf(e.is_qr_scan) AS qr_scan_count
+FROM
+  limq.link_events e
+  JOIN limq.links l ON e.link_id = l.link_id
+WHERE
+  l.project_id != ''
+GROUP BY
+  date,
+  l.project_id;

windmill/README.md

@@ -0,0 +1,73 @@
+
+
+Read file: /Users/liam/code/shorturl-analytics/windmill/sync_shorturl_from_mongo_to_clickhouse.ts
+
+Read file: /Users/liam/code/shorturl-analytics/windmill/sync_shorturl_from_mongo_to_clickhouse.ts
+
+Read file: /Users/liam/code/shorturl-analytics/windmill/sync_shorturl_event_from_mongo.ts
+
+Read file: /Users/liam/code/shorturl-analytics/windmill/sync_shorturl_event_from_mongo.ts
+
+Read file: /Users/liam/code/shorturl-analytics/windmill/sync_shorturl_from_mongo_to_clickhouse.ts
+
+Read file: /Users/liam/code/shorturl-analytics/windmill/sync_shorturl_event_from_mongo.ts
+这两个脚本是使用 Windmill 平台开发的数据同步工具，用于将短链接相关数据从 MongoDB 数据库同步到 ClickHouse 数据库。
+
+## 1. sync_shorturl_from_mongo_to_clickhouse.ts
+
+**功能**: 将 MongoDB 中的短链接数据（short 表）同步到 ClickHouse 的 links 表
+
+**主要特点**:
+- 增量同步: 记录上次同步位置，只处理新增数据
+- 批量处理: 默认每批次处理 100 条记录，可配置
+- 超时控制: 设置最大运行时间（默认 30 分钟）
+- 数据重复检查: 检查 ClickHouse 中是否已存在相同记录
+- 错误处理: 完善的错误处理和日志记录
+
+**数据转换**:
+- 将 MongoDB 中的短链接记录（包含 slug、origin、创建时间等）转换为 ClickHouse 表结构
+- 处理特殊字段如日期时间、标签数组等
+- 转换字段包括: link_id、original_url、created_at、created_by、title、description、tags、is_active、expires_at、team_id、project_id
+
+**执行流程**:
+1. 从 Windmill 变量获取 MongoDB 和 ClickHouse 连接配置
+2. 获取上次同步状态（时间戳和记录ID）
+3. 连接 MongoDB，批量查询符合条件的新记录
+4. 检查这些记录是否已存在于 ClickHouse
+5. 转换数据格式并生成 SQL 插入语句
+6. 执行插入操作并记录结果
+7. 更新同步状态，为下次同步做准备
+
+## 2. sync_shorturl_event_from_mongo.ts
+
+**功能**: 将 MongoDB 中的短链接点击事件数据（trace 表）同步到 ClickHouse 的 link_events 表
+
+**主要特点**:
+- 与第一个脚本类似，但处理的是访问事件数据
+- 默认批量处理规模更大（1000 条/批次）
+- 超时时间更长（60 分钟）
+- 支持完整的事件元数据保存
+
+**数据转换**:
+- 将 MongoDB 中的访问事件记录转换为 ClickHouse 事件表结构
+- 记录的字段更丰富，包括:
+  - link_id: 短链接ID
+  - visitor_id: 访客ID
+  - session_id: 会话ID
+  - event_type: 事件类型（点击、转化等）
+  - 设备信息: ip_address、user_agent、device_type、browser、os
+  - 来源信息: referrer、utm 参数
+  - 行为数据: time_spent_sec、is_bounce、conversion_type 等
+
+**执行流程**:
+与第一个脚本基本相同，但处理的是 trace 表的数据，并且将其转换为 link_events 表所需的格式。
+
+## 两者共同点:
+
+1. **增量同步机制**: 记录同步状态，每次只处理新数据
+2. **容错设计**: 超时控制、错误处理、异常恢复机制
+3. **配置灵活**: 可通过参数控制批量大小、超时时间等
+4. **数据验证**: 确保已同步数据不会重复
+5. **详细日志**: 记录同步过程中的关键事件和状态
+
+这两个脚本共同构成了短链接分析系统的数据管道，实现了从 MongoDB（可能是原始数据存储）到 ClickHouse（分析型数据库）的数据迁移，为短链接分析平台提供数据基础。

windmill/sync_shorturl_event_from_mongo.ts

@@ -40,12 +40,11 @@ interface SyncState {
 }
 
 export async function main(
-  batch_size = 1000, // 减小批处理大小为5
+  batch_size = 1000,
-  initial_sync = false,
+  max_records = 9999999,
-  max_records = 9999999, // 只同步10条记录用于测试
+  timeout_minutes = 60,
-  timeout_minutes = 60, // 减少超时时间为5分钟
+  skip_clickhouse_check = false,
-  skip_clickhouse_check = false, // 是否跳过ClickHouse重复检查
+  force_insert = false
-  force_insert = false // 强制插入所有记录，不检查是否已存在
 ) {
   const logWithTimestamp = (message: string) => {
     const now = new Date();
@@ -125,34 +124,6 @@ export async function main(
 
   console.log(`MongoDB连接URL: ${mongoUrl.replace(/:[^:]*@/, ":****@")}`);
 
-  // 获取上次同步的状态
-  let syncState: SyncState;
-  try {
-    const rawSyncState = await getVariable<string>("f/shorturl_analytics/clickhouse/shorturl_sync_state");
-    try {
-      syncState = JSON.parse(rawSyncState);
-      console.log(`获取同步状态成功: 上次同步时间 ${new Date(syncState.last_sync_time).toISOString()}`);
-    } catch (parseError) {
-      console.error("解析同步状态失败:", parseError);
-      throw parseError;
-    }
-  } catch (_unused_error) {
-    console.log("未找到同步状态，创建初始同步状态");
-    syncState = {
-      last_sync_time: 0,
-      records_synced: 0,
-    };
-  }
-
-  // 如果强制从头开始同步
-  if (initial_sync) {
-    console.log("强制从头开始同步");
-    syncState = {
-      last_sync_time: 0,
-      records_synced: 0,
-    };
-  }
-
   // 连接MongoDB
   const client = new MongoClient();
   try {
@@ -162,43 +133,28 @@ export async function main(
     const db = client.database(mongoConfig.db);
     const traceCollection = db.collection<TraceRecord>("trace");
     
-    // 构建查询条件，只查询新的记录
+    // 构建查询条件，获取所有记录
-    const query: Record<string, unknown> = {};
+    const query: Record<string, unknown> = {
-    
+      type: 1  // 只同步type为1的记录
-    if (syncState.last_sync_time > 0) {
+    };
-      query.createTime = { $gt: syncState.last_sync_time };
-    }
-
-    if (syncState.last_sync_id) {
-      // 如果有上次同步的ID，则从该ID之后开始查询
-      // 注意：这需要MongoDB中createTime相同的记录按_id排序
-      query._id = { $gt: new ObjectId(syncState.last_sync_id) };
-    }
 
     // 计算总记录数
     const totalRecords = await traceCollection.countDocuments(query);
-    console.log(`找到 ${totalRecords} 条新记录需要同步`);
+    console.log(`找到 ${totalRecords} 条记录需要同步`);
 
     // 限制此次处理的记录数量
     const recordsToProcess = Math.min(totalRecords, max_records);
     console.log(`本次将处理 ${recordsToProcess} 条记录`);
 
     if (totalRecords === 0) {
-      console.log("没有新记录需要同步，任务完成");
+      console.log("没有记录需要同步，任务完成");
       return {
         success: true,
         records_synced: 0,
-        total_synced: syncState.records_synced,
+        message: "没有记录需要同步"
-        message: "没有新记录需要同步"
       };
     }
 
-    // 分批处理记录
-    let processedRecords = 0;
-    let lastId: string | undefined;
-    let lastCreateTime = syncState.last_sync_time;
-    let totalBatchRecords = 0;
-
     // 检查ClickHouse连接状态
     const checkClickHouseConnection = async (): Promise<boolean> => {
       if (skip_clickhouse_check) {
@@ -358,10 +314,6 @@ export async function main(
       
       if (newRecords.length === 0) {
         logWithTimestamp("所有记录都已存在，跳过处理");
-        // 更新同步状态，即使没有新增记录
-        const lastRecord = records[records.length - 1];
-        lastId = lastRecord._id.toString();
-        lastCreateTime = lastRecord.createTime;
         return 0;
       }
       
@@ -385,7 +337,7 @@ export async function main(
           utm_medium: "",
           utm_campaign: "",
           user_agent: record.browser + " " + record.browserVersion,
-          device_type: record.platform === "mobile" ? 1 : (record.platform === "tablet" ? 2 : 3),
+          device_type: record.platform || "unknown",
           browser: record.browser || "",
           os: record.platformOS || "",
           time_spent_sec: 0,
@@ -398,25 +350,27 @@ export async function main(
         };
       });
 
-      // 更新同步状态（使用原始records的最后一条，以确保进度正确）
-      const lastRecord = records[records.length - 1];
-      lastId = lastRecord._id.toString();
-      lastCreateTime = lastRecord.createTime;
-      logWithTimestamp(`更新同步位置到: ID=${lastId}, 时间=${new Date(lastCreateTime).toISOString()}`);
-
       // 生成ClickHouse插入SQL
       const insertSQL = `
         INSERT INTO ${clickhouseConfig.clickhouse_database}.link_events 
         (link_id, channel_id, visitor_id, session_id, event_type, ip_address, country, city, 
          referrer, utm_source, utm_medium, utm_campaign, user_agent, device_type, browser, os, 
          time_spent_sec, is_bounce, is_qr_scan, qr_code_id, conversion_type, conversion_value, custom_data)
-        VALUES ${clickhouseData.map(record => 
+        VALUES ${clickhouseData.map(record => {
-          `('${record.link_id}', '${record.channel_id.replace(/'/g, "''")}', '${record.visitor_id}', '${record.session_id}', 
+          // 确保所有字符串值都是字符串类型，并安全处理替换
-            ${record.event_type}, '${record.ip_address}', '', '', 
+          const safeReplace = (val: any): string => {
-            '${record.referrer.replace(/'/g, "''")}', '', '', '', '${record.user_agent.replace(/'/g, "''")}', ${record.device_type}, 
+            // 确保值是字符串，如果是null或undefined则使用空字符串
-            '${record.browser.replace(/'/g, "''")}', '${record.os.replace(/'/g, "''")}', 
+            const str = val === null || val === undefined ? "" : String(val);
-            0, true, false, '', 1, 0, '${record.custom_data}')`
+            // 安全替换单引号
-        ).join(", ")}
+            return str.replace(/'/g, "''");
+          };
+          
+          return `('${record.link_id}', '${safeReplace(record.channel_id)}', '${record.visitor_id}', '${record.session_id}', 
+            ${record.event_type}, '${safeReplace(record.ip_address)}', '', '', 
+            '${safeReplace(record.referrer)}', '', '', '', '${safeReplace(record.user_agent)}', '${safeReplace(record.device_type)}', 
+            '${safeReplace(record.browser)}', '${safeReplace(record.os)}', 
+            0, true, false, '', 1, 0, '${safeReplace(record.custom_data)}')`;
+        }).join(", ")}
       `;
 
       if (insertSQL.length === 0) {
@@ -453,6 +407,9 @@ export async function main(
     };
 
     // 批量处理记录
+    let processedRecords = 0;
+    let totalBatchRecords = 0;
+
     for (let page = 0; processedRecords < recordsToProcess; page++) {
       // 检查超时
       if (checkTimeout()) {
@@ -464,17 +421,22 @@ export async function main(
       logWithTimestamp(`开始处理第 ${page+1} 批次，已完成 ${processedRecords}/${recordsToProcess} 条记录 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
       
       logWithTimestamp(`正在从MongoDB获取第 ${page+1} 批次数据...`);
-      const records = await traceCollection.find(query)
+      const records = await traceCollection.find(
-        .sort({ createTime: 1, _id: 1 })
+        query,
-        .skip(page * batch_size)
+        {
-        .limit(batch_size)
+          allowDiskUse: true,
-        .toArray();
+          sort: { createTime: 1 },
+          skip: page * batch_size,
+          limit: batch_size
+        }
+      ).toArray();
 
       if (records.length === 0) {
-        logWithTimestamp(`第 ${page+1} 批次没有找到数据，结束处理`);
+        logWithTimestamp("没有找到更多数据，同步结束");
         break;
       }
       
+      // 找到数据，开始处理
       logWithTimestamp(`获取到 ${records.length} 条记录，开始处理...`);
       // 输出当前批次的部分数据信息
       if (records.length > 0) {
@@ -485,35 +447,16 @@ export async function main(
       }
 
       const batchSize = await processRecords(records);
-      processedRecords += records.length; // 总是增加处理的记录数，即使有些记录已存在
+      processedRecords += records.length;
-      totalBatchRecords += batchSize; // 只增加实际插入的记录数
+      totalBatchRecords += batchSize;
 
       logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
-
-      // 更新查询条件，以便下一批次查询
-      query.createTime = { $gt: lastCreateTime };
-      if (lastId) {
-        query._id = { $gt: new ObjectId(lastId) };
-      }
-      logWithTimestamp(`更新查询条件: 创建时间 > ${new Date(lastCreateTime).toISOString()}, ID > ${lastId || 'none'}`);
     }
 
-    // 更新同步状态
-    const newSyncState: SyncState = {
-      last_sync_time: lastCreateTime,
-      records_synced: syncState.records_synced + totalBatchRecords,
-      last_sync_id: lastId
-    };
-
-    await setVariable("f/shorturl_analytics/clickhouse/shorturl_sync_state", JSON.stringify(newSyncState));
-    console.log(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 总同步记录数 ${newSyncState.records_synced}`);
-
     return {
       success: true,
       records_processed: processedRecords,
       records_synced: totalBatchRecords,
-      total_synced: newSyncState.records_synced,
-      last_sync_time: new Date(newSyncState.last_sync_time).toISOString(),
       message: "数据同步完成"
     };
   } catch (err) {

windmill/sync_shorturl_from_mongo_to_clickhouse.ts

@@ -22,9 +22,10 @@ interface ClickHouseConfig {
 interface ShortRecord {
   _id: ObjectId;
   slug: string;          // 短链接的slug部分
-  url: string;           // 原始URL
+  origin: string;        // 原始URL
+  domain?: string;       // 域名
   createTime: number;    // 创建时间戳
-  user: string;          // 创建用户
+  user?: string;         // 创建用户
   title?: string;        // 标题
   description?: string;  // 描述
   tags?: string[];       // 标签
@@ -41,9 +42,9 @@ interface SyncState {
 }
 
 export async function main(
-  batch_size = 50,
+  batch_size = 100,
   initial_sync = false,
-  max_records = 1000,
+  max_records = 999999,
   timeout_minutes = 30,
   skip_clickhouse_check = false,
   force_insert = false
@@ -248,7 +249,7 @@ export async function main(
       
       try {
         // 提取所有记录的ID
-        const recordIds = records.map(record => record.slug);
+        const recordIds = records.map(record => record._id.toString());
         logWithTimestamp(`待检查的短链接ID: ${recordIds.join(', ')}`);
         
         // 构建查询SQL，检查记录是否已存在
@@ -311,7 +312,7 @@ export async function main(
         }
         
         // 过滤出不存在的记录
-        const newRecords = records.filter(record => !existingIds.has(record.slug));
+        const newRecords = records.filter(record => !existingIds.has(record._id.toString()));
         logWithTimestamp(`过滤后剩余 ${newRecords.length} 条新记录需要插入`);
         
         return newRecords;
@@ -374,11 +375,11 @@ export async function main(
         const expiresAtStr = record.expiresAt ? new Date(record.expiresAt).toISOString().replace('Z', '') : null;
         
         return {
-          link_id: record.slug,
+          link_id: record._id.toString(), // 使用MongoDB的_id作为link_id
-          original_url: record.url || "",
+          original_url: record.origin || "",
           created_at: createdAtStr,
           created_by: record.user || "unknown",
-          title: record.title || (record.url ? record.url.substring(0, 50) : "无标题"),
+          title: record.slug, // 使用slug作为title
           description: record.description || "",
           tags: record.tags || [],
           is_active: record.active !== undefined ? record.active : true,