From 2cb45781c78cec5a28c05f21818e48759265d61f Mon Sep 17 00:00:00 2001 From: William Tso Date: Thu, 17 Apr 2025 22:23:38 +0800 Subject: [PATCH] Add req_full_path to Event interface, implement activities API for event retrieval, and enhance sync script with short link details --- app/analytics/page.tsx | 3 +- app/api/activities/route.ts | 141 +++++++++++++++++++++++++++++++ windmill/sync_mongo_to_events.ts | 57 ++++++++++--- 3 files changed, 190 insertions(+), 11 deletions(-) create mode 100644 app/api/activities/route.ts diff --git a/app/analytics/page.tsx b/app/analytics/page.tsx index 8315169..c290ed4 100644 --- a/app/analytics/page.tsx +++ b/app/analytics/page.tsx @@ -41,6 +41,7 @@ interface Event { link_slug?: string; link_tags?: string; ip_address?: string; + req_full_path?: string; } // 格式化日期函数 @@ -100,7 +101,7 @@ const extractEventInfo = (event: Event) => { eventTime: event.created_at || event.event_time, linkName: event.link_label || linkAttrs?.name || eventAttrs?.link_name || event.link_slug || '-', originalUrl: event.link_original_url || eventAttrs?.origin_url || '-', - fullUrl: eventAttrs?.full_url || '-', + fullUrl: event.req_full_path || eventAttrs?.full_url || '-', eventType: event.event_type || '-', visitorId: event.visitor_id?.substring(0, 8) || '-', referrer: eventAttrs?.referrer || '-', diff --git a/app/api/activities/route.ts b/app/api/activities/route.ts new file mode 100644 index 0000000..809dfaa --- /dev/null +++ b/app/api/activities/route.ts @@ -0,0 +1,141 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getEvents } from '@/lib/analytics'; +import { ApiResponse } from '@/lib/types'; + +export async function GET(request: NextRequest) { + try { + const { searchParams } = new URL(request.url); + + // Get required parameters + const slug = searchParams.get('slug'); + const domain = searchParams.get('domain'); + + // If slug or domain is missing, return an error + if (!slug || !domain) { + return NextResponse.json({ + success: false, + error: 'Missing required parameters: slug and domain are required' + }, { status: 400 }); + } + + // Construct the shortUrl from domain and slug + const shortUrl = `https://${domain}/${slug}`; + + // Log the request for debugging + console.log('Activities API received parameters:', { + slug, + domain, + shortUrl + }); + + // Set default page size and page + const page = parseInt(searchParams.get('page') || '1'); + const pageSize = parseInt(searchParams.get('pageSize') || '50'); + + // Optional date range parameters + const startTime = searchParams.get('startTime') || undefined; + const endTime = searchParams.get('endTime') || undefined; + + // Get events for the specified shortUrl + const { events, total } = await getEvents({ + linkSlug: slug, + page, + pageSize, + startTime, + endTime, + sortBy: 'event_time', + sortOrder: 'desc' + }); + + // Process the events to extract useful information + const processedEvents = events.map(event => { + // Parse JSON strings to objects safely + let eventAttributes: Record = {}; + + try { + if (typeof event.event_attributes === 'string') { + eventAttributes = JSON.parse(event.event_attributes); + } else if (typeof event.event_attributes === 'object') { + eventAttributes = event.event_attributes; + } + } catch { + // Keep default empty object if parsing fails + } + + // Extract tags + let tags: string[] = []; + + try { + if (typeof event.link_tags === 'string') { + const parsedTags = JSON.parse(event.link_tags); + if (Array.isArray(parsedTags)) { + tags = parsedTags; + } + } else if (Array.isArray(event.link_tags)) { + tags = event.link_tags; + } + } catch { + // If parsing fails, keep tags as empty array + } + + // Return a simplified event object + return { + id: event.event_id, + type: event.event_type, + time: event.event_time, + visitor: { + id: event.visitor_id, + ipAddress: event.ip_address, + userAgent: eventAttributes.user_agent as string || null, + referrer: eventAttributes.referrer as string || null + }, + device: { + type: event.device_type, + browser: event.browser, + os: event.os + }, + location: { + country: event.country, + city: event.city + }, + link: { + id: event.link_id, + slug: event.link_slug, + originalUrl: event.link_original_url, + label: event.link_label, + tags + }, + utm: { + source: eventAttributes.utm_source as string || null, + medium: eventAttributes.utm_medium as string || null, + campaign: eventAttributes.utm_campaign as string || null, + term: eventAttributes.utm_term as string || null, + content: eventAttributes.utm_content as string || null + } + }; + }); + + // Return processed events + const response: ApiResponse = { + success: true, + data: processedEvents, + meta: { + total, + page, + pageSize + } + }; + + return NextResponse.json(response); + } catch (error) { + console.error('Error retrieving activities:', error); + + const response: ApiResponse = { + success: false, + data: null, + error: error instanceof Error ? error.message : 'An error occurred while retrieving activities' + }; + + return NextResponse.json(response, { status: 500 }); + } +} \ No newline at end of file diff --git a/windmill/sync_mongo_to_events.ts b/windmill/sync_mongo_to_events.ts index 1a5e989..d2982df 100644 --- a/windmill/sync_mongo_to_events.ts +++ b/windmill/sync_mongo_to_events.ts @@ -33,6 +33,23 @@ interface TraceRecord { createTime: number; } +// 添加 ShortRecord 接口定义 +interface ShortRecord { + _id: ObjectId; + slug: string; // 短链接的slug部分 + origin: string; // 原始URL + domain?: string; // 域名 + createTime: number; // 创建时间戳 + user?: string; // 创建用户 + title?: string; // 标题 + description?: string; // 描述 + tags?: string[]; // 标签 + active?: boolean; // 是否活跃 + expiresAt?: number; // 过期时间戳 + teamId?: string; // 团队ID + projectId?: string; // 项目ID +} + interface SyncState { last_sync_time: number; records_synced: number; @@ -181,6 +198,8 @@ export async function main( const db = client.database(mongoConfig.db); const traceCollection = db.collection("trace"); + // 添加对short集合的引用 + const shortCollection = db.collection("short"); // 构建查询条件，根据上次同步状态获取新记录 const query: Record = { @@ -380,9 +399,23 @@ export async function main( logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`); + // 获取链接信息 - 新增代码 + const slugIds = newRecords.map(record => record.slugId); + logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`); + const shortLinks = await shortCollection.find({ + _id: { $in: slugIds } + }).toArray(); + + // 创建映射用于快速查找 - 新增代码 + const shortLinksMap = new Map(shortLinks.map(link => [link._id.toString(), link])); + logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息`); + // 准备ClickHouse插入数据 const clickhouseData = newRecords.map(record => { const eventTime = new Date(record.createTime); + // 获取对应的短链接信息 - 新增代码 + const shortLink = shortLinksMap.get(record.slugId.toString()); + // 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构 return { // UUID将由ClickHouse自动生成 (event_id) @@ -390,22 +423,26 @@ export async function main( event_type: record.type === 1 ? "visit" : "custom", event_attributes: `{"mongo_id":"${record._id.toString()}"}`, link_id: record.slugId.toString(), - link_slug: "", // 这些字段可能需要从其他表获取 + link_slug: shortLink?.slug || "", // 新增: 从short获取slug link_label: record.label || "", - link_title: "", - link_original_url: "", - link_attributes: "{}", - link_created_at: eventTime.toISOString().replace('T', ' ').replace('Z', ''), // 暂用访问时间代替，可能需要从其他表获取 - link_expires_at: null, - link_tags: "[]", - user_id: "", + link_title: shortLink?.title || "", // 新增: 从short获取标题 + link_original_url: shortLink?.origin || "", // 新增: 从short获取原始URL + link_attributes: JSON.stringify({ domain: shortLink?.domain || null }), // 新增: 从short获取域名信息 + link_created_at: shortLink?.createTime + ? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '') + : eventTime.toISOString().replace('T', ' ').replace('Z', ''), // 新增: 使用真实的创建时间 + link_expires_at: shortLink?.expiresAt + ? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '') + : null, // 新增: 使用真实的过期时间 + link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]", // 新增: 从short获取标签 + user_id: shortLink?.user || "", // 新增: 从short获取用户ID user_name: "", user_email: "", user_attributes: "{}", - team_id: "", + team_id: shortLink?.teamId || "", // 新增: 从short获取团队ID team_name: "", team_attributes: "{}", - project_id: "", + project_id: shortLink?.projectId || "", // 新增: 从short获取项目ID project_name: "", project_attributes: "{}", qr_code_id: "",