Add req_full_path to Event interface, implement activities API for event retrieval, and enhance sync script with short link details
This commit is contained in:
@@ -41,6 +41,7 @@ interface Event {
|
||||
link_slug?: string;
|
||||
link_tags?: string;
|
||||
ip_address?: string;
|
||||
req_full_path?: string;
|
||||
}
|
||||
|
||||
// 格式化日期函数
|
||||
@@ -100,7 +101,7 @@ const extractEventInfo = (event: Event) => {
|
||||
eventTime: event.created_at || event.event_time,
|
||||
linkName: event.link_label || linkAttrs?.name || eventAttrs?.link_name || event.link_slug || '-',
|
||||
originalUrl: event.link_original_url || eventAttrs?.origin_url || '-',
|
||||
fullUrl: eventAttrs?.full_url || '-',
|
||||
fullUrl: event.req_full_path || eventAttrs?.full_url || '-',
|
||||
eventType: event.event_type || '-',
|
||||
visitorId: event.visitor_id?.substring(0, 8) || '-',
|
||||
referrer: eventAttrs?.referrer || '-',
|
||||
|
||||
141
app/api/activities/route.ts
Normal file
141
app/api/activities/route.ts
Normal file
@@ -0,0 +1,141 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getEvents } from '@/lib/analytics';
|
||||
import { ApiResponse } from '@/lib/types';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
|
||||
// Get required parameters
|
||||
const slug = searchParams.get('slug');
|
||||
const domain = searchParams.get('domain');
|
||||
|
||||
// If slug or domain is missing, return an error
|
||||
if (!slug || !domain) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Missing required parameters: slug and domain are required'
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// Construct the shortUrl from domain and slug
|
||||
const shortUrl = `https://${domain}/${slug}`;
|
||||
|
||||
// Log the request for debugging
|
||||
console.log('Activities API received parameters:', {
|
||||
slug,
|
||||
domain,
|
||||
shortUrl
|
||||
});
|
||||
|
||||
// Set default page size and page
|
||||
const page = parseInt(searchParams.get('page') || '1');
|
||||
const pageSize = parseInt(searchParams.get('pageSize') || '50');
|
||||
|
||||
// Optional date range parameters
|
||||
const startTime = searchParams.get('startTime') || undefined;
|
||||
const endTime = searchParams.get('endTime') || undefined;
|
||||
|
||||
// Get events for the specified shortUrl
|
||||
const { events, total } = await getEvents({
|
||||
linkSlug: slug,
|
||||
page,
|
||||
pageSize,
|
||||
startTime,
|
||||
endTime,
|
||||
sortBy: 'event_time',
|
||||
sortOrder: 'desc'
|
||||
});
|
||||
|
||||
// Process the events to extract useful information
|
||||
const processedEvents = events.map(event => {
|
||||
// Parse JSON strings to objects safely
|
||||
let eventAttributes: Record<string, unknown> = {};
|
||||
|
||||
try {
|
||||
if (typeof event.event_attributes === 'string') {
|
||||
eventAttributes = JSON.parse(event.event_attributes);
|
||||
} else if (typeof event.event_attributes === 'object') {
|
||||
eventAttributes = event.event_attributes;
|
||||
}
|
||||
} catch {
|
||||
// Keep default empty object if parsing fails
|
||||
}
|
||||
|
||||
// Extract tags
|
||||
let tags: string[] = [];
|
||||
|
||||
try {
|
||||
if (typeof event.link_tags === 'string') {
|
||||
const parsedTags = JSON.parse(event.link_tags);
|
||||
if (Array.isArray(parsedTags)) {
|
||||
tags = parsedTags;
|
||||
}
|
||||
} else if (Array.isArray(event.link_tags)) {
|
||||
tags = event.link_tags;
|
||||
}
|
||||
} catch {
|
||||
// If parsing fails, keep tags as empty array
|
||||
}
|
||||
|
||||
// Return a simplified event object
|
||||
return {
|
||||
id: event.event_id,
|
||||
type: event.event_type,
|
||||
time: event.event_time,
|
||||
visitor: {
|
||||
id: event.visitor_id,
|
||||
ipAddress: event.ip_address,
|
||||
userAgent: eventAttributes.user_agent as string || null,
|
||||
referrer: eventAttributes.referrer as string || null
|
||||
},
|
||||
device: {
|
||||
type: event.device_type,
|
||||
browser: event.browser,
|
||||
os: event.os
|
||||
},
|
||||
location: {
|
||||
country: event.country,
|
||||
city: event.city
|
||||
},
|
||||
link: {
|
||||
id: event.link_id,
|
||||
slug: event.link_slug,
|
||||
originalUrl: event.link_original_url,
|
||||
label: event.link_label,
|
||||
tags
|
||||
},
|
||||
utm: {
|
||||
source: eventAttributes.utm_source as string || null,
|
||||
medium: eventAttributes.utm_medium as string || null,
|
||||
campaign: eventAttributes.utm_campaign as string || null,
|
||||
term: eventAttributes.utm_term as string || null,
|
||||
content: eventAttributes.utm_content as string || null
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Return processed events
|
||||
const response: ApiResponse<typeof processedEvents> = {
|
||||
success: true,
|
||||
data: processedEvents,
|
||||
meta: {
|
||||
total,
|
||||
page,
|
||||
pageSize
|
||||
}
|
||||
};
|
||||
|
||||
return NextResponse.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error retrieving activities:', error);
|
||||
|
||||
const response: ApiResponse<null> = {
|
||||
success: false,
|
||||
data: null,
|
||||
error: error instanceof Error ? error.message : 'An error occurred while retrieving activities'
|
||||
};
|
||||
|
||||
return NextResponse.json(response, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,23 @@ interface TraceRecord {
|
||||
createTime: number;
|
||||
}
|
||||
|
||||
// 添加 ShortRecord 接口定义
|
||||
interface ShortRecord {
|
||||
_id: ObjectId;
|
||||
slug: string; // 短链接的slug部分
|
||||
origin: string; // 原始URL
|
||||
domain?: string; // 域名
|
||||
createTime: number; // 创建时间戳
|
||||
user?: string; // 创建用户
|
||||
title?: string; // 标题
|
||||
description?: string; // 描述
|
||||
tags?: string[]; // 标签
|
||||
active?: boolean; // 是否活跃
|
||||
expiresAt?: number; // 过期时间戳
|
||||
teamId?: string; // 团队ID
|
||||
projectId?: string; // 项目ID
|
||||
}
|
||||
|
||||
interface SyncState {
|
||||
last_sync_time: number;
|
||||
records_synced: number;
|
||||
@@ -181,6 +198,8 @@ export async function main(
|
||||
|
||||
const db = client.database(mongoConfig.db);
|
||||
const traceCollection = db.collection<TraceRecord>("trace");
|
||||
// 添加对short集合的引用
|
||||
const shortCollection = db.collection<ShortRecord>("short");
|
||||
|
||||
// 构建查询条件，根据上次同步状态获取新记录
|
||||
const query: Record<string, unknown> = {
|
||||
@@ -380,9 +399,23 @@ export async function main(
|
||||
|
||||
logWithTimestamp(`准备处理 ${newRecords.length} 条新记录...`);
|
||||
|
||||
// 获取链接信息 - 新增代码
|
||||
const slugIds = newRecords.map(record => record.slugId);
|
||||
logWithTimestamp(`正在查询 ${slugIds.length} 条短链接信息...`);
|
||||
const shortLinks = await shortCollection.find({
|
||||
_id: { $in: slugIds }
|
||||
}).toArray();
|
||||
|
||||
// 创建映射用于快速查找 - 新增代码
|
||||
const shortLinksMap = new Map(shortLinks.map(link => [link._id.toString(), link]));
|
||||
logWithTimestamp(`获取到 ${shortLinks.length} 条短链接信息`);
|
||||
|
||||
// 准备ClickHouse插入数据
|
||||
const clickhouseData = newRecords.map(record => {
|
||||
const eventTime = new Date(record.createTime);
|
||||
// 获取对应的短链接信息 - 新增代码
|
||||
const shortLink = shortLinksMap.get(record.slugId.toString());
|
||||
|
||||
// 转换MongoDB记录为ClickHouse格式，匹配ClickHouse表结构
|
||||
return {
|
||||
// UUID将由ClickHouse自动生成 (event_id)
|
||||
@@ -390,22 +423,26 @@ export async function main(
|
||||
event_type: record.type === 1 ? "visit" : "custom",
|
||||
event_attributes: `{"mongo_id":"${record._id.toString()}"}`,
|
||||
link_id: record.slugId.toString(),
|
||||
link_slug: "", // 这些字段可能需要从其他表获取
|
||||
link_slug: shortLink?.slug || "", // 新增: 从short获取slug
|
||||
link_label: record.label || "",
|
||||
link_title: "",
|
||||
link_original_url: "",
|
||||
link_attributes: "{}",
|
||||
link_created_at: eventTime.toISOString().replace('T', ' ').replace('Z', ''), // 暂用访问时间代替，可能需要从其他表获取
|
||||
link_expires_at: null,
|
||||
link_tags: "[]",
|
||||
user_id: "",
|
||||
link_title: shortLink?.title || "", // 新增: 从short获取标题
|
||||
link_original_url: shortLink?.origin || "", // 新增: 从short获取原始URL
|
||||
link_attributes: JSON.stringify({ domain: shortLink?.domain || null }), // 新增: 从short获取域名信息
|
||||
link_created_at: shortLink?.createTime
|
||||
? new Date(shortLink.createTime).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: eventTime.toISOString().replace('T', ' ').replace('Z', ''), // 新增: 使用真实的创建时间
|
||||
link_expires_at: shortLink?.expiresAt
|
||||
? new Date(shortLink.expiresAt).toISOString().replace('T', ' ').replace('Z', '')
|
||||
: null, // 新增: 使用真实的过期时间
|
||||
link_tags: shortLink?.tags ? JSON.stringify(shortLink.tags) : "[]", // 新增: 从short获取标签
|
||||
user_id: shortLink?.user || "", // 新增: 从short获取用户ID
|
||||
user_name: "",
|
||||
user_email: "",
|
||||
user_attributes: "{}",
|
||||
team_id: "",
|
||||
team_id: shortLink?.teamId || "", // 新增: 从short获取团队ID
|
||||
team_name: "",
|
||||
team_attributes: "{}",
|
||||
project_id: "",
|
||||
project_id: shortLink?.projectId || "", // 新增: 从short获取项目ID
|
||||
project_name: "",
|
||||
project_attributes: "{}",
|
||||
qr_code_id: "",
|
||||
|
||||
Reference in New Issue
Block a user