vitalitymailg/shorturl-analytics
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

links

Browse Source
This commit is contained in:
William Tso
2025-04-07 21:48:24 +08:00
parent 1a9e28bd7e
commit 33dbf62665
7 changed files with 1100 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
app/api/shortlinks/route.ts Normal file
View File

@@ -0,0 +1,53 @@
import { NextResponse } from 'next/server';
import { executeQuery } from '@/lib/clickhouse';
export async function GET() {
try {
// Query to fetch shorturl data from ClickHouse
const query = `
SELECT
id,
external_id,
type,
slug,
original_url,
title,
description,
attributes,
schema_version,
creator_id,
creator_email,
creator_name,
created_at,
updated_at,
deleted_at,
projects,
teams,
tags,
qr_codes AS qr_codes,
channels,
favorites,
expires_at,
click_count,
unique_visitors
FROM shorturl_analytics.shorturl
WHERE deleted_at IS NULL
ORDER BY created_at DESC
`;
// Execute the query using the shared client
const rows = await executeQuery(query);
// Return the data
return NextResponse.json({
links: rows,
total: rows.length
});
} catch (error) {
console.error('Error fetching shortlinks from ClickHouse:', error);
return NextResponse.json(
{ error: 'Failed to fetch shortlinks' },
{ status: 500 }
);
}
}

17
app/components/layout/Header.tsx
View File

@@ -30,6 +30,23 @@ export default function Header() {
</svg>
<span className="text-xl font-bold text-gray-900">ShortURL Analytics</span>
</Link>
{user && (
<nav className="ml-6">
<ul className="flex space-x-4">
<li>
<Link href="/" className="text-sm text-gray-700 hover:text-blue-500">
Dashboard
</Link>
</li>
<li>
<Link href="/links" className="text-sm text-gray-700 hover:text-blue-500">
Short Links
</Link>
</li>
</ul>
</nav>
)}
</div>
{user && (

10
app/ip-test/page.tsx
View File

@@ -1,10 +0,0 @@
import IpLocationTest from '../components/ipLocationTest';
export default function IpTestPage() {
return (
<div className="container mx-auto p-4 max-w-4xl">
<h1 className="text-2xl font-bold mb-6">IP to Location Test</h1>
<IpLocationTest />
</div>
);
}

384
app/links/page.tsx Normal file
View File

@@ -0,0 +1,384 @@
"use client";
import { useEffect, useState } from 'react';
import { getSupabaseClient } from '../utils/supabase';
import { AuthChangeEvent } from '@supabase/supabase-js';
import { Loader2, ExternalLink, Copy, Search } from 'lucide-react';
// Define attribute type to avoid using 'any'
interface LinkAttributes {
title?: string;
name?: string;
slug?: string;
original_url?: string;
originalUrl?: string;
visits?: number;
click_count?: number;
team_id?: string;
team_name?: string;
tags?: string[];
[key: string]: unknown;
}
// Define Link type based on the database schema
interface ShortLink {
id: string;
external_id: string | null;
attributes: LinkAttributes | null;
type: string | null;
creator_id: string | null;
created_at: string | null;
updated_at: string | null;
deleted_at: string | null;
schema_version: number | null;
}
// Define ClickHouse shorturl type
interface ClickHouseShortUrl {
id: string;
external_id: string;
type: string;
slug: string;
original_url: string;
title: string;
description: string;
attributes: string; // JSON string
schema_version: number;
creator_id: string;
creator_email: string;
creator_name: string;
created_at: string;
updated_at: string;
deleted_at: string | null;
projects: string; // JSON string
teams: string; // JSON string
tags: string; // JSON string
qr_codes: string; // JSON string
channels: string; // JSON string
favorites: string; // JSON string
expires_at: string | null;
click_count: number;
unique_visitors: number;
link_attributes?: string; // Optional JSON string containing link-specific attributes
}
// Convert ClickHouse data to ShortLink format
function convertClickHouseToShortLink(chData: ClickHouseShortUrl): ShortLink {
// Parse JSON strings
const attributesJson = JSON.parse(chData.attributes || '{}');
const teamsJson = JSON.parse(chData.teams || '[]');
const tagsJson = JSON.parse(chData.tags || '[]');
// Extract team info from the first team if available
const teamInfo = teamsJson.length > 0 ? {
team_id: teamsJson[0].team_id,
team_name: teamsJson[0].team_name
} : {};
// Extract tag names
const tagNames = tagsJson.map((tag: { tag_name: string }) => tag.tag_name);
// Parse link_attributes to get domain if available
let domain = 'shorturl.example.com';
try {
if (chData.link_attributes) {
const linkAttrObj = JSON.parse(chData.link_attributes);
if (linkAttrObj.domain) {
domain = linkAttrObj.domain;
}
}
// Extract domain from shortUrl in attributes if available
const attributesObj = JSON.parse(chData.attributes || '{}');
if (attributesObj.shortUrl) {
try {
const urlObj = new URL(attributesObj.shortUrl);
domain = urlObj.hostname;
} catch (e) {
console.error('Error parsing shortUrl:', e);
}
}
} catch (e) {
console.error('Error parsing link_attributes:', e);
}
return {
id: chData.id,
external_id: chData.external_id,
type: chData.type,
creator_id: chData.creator_id,
created_at: chData.created_at,
updated_at: chData.updated_at,
deleted_at: chData.deleted_at,
schema_version: chData.schema_version,
attributes: {
...attributesJson,
title: chData.title || attributesJson.title || '',
slug: chData.slug,
original_url: chData.original_url,
click_count: chData.click_count,
visits: chData.click_count,
unique_visitors: chData.unique_visitors,
domain: attributesJson.domain || domain,
...teamInfo,
tags: tagNames
}
};
}
export default function LinksPage() {
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [links, setLinks] = useState<ShortLink[]>([]);
const [searchQuery, setSearchQuery] = useState('');
const [teamFilter, setTeamFilter] = useState<string | null>(null);
// Copy link to clipboard
const copyToClipboard = (text: string) => {
navigator.clipboard.writeText(text).then(() => {
// Could add a notification here
console.log('Copied to clipboard');
});
};
// Extract link metadata from attributes
const getLinkMetadata = (link: ShortLink) => {
const attrs = link.attributes || {};
return {
title: attrs.title || attrs.name || 'Untitled Link',
slug: attrs.slug || 'no-slug',
originalUrl: attrs.original_url || attrs.originalUrl || '#',
visits: attrs.visits || attrs.click_count || 0,
teamId: attrs.team_id || '',
teamName: attrs.team_name || 'Personal',
createdAt: new Date(link.created_at || Date.now()).toLocaleDateString(),
tags: attrs.tags || [],
domain: attrs.domain || 'shorturl.example.com',
};
};
// Filter links by search query
const filteredLinks = links.length > 0 ?
links.filter(link => {
if (!searchQuery && !teamFilter) return true;
const metadata = getLinkMetadata(link);
const matchesSearch = searchQuery ?
(metadata.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
metadata.slug.toLowerCase().includes(searchQuery.toLowerCase()) ||
metadata.originalUrl.toLowerCase().includes(searchQuery.toLowerCase())) :
true;
const matchesTeam = teamFilter ?
metadata.teamId === teamFilter :
true;
return matchesSearch && matchesTeam;
}) : [];
// Get unique teams for filtering
const teams = links.length > 0 ?
Array.from(
links.reduce((teams, link) => {
const metadata = getLinkMetadata(link);
if (metadata.teamId) {
teams.set(metadata.teamId, metadata.teamName);
}
return teams;
}, new Map<string, string>())
).map(([id, name]) => ({ id, name })) : [];
useEffect(() => {
let isMounted = true;
const fetchLinks = async () => {
if (!isMounted) return;
setLoading(true);
setError(null);
try {
// Fetch data from ClickHouse API instead of Supabase
const response = await fetch('/api/shortlinks');
if (!response.ok) {
throw new Error(`Failed to fetch links: ${response.status} ${response.statusText}`);
}
const data = await response.json();
if (!data || !data.links || data.links.length === 0) {
if (isMounted) setLinks([]);
return;
}
// Convert ClickHouse data format to ShortLink format
const convertedLinks = data.links.map(convertClickHouseToShortLink);
if (isMounted) {
setLinks(convertedLinks);
}
} catch (err) {
if (isMounted) {
setError(err instanceof Error ? err.message : 'Failed to load short URLs');
console.error("Error fetching links:", err);
}
} finally {
if (isMounted) {
setLoading(false);
}
}
};
const supabase = getSupabaseClient();
const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent) => {
if (event === 'SIGNED_IN') {
fetchLinks();
} else if (event === 'SIGNED_OUT') {
setLinks([]);
setError(null);
}
});
supabase.auth.getSession().then(() => {
fetchLinks();
});
return () => {
isMounted = false;
subscription.unsubscribe();
};
}, []);
if (loading) {
return (
<div className="flex w-full items-center justify-center p-12">
<Loader2 className="h-8 w-8 animate-spin text-blue-500" />
</div>
);
}
if (error) {
return (
<div className="flex w-full items-center justify-center p-8 text-red-500">
<p>Error: {error}</p>
</div>
);
}
if (links.length === 0) {
return (
<div className="flex w-full flex-col items-center justify-center p-12 text-gray-500">
<p className="mb-4 text-xl">No short links found</p>
<p>Create your first short URL to get started</p>
</div>
);
}
return (
<div className="container mx-auto px-4 py-8">
<h1 className="mb-6 text-2xl font-bold text-gray-900">Short URL Links</h1>
{/* Search and filters */}
<div className="mb-6 flex flex-wrap items-center gap-4">
<div className="relative flex-grow">
<Search className="absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-gray-400" />
<input
type="text"
placeholder="Search links..."
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
className="w-full rounded-md border border-gray-300 py-2 pl-10 pr-4 text-sm focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500"
/>
</div>
<select
value={teamFilter || ''}
onChange={(e) => setTeamFilter(e.target.value || null)}
className="rounded-md border border-gray-300 py-2 pl-3 pr-10 text-sm focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500"
>
<option value="">All Teams</option>
{teams.map(team => (
<option key={team.id} value={team.id}>{team.name}</option>
))}
</select>
</div>
{/* Links table */}
<div className="overflow-hidden rounded-lg border border-gray-200 shadow">
<table className="min-w-full divide-y divide-gray-200">
<thead className="bg-gray-50">
<tr>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500">Link</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500">Original URL</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500">Team</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500">Created</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500">Visits</th>
<th scope="col" className="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500">Actions</th>
</tr>
</thead>
<tbody className="divide-y divide-gray-200 bg-white">
{filteredLinks.map(link => {
const metadata = getLinkMetadata(link);
const shortUrl = `https://${metadata.domain}/${metadata.slug}`;
return (
<tr key={link.id} className="hover:bg-gray-50">
<td className="whitespace-nowrap px-6 py-4">
<div className="flex flex-col">
<span className="font-medium text-blue-600">{metadata.title}</span>
<span className="text-xs text-gray-500">{shortUrl}</span>
</div>
</td>
<td className="max-w-xs truncate px-6 py-4 text-sm text-gray-500">
<a href={metadata.originalUrl} target="_blank" rel="noopener noreferrer" className="flex items-center hover:text-blue-500">
{metadata.originalUrl}
<ExternalLink className="ml-1 h-3 w-3" />
</a>
</td>
<td className="px-6 py-4 text-sm text-gray-500">
{metadata.teamName}
</td>
<td className="px-6 py-4 text-sm text-gray-500">
{metadata.createdAt}
</td>
<td className="px-6 py-4 text-sm text-gray-500">
{metadata.visits}
</td>
<td className="px-6 py-4 text-sm">
<div className="flex space-x-2">
<button
onClick={() => copyToClipboard(shortUrl)}
className="rounded-md p-1 text-gray-400 hover:bg-gray-100 hover:text-gray-500"
title="Copy link"
>
<Copy className="h-5 w-5" />
</button>
<a
href={shortUrl}
target="_blank"
rel="noopener noreferrer"
className="rounded-md p-1 text-gray-400 hover:bg-gray-100 hover:text-gray-500"
title="Open link"
>
<ExternalLink className="h-5 w-5" />
</a>
</div>
</td>
</tr>
);
})}
</tbody>
</table>
</div>
{/* Pagination or "Load More" could be added here */}
{filteredLinks.length === 0 && links.length > 0 && (
<div className="mt-6 rounded-md bg-gray-50 p-6 text-center text-gray-500">
No links match your search criteria
</div>
)}
</div>
);
}

225
scripts/db/db-reports/clickhouse-schema-2025-03-20T13-57-59-013Z.log
View File

@@ -1,225 +0,0 @@
获取所有表...
数据库 limq 中找到以下表:
- .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb
- .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc
- .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1
- .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0
- .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024
- .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea
- link_daily_stats
- link_events
- link_hourly_patterns
- links
- platform_distribution
- project_daily_stats
- projects
- qr_scans
- qrcode_daily_stats
- qrcodes
- sessions
- team_daily_stats
- team_members
- teams
所有ClickHouse表:
.inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb, .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc, .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1, .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0, .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024, .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea, link_daily_stats, link_events, link_hourly_patterns, links, platform_distribution, project_daily_stats, projects, qr_scans, qrcode_daily_stats, qrcodes, sessions, team_daily_stats, team_members, teams
获取表 .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb 的结构...
获取表 .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc 的结构...
获取表 .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1 的结构...
获取表 .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0 的结构...
获取表 .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024 的结构...
获取表 .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea 的结构...
获取表 link_daily_stats 的结构...
表 link_daily_stats 的列:
- date (Date, 无默认值)
- link_id (String, 无默认值)
- total_clicks (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
- unique_sessions (UInt64, 无默认值)
- total_time_spent (UInt64, 无默认值)
- avg_time_spent (Float64, 无默认值)
- bounce_count (UInt64, 无默认值)
- conversion_count (UInt64, 无默认值)
- unique_referrers (UInt64, 无默认值)
- mobile_count (UInt64, 无默认值)
- tablet_count (UInt64, 无默认值)
- desktop_count (UInt64, 无默认值)
- qr_scan_count (UInt64, 无默认值)
- total_conversion_value (Float64, 无默认值)
获取表 link_events 的结构...
表 link_events 的列:
- event_id (UUID, 默认值: generateUUIDv4())
- event_time (DateTime64(3), 默认值: now64())
- date (Date, 默认值: toDate(event_time))
- link_id (String, 无默认值)
- channel_id (String, 无默认值)
- visitor_id (String, 无默认值)
- session_id (String, 无默认值)
- event_type (Enum8('click' = 1, 'redirect' = 2, 'conversion' = 3, 'error' = 4), 无默认值)
- ip_address (String, 无默认值)
- country (String, 无默认值)
- city (String, 无默认值)
- referrer (String, 无默认值)
- utm_source (String, 无默认值)
- utm_medium (String, 无默认值)
- utm_campaign (String, 无默认值)
- user_agent (String, 无默认值)
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
- browser (String, 无默认值)
- os (String, 无默认值)
- time_spent_sec (UInt32, 默认值: 0)
- is_bounce (Bool, 默认值: true)
- is_qr_scan (Bool, 默认值: false)
- qr_code_id (String, 默认值: '')
- conversion_type (Enum8('visit' = 1, 'stay' = 2, 'interact' = 3, 'signup' = 4, 'subscription' = 5, 'purchase' = 6), 默认值: 'visit')
- conversion_value (Float64, 默认值: 0)
- custom_data (String, 默认值: '{}')
获取表 link_hourly_patterns 的结构...
表 link_hourly_patterns 的列:
- date (Date, 无默认值)
- hour (UInt8, 无默认值)
- link_id (String, 无默认值)
- visits (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
获取表 links 的结构...
表 links 的列:
- link_id (String, 无默认值)
- original_url (String, 无默认值)
- created_at (DateTime64(3), 无默认值)
- created_by (String, 无默认值)
- title (String, 无默认值)
- description (String, 无默认值)
- tags (Array(String), 无默认值)
- is_active (Bool, 默认值: true)
- expires_at (Nullable(DateTime), 无默认值)
- team_id (String, 默认值: '')
- project_id (String, 默认值: '')
获取表 platform_distribution 的结构...
表 platform_distribution 的列:
- date (Date, 无默认值)
- utm_source (String, 无默认值)
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
- visits (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
获取表 project_daily_stats 的结构...
表 project_daily_stats 的列:
- date (Date, 无默认值)
- project_id (String, 无默认值)
- total_clicks (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
- conversion_count (UInt64, 无默认值)
- links_used (UInt64, 无默认值)
- qr_scan_count (UInt64, 无默认值)
获取表 projects 的结构...
表 projects 的列:
- project_id (String, 无默认值)
- team_id (String, 无默认值)
- name (String, 无默认值)
- created_at (DateTime, 无默认值)
- created_by (String, 无默认值)
- description (String, 默认值: '')
- is_archived (Bool, 默认值: false)
- links_count (UInt32, 默认值: 0)
- total_clicks (UInt64, 默认值: 0)
- last_updated (DateTime, 默认值: now())
获取表 qr_scans 的结构...
表 qr_scans 的列:
- scan_id (UUID, 默认值: generateUUIDv4())
- qr_code_id (String, 无默认值)
- link_id (String, 无默认值)
- scan_time (DateTime64(3), 无默认值)
- visitor_id (String, 无默认值)
- location (String, 无默认值)
- device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值)
- led_to_conversion (Bool, 默认值: false)
获取表 qrcode_daily_stats 的结构...
表 qrcode_daily_stats 的列:
- date (Date, 无默认值)
- qr_code_id (String, 无默认值)
- total_scans (UInt64, 无默认值)
- unique_scanners (UInt64, 无默认值)
- conversions (UInt64, 无默认值)
- mobile_scans (UInt64, 无默认值)
- tablet_scans (UInt64, 无默认值)
- desktop_scans (UInt64, 无默认值)
- unique_locations (UInt64, 无默认值)
获取表 qrcodes 的结构...
表 qrcodes 的列:
- qr_code_id (String, 无默认值)
- link_id (String, 无默认值)
- team_id (String, 无默认值)
- project_id (String, 默认值: '')
- name (String, 无默认值)
- description (String, 默认值: '')
- created_at (DateTime, 无默认值)
- created_by (String, 无默认值)
- updated_at (DateTime, 默认值: now())
- qr_type (Enum8('standard' = 1, 'custom' = 2, 'dynamic' = 3), 默认值: 'standard')
- image_url (String, 默认值: '')
- design_config (String, 默认值: '{}')
- is_active (Bool, 默认值: true)
- total_scans (UInt64, 默认值: 0)
- unique_scanners (UInt32, 默认值: 0)
获取表 sessions 的结构...
表 sessions 的列:
- session_id (String, 无默认值)
- visitor_id (String, 无默认值)
- link_id (String, 无默认值)
- started_at (DateTime64(3), 无默认值)
- last_activity (DateTime64(3), 无默认值)
- ended_at (Nullable(DateTime64(3)), 无默认值)
- duration_sec (UInt32, 默认值: 0)
- session_pages (UInt8, 默认值: 1)
- is_completed (Bool, 默认值: false)
获取表 team_daily_stats 的结构...
表 team_daily_stats 的列:
- date (Date, 无默认值)
- team_id (String, 无默认值)
- total_clicks (UInt64, 无默认值)
- unique_visitors (UInt64, 无默认值)
- conversion_count (UInt64, 无默认值)
- links_used (UInt64, 无默认值)
- qr_scan_count (UInt64, 无默认值)
获取表 team_members 的结构...
表 team_members 的列:
- team_id (String, 无默认值)
- user_id (String, 无默认值)
- role (Enum8('owner' = 1, 'admin' = 2, 'editor' = 3, 'viewer' = 4), 无默认值)
- joined_at (DateTime, 默认值: now())
- invited_by (String, 无默认值)
- is_active (Bool, 默认值: true)
- last_active (DateTime, 默认值: now())
获取表 teams 的结构...
表 teams 的列:
- team_id (String, 无默认值)
- name (String, 无默认值)
- created_at (DateTime, 无默认值)
- created_by (String, 无默认值)
- description (String, 默认值: '')
- avatar_url (String, 默认值: '')
- is_active (Bool, 默认值: true)
- plan_type (Enum8('free' = 1, 'pro' = 2, 'enterprise' = 3), 无默认值)
- members_count (UInt32, 默认值: 1)
ClickHouse数据库结构检查完成

46
scripts/db/sql/clickhouse/create_shorturl_table.sql Normal file
View File

@@ -0,0 +1,46 @@
-- 使用shorturl_analytics数据库
USE shorturl_analytics;
-- 删除已存在的shorturl表
DROP TABLE IF EXISTS shorturl_analytics.shorturl;
-- 创建shorturl表
CREATE TABLE IF NOT EXISTS shorturl_analytics.shorturl (
-- 短链接基本信息（来源于resources表）
id String COMMENT '资源ID (resources.id)',
external_id String COMMENT '外部ID (resources.external_id)',
type String COMMENT '类型，值为shorturl',
slug String COMMENT '短链接slug (存储在attributes中)',
original_url String COMMENT '原始URL (存储在attributes中)',
title String COMMENT '标题 (存储在attributes中)',
description String COMMENT '描述 (存储在attributes中)',
attributes String DEFAULT '{}' COMMENT '资源属性JSON',
schema_version Int32 COMMENT 'Schema版本',
-- 创建者信息
creator_id String COMMENT '创建者ID (resources.creator_id)',
creator_email String COMMENT '创建者邮箱 (来自users表)',
creator_name String COMMENT '创建者名称 (来自users表)',
-- 时间信息
created_at DateTime64(3) COMMENT '创建时间 (resources.created_at)',
updated_at DateTime64(3) COMMENT '更新时间 (resources.updated_at)',
deleted_at Nullable(DateTime64(3)) COMMENT '删除时间 (resources.deleted_at)',
-- 项目关联 (project_resources表)
projects String DEFAULT '[]' COMMENT '项目关联信息数组。结构: [{project_id: String, project_name: String, project_description: String, assigned_at: DateTime64}]',
-- 团队关联 (通过项目关联到团队)
teams String DEFAULT '[]' COMMENT '团队关联信息数组。结构: [{team_id: String, team_name: String, team_description: String, via_project_id: String}]',
-- 标签关联 (resource_tags表)
tags String DEFAULT '[]' COMMENT '标签关联信息数组。结构: [{tag_id: String, tag_name: String, tag_type: String, created_at: DateTime64}]',
-- QR码关联 (qr_code表)
qr_codes String DEFAULT '[]' COMMENT 'QR码信息数组。结构: [{qr_id: String, scan_count: Int32, url: String, template_name: String, created_at: DateTime64}]',
-- 渠道关联 (channel表)
channels String DEFAULT '[]' COMMENT '渠道信息数组。结构: [{channel_id: String, channel_name: String, channel_path: String, is_user_created: Boolean}]',
-- 收藏关联 (favorite表)
favorites String DEFAULT '[]' COMMENT '收藏信息数组。结构: [{favorite_id: String, user_id: String, user_name: String, created_at: DateTime64}]',
-- 自定义过期时间 (存储在attributes中)
expires_at Nullable(DateTime64(3)) COMMENT '过期时间',
-- 统计信息 (分析时聚合计算)
click_count UInt32 DEFAULT 0 COMMENT '点击次数',
unique_visitors UInt32 DEFAULT 0 COMMENT '唯一访问者数'
) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at)
ORDER BY
(id, created_at) SETTINGS index_granularity = 8192 COMMENT '用于存储所有shorturl类型资源的统一表，集成了相关联的项目、团队、标签、QR码、渠道和收藏信息';

600
windmill/sync_shorturl_to_clickhouse.ts Normal file
View File

@@ -0,0 +1,600 @@
// Windmill script to sync shorturl data from PostgreSQL to ClickHouse
// 作者: AI Assistant
// 创建日期: 2023-10-30
// 描述: 此脚本从PostgreSQL数据库获取所有shorturl类型的资源及其关联数据，并同步到ClickHouse
import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts";
// 资源属性接口
interface ResourceAttributes {
slug?: string;
original_url?: string;
originalUrl?: string;
title?: string;
description?: string;
expires_at?: string;
expiresAt?: string;
[key: string]: unknown;
}
// ClickHouse配置接口
interface ChConfig {
clickhouse_host: string;
clickhouse_port: number;
clickhouse_user: string;
clickhouse_password: string;
clickhouse_url: string;
}
// PostgreSQL配置接口
interface PgConfig {
host: string;
port: number;
user: string;
password: string;
dbname?: string;
[key: string]: unknown;
}
// Windmill函数定义
export async function main(
/** PostgreSQL和ClickHouse同步脚本 */
params: {
/** 同步的资源数量限制，默认500 */
limit?: number;
/** 是否包含已删除资源 */
includeDeleted?: boolean;
/** 是否执行实际写入操作 */
dryRun?: boolean;
/** 开始时间（ISO格式）*/
startTime?: string;
/** 结束时间（ISO格式）*/
endTime?: string;
}
) {
// 设置默认参数
const limit = params.limit || 500;
const includeDeleted = params.includeDeleted || false;
const dryRun = params.dryRun || false;
const startTime = params.startTime ? new Date(params.startTime) : undefined;
const endTime = params.endTime ? new Date(params.endTime) : undefined;
console.log(`开始同步PostgreSQL shorturl数据到ClickHouse`);
console.log(`参数: limit=${limit}, includeDeleted=${includeDeleted}, dryRun=${dryRun}`);
if (startTime) console.log(`开始时间: ${startTime.toISOString()}`);
if (endTime) console.log(`结束时间: ${endTime.toISOString()}`);
// 获取数据库配置
console.log("获取PostgreSQL数据库配置...");
const pgConfig = await getResource('f/limq/postgresql') as PgConfig;
console.log(`数据库连接配置: host=${pgConfig.host}, port=${pgConfig.port}, database=${pgConfig.dbname || 'postgres'}, user=${pgConfig.user}`);
let pgPool: Pool | null = null;
try {
console.log("创建PostgreSQL连接池...");
pgPool = new Pool({
hostname: pgConfig.host,
port: pgConfig.port,
user: pgConfig.user,
password: pgConfig.password,
database: pgConfig.dbname || 'postgres'
}, 3);
console.log("PostgreSQL连接池创建完成，尝试连接...");
// 测试连接
const client = await pgPool.connect();
try {
console.log("连接成功，执行测试查询...");
const testResult = await client.queryObject(`SELECT 1 AS test`);
console.log(`测试查询结果: ${JSON.stringify(testResult.rows)}`);
} finally {
client.release();
}
// 获取所有shorturl类型的资源
const shorturls = await fetchShorturlResources(pgPool, {
limit,
includeDeleted,
startTime,
endTime,
});
console.log(`获取到 ${shorturls.length} 个shorturl资源`);
if (shorturls.length === 0) {
return { synced: 0, message: "没有找到需要同步的shorturl资源" };
}
// 为每个资源获取关联数据
const enrichedShorturls = await enrichShorturlData(pgPool, shorturls);
console.log(`已丰富 ${enrichedShorturls.length} 个shorturl资源的关联数据`);
// 转换为ClickHouse格式
const clickhouseData = formatForClickhouse(enrichedShorturls);
if (!dryRun) {
// 写入ClickHouse
const inserted = await insertToClickhouse(clickhouseData);
console.log(`成功写入 ${inserted} 条记录到ClickHouse`);
return { synced: inserted, message: "同步完成" };
} else {
console.log("Dry run模式 - 不执行实际写入");
console.log(`将写入 ${clickhouseData.length} 条记录到ClickHouse`);
// 输出示例数据
if (clickhouseData.length > 0) {
console.log("示例数据:");
console.log(JSON.stringify(clickhouseData[0], null, 2));
}
return { synced: 0, dryRun: true, sampleData: clickhouseData.slice(0, 1) };
}
} catch (error: unknown) {
console.error(`同步过程中发生错误: ${(error as Error).message}`);
console.error(`错误类型: ${(error as Error).name}`);
if ((error as Error).stack) {
console.error(`错误堆栈: ${(error as Error).stack}`);
}
throw error;
} finally {
if (pgPool) {
await pgPool.end();
console.log("PostgreSQL连接池已关闭");
}
}
}
// 从PostgreSQL获取所有shorturl资源
async function fetchShorturlResources(
pgPool: Pool,
options: {
limit: number;
includeDeleted: boolean;
startTime?: Date;
endTime?: Date;
}
) {
let query = `
SELECT
r.id,
r.external_id,
r.type,
r.attributes,
r.schema_version,
r.creator_id,
r.created_at,
r.updated_at,
r.deleted_at,
u.email as creator_email,
u.first_name as creator_first_name,
u.last_name as creator_last_name
FROM
limq.resources r
LEFT JOIN
limq.users u ON r.creator_id = u.id
WHERE
r.type = 'shorturl'
`;
const params = [];
let paramCount = 1;
if (!options.includeDeleted) {
query += ` AND r.deleted_at IS NULL`;
}
if (options.startTime) {
query += ` AND r.created_at >= $${paramCount}`;
params.push(options.startTime);
paramCount++;
}
if (options.endTime) {
query += ` AND r.created_at <= $${paramCount}`;
params.push(options.endTime);
paramCount++;
}
query += ` ORDER BY r.created_at DESC LIMIT $${paramCount}`;
params.push(options.limit);
const client = await pgPool.connect();
try {
const result = await client.queryObject(query, params);
// 添加调试日志 - 显示获取的数据样本
if (result.rows.length > 0) {
console.log(`获取到 ${result.rows.length} 条shorturl记录`);
console.log(`第一条记录ID: ${result.rows[0].id}`);
console.log(`attributes类型: ${typeof result.rows[0].attributes}`);
console.log(`attributes内容示例: ${JSON.stringify(String(result.rows[0].attributes)).substring(0, 100)}...`);
}
return result.rows;
} finally {
client.release();
}
}
// 为每个shorturl资源获取关联数据
async function enrichShorturlData(pgPool: Pool, shorturls: Record<string, unknown>[]) {
const client = await pgPool.connect();
const enriched = [];
try {
for (const shorturl of shorturls) {
// 1. 获取项目关联
const projectsResult = await client.queryObject(`
SELECT
pr.resource_id, pr.project_id,
p.name as project_name, p.description as project_description,
pr.assigned_at
FROM
limq.project_resources pr
JOIN
limq.projects p ON pr.project_id = p.id
WHERE
pr.resource_id = $1
`, [shorturl.id]);
// 2. 获取团队关联（通过项目）
const teamIds = projectsResult.rows.map((p: Record<string, unknown>) => p.project_id);
const teamsResult = teamIds.length > 0 ? await client.queryObject(`
SELECT
tp.team_id, tp.project_id,
t.name as team_name, t.description as team_description
FROM
limq.team_projects tp
JOIN
limq.teams t ON tp.team_id = t.id
WHERE
tp.project_id = ANY($1::uuid[])
`, [teamIds]) : { rows: [] };
// 3. 获取标签关联
const tagsResult = await client.queryObject(`
SELECT
rt.resource_id, rt.tag_id, rt.created_at,
t.name as tag_name, t.type as tag_type
FROM
limq.resource_tags rt
JOIN
limq.tags t ON rt.tag_id = t.id
WHERE
rt.resource_id = $1
`, [shorturl.id]);
// 4. 获取QR码关联
const qrCodesResult = await client.queryObject(`
SELECT
id as qr_id, scan_count, url, template_name, created_at
FROM
limq.qr_code
WHERE
resource_id = $1
`, [shorturl.id]);
// 5. 获取渠道关联
const channelsResult = await client.queryObject(`
SELECT
id as channel_id, name as channel_name, path as channel_path,
"isUserCreated" as is_user_created
FROM
limq.channel
WHERE
"shortUrlId" = $1
`, [shorturl.id]);
// 6. 获取收藏关联
const favoritesResult = await client.queryObject(`
SELECT
f.id as favorite_id, f.user_id, f.created_at,
u.first_name, u.last_name
FROM
limq.favorite f
JOIN
limq.users u ON f.user_id = u.id
WHERE
f.favoritable_id = $1 AND f.favoritable_type = 'resource'
`, [shorturl.id]);
// 调试日志
console.log(`\n处理资源ID: ${shorturl.id}`);
console.log(`attributes类型: ${typeof shorturl.attributes}`);
// 改进的attributes解析逻辑
let attributes: ResourceAttributes = {};
try {
if (typeof shorturl.attributes === 'string') {
// 如果是字符串，尝试解析为JSON
console.log(`尝试解析attributes字符串，长度: ${shorturl.attributes.length}`);
attributes = JSON.parse(shorturl.attributes);
} else if (typeof shorturl.attributes === 'object' && shorturl.attributes !== null) {
// 如果已经是对象，直接使用
console.log('attributes已经是对象类型');
attributes = shorturl.attributes as ResourceAttributes;
} else {
console.log(`无效的attributes类型: ${typeof shorturl.attributes}`);
attributes = {};
}
} catch (err) {
const error = err as Error;
console.warn(`无法解析资源 ${shorturl.id} 的attributes JSON:`, error.message);
// 尝试进行更多原始数据分析
if (typeof shorturl.attributes === 'string') {
console.log(`原始字符串前100字符: ${shorturl.attributes.substring(0, 100)}`);
}
attributes = {};
}
// 尝试从QR码获取数据
let slugFromQr = "";
const urlFromQr = "";
if (qrCodesResult.rows.length > 0 && qrCodesResult.rows[0].url) {
const qrUrl = qrCodesResult.rows[0].url as string;
console.log(`找到QR码URL: ${qrUrl}`);
try {
const urlParts = qrUrl.split('/');
slugFromQr = urlParts[urlParts.length - 1];
console.log(`从QR码URL提取的slug: ${slugFromQr}`);
} catch (err) {
const error = err as Error;
console.log('无法从QR码URL提取slug:', error.message);
}
}
// 日志输出实际字段值
console.log(`提取字段 - name: ${attributes.name || 'N/A'}, slug: ${attributes.slug || 'N/A'}`);
console.log(`提取字段 - originalUrl: ${attributes.originalUrl || 'N/A'}, original_url: ${attributes.original_url || 'N/A'}`);
// 整合所有数据
const slug = attributes.slug || attributes.name || slugFromQr || "";
const originalUrl = attributes.originalUrl || attributes.original_url || urlFromQr || "";
console.log(`最终使用的slug: ${slug}`);
console.log(`最终使用的originalUrl: ${originalUrl}`);
enriched.push({
...shorturl,
attributes,
projects: projectsResult.rows,
teams: teamsResult.rows,
tags: tagsResult.rows,
qrCodes: qrCodesResult.rows,
channels: channelsResult.rows,
favorites: favoritesResult.rows,
// 从attributes中提取特定字段 - 使用改进的顺序和QR码备选
slug,
originalUrl,
title: attributes.title || "",
description: attributes.description || "",
expiresAt: attributes.expires_at || attributes.expiresAt || null
});
}
} finally {
client.release();
}
return enriched;
}
// 将PostgreSQL数据格式化为ClickHouse格式
function formatForClickhouse(shorturls: Record<string, unknown>[]) {
// 将日期格式化为ClickHouse兼容的DateTime64(3)格式
const formatDateTime = (date: Date | string | number | null | undefined): string | null => {
if (!date) return null;
// 转换为Date对象
const dateObj = date instanceof Date ? date : new Date(date);
// 返回格式化的字符串: YYYY-MM-DD HH:MM:SS.SSS
return dateObj.toISOString().replace('T', ' ').replace('Z', '');
};
console.log(`\n准备格式化 ${shorturls.length} 条记录为ClickHouse格式`);
return shorturls.map(shorturl => {
// 调试日志：输出关键字段
console.log(`处理资源: ${shorturl.id}`);
console.log(`slug: ${shorturl.slug || 'EMPTY'}`);
console.log(`originalUrl: ${shorturl.originalUrl || 'EMPTY'}`);
// 记录attributes状态
const attributesStr = JSON.stringify(shorturl.attributes || {});
const attributesPrev = attributesStr.length > 100 ?
attributesStr.substring(0, 100) + '...' :
attributesStr;
console.log(`attributes: ${attributesPrev}`);
const creatorName = [shorturl.creator_first_name, shorturl.creator_last_name]
.filter(Boolean)
.join(" ");
// 格式化项目数据为JSON数组
const projects = JSON.stringify((shorturl.projects as Record<string, unknown>[]).map((p) => ({
project_id: p.project_id,
project_name: p.project_name,
project_description: p.project_description,
assigned_at: p.assigned_at
})));
// 格式化团队数据为JSON数组
const teams = JSON.stringify((shorturl.teams as Record<string, unknown>[]).map((t) => ({
team_id: t.team_id,
team_name: t.team_name,
team_description: t.team_description,
via_project_id: t.project_id
})));
// 格式化标签数据为JSON数组
const tags = JSON.stringify((shorturl.tags as Record<string, unknown>[]).map((t) => ({
tag_id: t.tag_id,
tag_name: t.tag_name,
tag_type: t.tag_type,
created_at: t.created_at
})));
// 格式化QR码数据为JSON数组
const qrCodes = JSON.stringify((shorturl.qrCodes as Record<string, unknown>[]).map((q) => ({
qr_id: q.qr_id,
scan_count: q.scan_count,
url: q.url,
template_name: q.template_name,
created_at: q.created_at
})));
// 格式化渠道数据为JSON数组
const channels = JSON.stringify((shorturl.channels as Record<string, unknown>[]).map((c) => ({
channel_id: c.channel_id,
channel_name: c.channel_name,
channel_path: c.channel_path,
is_user_created: c.is_user_created
})));
// 格式化收藏数据为JSON数组
const favorites = JSON.stringify((shorturl.favorites as Record<string, unknown>[]).map((f) => ({
favorite_id: f.favorite_id,
user_id: f.user_id,
user_name: `${f.first_name || ""} ${f.last_name || ""}`.trim(),
created_at: f.created_at
})));
// 统计信息（可通过events表聚合或在其他地方设置）
const clickCount = (shorturl.attributes as ResourceAttributes).click_count as number || 0;
const uniqueVisitors = 0;
// 返回ClickHouse格式数据
return {
id: shorturl.id,
external_id: shorturl.external_id || "",
type: shorturl.type,
slug: shorturl.slug || "",
original_url: shorturl.originalUrl || "",
title: shorturl.title || "",
description: shorturl.description || "",
attributes: JSON.stringify(shorturl.attributes || {}),
schema_version: shorturl.schema_version || 1,
creator_id: shorturl.creator_id || "",
creator_email: shorturl.creator_email || "",
creator_name: creatorName,
created_at: formatDateTime(shorturl.created_at as Date),
updated_at: formatDateTime(shorturl.updated_at as Date),
deleted_at: formatDateTime(shorturl.deleted_at as Date | null),
projects,
teams,
tags,
qr_codes: qrCodes,
channels,
favorites,
expires_at: formatDateTime(shorturl.expiresAt as Date | null),
click_count: clickCount,
unique_visitors: uniqueVisitors
};
});
}
// 获取ClickHouse配置
async function getClickHouseConfig(): Promise<ChConfig> {
try {
// 使用getVariable而不是getResource获取ClickHouse配置
const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse");
console.log("原始ClickHouse配置:", typeof chConfigJson);
// 确保配置不为空
if (!chConfigJson) {
throw new Error("未找到ClickHouse配置");
}
// 解析JSON字符串为对象
let chConfig: ChConfig;
if (typeof chConfigJson === 'string') {
try {
chConfig = JSON.parse(chConfigJson);
} catch (parseError) {
console.error("解析JSON失败:", parseError);
throw new Error("ClickHouse配置不是有效的JSON");
}
} else {
chConfig = chConfigJson as ChConfig;
}
// 验证配置
if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) {
chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`;
console.log(`已构建ClickHouse URL: ${chConfig.clickhouse_url}`);
}
if (!chConfig.clickhouse_url) {
throw new Error("ClickHouse配置缺少URL");
}
return chConfig;
} catch (error) {
console.error("获取ClickHouse配置失败:", error);
throw error;
}
}
// 写入数据到ClickHouse
async function insertToClickhouse(data: Record<string, unknown>[]) {
if (data.length === 0) return 0;
// 获取ClickHouse连接信息
const chConfig = await getClickHouseConfig();
// 确保URL有效
if (!chConfig.clickhouse_url) {
throw new Error("无效的ClickHouse URL: 未定义");
}
console.log(`准备写入数据到ClickHouse: ${chConfig.clickhouse_url}`);
// 构建INSERT查询
const columns = Object.keys(data[0]).join(", ");
const query = `
INSERT INTO shorturl_analytics.shorturl (${columns})
FORMAT JSONEachRow
`;
// 批量插入
let inserted = 0;
const batchSize = 100;
for (let i = 0; i < data.length; i += batchSize) {
const batch = data.slice(i, i + batchSize);
// 使用JSONEachRow格式
const rows = batch.map(row => JSON.stringify(row)).join('\n');
// 使用HTTP接口执行查询
try {
console.log(`正在发送请求到: ${chConfig.clickhouse_url}`);
console.log(`认证信息: ${chConfig.clickhouse_user}:***`);
const response = await fetch(chConfig.clickhouse_url, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}`
},
body: `${query}\n${rows}`,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ClickHouse插入失败: ${errorText}`);
}
inserted += batch.length;
console.log(`已插入 ${inserted}/${data.length} 条记录`);
} catch (error) {
console.error(`请求ClickHouse时出错:`, error);
throw error;
}
}
return inserted;
}