diff --git a/app/api/shortlinks/route.ts b/app/api/shortlinks/route.ts new file mode 100644 index 0000000..198a845 --- /dev/null +++ b/app/api/shortlinks/route.ts @@ -0,0 +1,53 @@ +import { NextResponse } from 'next/server'; +import { executeQuery } from '@/lib/clickhouse'; + +export async function GET() { + try { + // Query to fetch shorturl data from ClickHouse + const query = ` + SELECT + id, + external_id, + type, + slug, + original_url, + title, + description, + attributes, + schema_version, + creator_id, + creator_email, + creator_name, + created_at, + updated_at, + deleted_at, + projects, + teams, + tags, + qr_codes AS qr_codes, + channels, + favorites, + expires_at, + click_count, + unique_visitors + FROM shorturl_analytics.shorturl + WHERE deleted_at IS NULL + ORDER BY created_at DESC + `; + + // Execute the query using the shared client + const rows = await executeQuery(query); + + // Return the data + return NextResponse.json({ + links: rows, + total: rows.length + }); + } catch (error) { + console.error('Error fetching shortlinks from ClickHouse:', error); + return NextResponse.json( + { error: 'Failed to fetch shortlinks' }, + { status: 500 } + ); + } +} \ No newline at end of file diff --git a/app/components/layout/Header.tsx b/app/components/layout/Header.tsx index 29a5e7a..3bceb9b 100644 --- a/app/components/layout/Header.tsx +++ b/app/components/layout/Header.tsx @@ -30,6 +30,23 @@ export default function Header() { ShortURL Analytics + + {user && ( + + )} {user && ( diff --git a/app/ip-test/page.tsx b/app/ip-test/page.tsx deleted file mode 100644 index abc43d5..0000000 --- a/app/ip-test/page.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import IpLocationTest from '../components/ipLocationTest'; - -export default function IpTestPage() { - return ( -
-

IP to Location Test

- -
- ); -} \ No newline at end of file diff --git a/app/links/page.tsx b/app/links/page.tsx new file mode 100644 index 0000000..e0908c8 --- /dev/null +++ b/app/links/page.tsx @@ -0,0 +1,384 @@ +"use client"; + +import { useEffect, useState } from 'react'; +import { getSupabaseClient } from '../utils/supabase'; +import { AuthChangeEvent } from '@supabase/supabase-js'; +import { Loader2, ExternalLink, Copy, Search } from 'lucide-react'; + +// Define attribute type to avoid using 'any' +interface LinkAttributes { + title?: string; + name?: string; + slug?: string; + original_url?: string; + originalUrl?: string; + visits?: number; + click_count?: number; + team_id?: string; + team_name?: string; + tags?: string[]; + [key: string]: unknown; +} + +// Define Link type based on the database schema +interface ShortLink { + id: string; + external_id: string | null; + attributes: LinkAttributes | null; + type: string | null; + creator_id: string | null; + created_at: string | null; + updated_at: string | null; + deleted_at: string | null; + schema_version: number | null; +} + +// Define ClickHouse shorturl type +interface ClickHouseShortUrl { + id: string; + external_id: string; + type: string; + slug: string; + original_url: string; + title: string; + description: string; + attributes: string; // JSON string + schema_version: number; + creator_id: string; + creator_email: string; + creator_name: string; + created_at: string; + updated_at: string; + deleted_at: string | null; + projects: string; // JSON string + teams: string; // JSON string + tags: string; // JSON string + qr_codes: string; // JSON string + channels: string; // JSON string + favorites: string; // JSON string + expires_at: string | null; + click_count: number; + unique_visitors: number; + link_attributes?: string; // Optional JSON string containing link-specific attributes +} + +// Convert ClickHouse data to ShortLink format +function convertClickHouseToShortLink(chData: ClickHouseShortUrl): ShortLink { + // Parse JSON strings + const attributesJson = JSON.parse(chData.attributes || '{}'); + const teamsJson = JSON.parse(chData.teams || '[]'); + const tagsJson = JSON.parse(chData.tags || '[]'); + + // Extract team info from the first team if available + const teamInfo = teamsJson.length > 0 ? { + team_id: teamsJson[0].team_id, + team_name: teamsJson[0].team_name + } : {}; + + // Extract tag names + const tagNames = tagsJson.map((tag: { tag_name: string }) => tag.tag_name); + + // Parse link_attributes to get domain if available + let domain = 'shorturl.example.com'; + try { + if (chData.link_attributes) { + const linkAttrObj = JSON.parse(chData.link_attributes); + if (linkAttrObj.domain) { + domain = linkAttrObj.domain; + } + } + + // Extract domain from shortUrl in attributes if available + const attributesObj = JSON.parse(chData.attributes || '{}'); + if (attributesObj.shortUrl) { + try { + const urlObj = new URL(attributesObj.shortUrl); + domain = urlObj.hostname; + } catch (e) { + console.error('Error parsing shortUrl:', e); + } + } + } catch (e) { + console.error('Error parsing link_attributes:', e); + } + + return { + id: chData.id, + external_id: chData.external_id, + type: chData.type, + creator_id: chData.creator_id, + created_at: chData.created_at, + updated_at: chData.updated_at, + deleted_at: chData.deleted_at, + schema_version: chData.schema_version, + attributes: { + ...attributesJson, + title: chData.title || attributesJson.title || '', + slug: chData.slug, + original_url: chData.original_url, + click_count: chData.click_count, + visits: chData.click_count, + unique_visitors: chData.unique_visitors, + domain: attributesJson.domain || domain, + ...teamInfo, + tags: tagNames + } + }; +} + +export default function LinksPage() { + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [links, setLinks] = useState([]); + const [searchQuery, setSearchQuery] = useState(''); + const [teamFilter, setTeamFilter] = useState(null); + + // Copy link to clipboard + const copyToClipboard = (text: string) => { + navigator.clipboard.writeText(text).then(() => { + // Could add a notification here + console.log('Copied to clipboard'); + }); + }; + + // Extract link metadata from attributes + const getLinkMetadata = (link: ShortLink) => { + const attrs = link.attributes || {}; + return { + title: attrs.title || attrs.name || 'Untitled Link', + slug: attrs.slug || 'no-slug', + originalUrl: attrs.original_url || attrs.originalUrl || '#', + visits: attrs.visits || attrs.click_count || 0, + teamId: attrs.team_id || '', + teamName: attrs.team_name || 'Personal', + createdAt: new Date(link.created_at || Date.now()).toLocaleDateString(), + tags: attrs.tags || [], + domain: attrs.domain || 'shorturl.example.com', + }; + }; + + // Filter links by search query + const filteredLinks = links.length > 0 ? + links.filter(link => { + if (!searchQuery && !teamFilter) return true; + + const metadata = getLinkMetadata(link); + const matchesSearch = searchQuery ? + (metadata.title.toLowerCase().includes(searchQuery.toLowerCase()) || + metadata.slug.toLowerCase().includes(searchQuery.toLowerCase()) || + metadata.originalUrl.toLowerCase().includes(searchQuery.toLowerCase())) : + true; + + const matchesTeam = teamFilter ? + metadata.teamId === teamFilter : + true; + + return matchesSearch && matchesTeam; + }) : []; + + // Get unique teams for filtering + const teams = links.length > 0 ? + Array.from( + links.reduce((teams, link) => { + const metadata = getLinkMetadata(link); + if (metadata.teamId) { + teams.set(metadata.teamId, metadata.teamName); + } + return teams; + }, new Map()) + ).map(([id, name]) => ({ id, name })) : []; + + useEffect(() => { + let isMounted = true; + + const fetchLinks = async () => { + if (!isMounted) return; + setLoading(true); + setError(null); + + try { + // Fetch data from ClickHouse API instead of Supabase + const response = await fetch('/api/shortlinks'); + + if (!response.ok) { + throw new Error(`Failed to fetch links: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + + if (!data || !data.links || data.links.length === 0) { + if (isMounted) setLinks([]); + return; + } + + // Convert ClickHouse data format to ShortLink format + const convertedLinks = data.links.map(convertClickHouseToShortLink); + + if (isMounted) { + setLinks(convertedLinks); + } + } catch (err) { + if (isMounted) { + setError(err instanceof Error ? err.message : 'Failed to load short URLs'); + console.error("Error fetching links:", err); + } + } finally { + if (isMounted) { + setLoading(false); + } + } + }; + + const supabase = getSupabaseClient(); + + const { data: { subscription } } = supabase.auth.onAuthStateChange((event: AuthChangeEvent) => { + if (event === 'SIGNED_IN') { + fetchLinks(); + } else if (event === 'SIGNED_OUT') { + setLinks([]); + setError(null); + } + }); + + supabase.auth.getSession().then(() => { + fetchLinks(); + }); + + return () => { + isMounted = false; + subscription.unsubscribe(); + }; + }, []); + + if (loading) { + return ( +
+ +
+ ); + } + + if (error) { + return ( +
+

Error: {error}

+
+ ); + } + + if (links.length === 0) { + return ( +
+

No short links found

+

Create your first short URL to get started

+
+ ); + } + + return ( +
+

Short URL Links

+ + {/* Search and filters */} +
+
+ + setSearchQuery(e.target.value)} + className="w-full rounded-md border border-gray-300 py-2 pl-10 pr-4 text-sm focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500" + /> +
+ + +
+ + {/* Links table */} +
+ + + + + + + + + + + + + {filteredLinks.map(link => { + const metadata = getLinkMetadata(link); + const shortUrl = `https://${metadata.domain}/${metadata.slug}`; + + return ( + + + + + + + + + ); + })} + +
LinkOriginal URLTeamCreatedVisitsActions
+
+ {metadata.title} + {shortUrl} +
+ 		+ + {metadata.originalUrl} + + + + {metadata.teamName} + + {metadata.createdAt} + + {metadata.visits} + +
+ + + + +
+
+
+ + {/* Pagination or "Load More" could be added here */} + + {filteredLinks.length === 0 && links.length > 0 && ( +
+ No links match your search criteria +
+ )} +
+ ); +} \ No newline at end of file diff --git a/scripts/db/db-reports/clickhouse-schema-2025-03-20T13-57-59-013Z.log b/scripts/db/db-reports/clickhouse-schema-2025-03-20T13-57-59-013Z.log deleted file mode 100644 index 05c9b73..0000000 --- a/scripts/db/db-reports/clickhouse-schema-2025-03-20T13-57-59-013Z.log +++ /dev/null @@ -1,225 +0,0 @@ - -获取所有表... -数据库 limq 中找到以下表: - - .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb - - .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc - - .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1 - - .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0 - - .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024 - - .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea - - link_daily_stats - - link_events - - link_hourly_patterns - - links - - platform_distribution - - project_daily_stats - - projects - - qr_scans - - qrcode_daily_stats - - qrcodes - - sessions - - team_daily_stats - - team_members - - teams - -所有ClickHouse表: -.inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb, .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc, .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1, .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0, .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024, .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea, link_daily_stats, link_events, link_hourly_patterns, links, platform_distribution, project_daily_stats, projects, qr_scans, qrcode_daily_stats, qrcodes, sessions, team_daily_stats, team_members, teams - -获取表 .inner_id.5d9e5f95-ad7d-4750-ae56-bffea63e14fb 的结构... - -获取表 .inner_id.711eb652-7c90-4f9a-80a0-8979011080cc 的结构... - -获取表 .inner_id.abec445d-1704-4482-bc72-66c9eb67ecd1 的结构... - -获取表 .inner_id.c1eb844d-7f11-4cfc-8931-c433faaa16b0 的结构... - -获取表 .inner_id.f9640e70-5b7f-444c-80de-bc5b25848024 的结构... - -获取表 .inner_id.fe81eeba-acc5-4260-ac9a-973c2f9ce1ea 的结构... - -获取表 link_daily_stats 的结构... -表 link_daily_stats 的列: - - date (Date, 无默认值) - - link_id (String, 无默认值) - - total_clicks (UInt64, 无默认值) - - unique_visitors (UInt64, 无默认值) - - unique_sessions (UInt64, 无默认值) - - total_time_spent (UInt64, 无默认值) - - avg_time_spent (Float64, 无默认值) - - bounce_count (UInt64, 无默认值) - - conversion_count (UInt64, 无默认值) - - unique_referrers (UInt64, 无默认值) - - mobile_count (UInt64, 无默认值) - - tablet_count (UInt64, 无默认值) - - desktop_count (UInt64, 无默认值) - - qr_scan_count (UInt64, 无默认值) - - total_conversion_value (Float64, 无默认值) - -获取表 link_events 的结构... -表 link_events 的列: - - event_id (UUID, 默认值: generateUUIDv4()) - - event_time (DateTime64(3), 默认值: now64()) - - date (Date, 默认值: toDate(event_time)) - - link_id (String, 无默认值) - - channel_id (String, 无默认值) - - visitor_id (String, 无默认值) - - session_id (String, 无默认值) - - event_type (Enum8('click' = 1, 'redirect' = 2, 'conversion' = 3, 'error' = 4), 无默认值) - - ip_address (String, 无默认值) - - country (String, 无默认值) - - city (String, 无默认值) - - referrer (String, 无默认值) - - utm_source (String, 无默认值) - - utm_medium (String, 无默认值) - - utm_campaign (String, 无默认值) - - user_agent (String, 无默认值) - - device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值) - - browser (String, 无默认值) - - os (String, 无默认值) - - time_spent_sec (UInt32, 默认值: 0) - - is_bounce (Bool, 默认值: true) - - is_qr_scan (Bool, 默认值: false) - - qr_code_id (String, 默认值: '') - - conversion_type (Enum8('visit' = 1, 'stay' = 2, 'interact' = 3, 'signup' = 4, 'subscription' = 5, 'purchase' = 6), 默认值: 'visit') - - conversion_value (Float64, 默认值: 0) - - custom_data (String, 默认值: '{}') - -获取表 link_hourly_patterns 的结构... -表 link_hourly_patterns 的列: - - date (Date, 无默认值) - - hour (UInt8, 无默认值) - - link_id (String, 无默认值) - - visits (UInt64, 无默认值) - - unique_visitors (UInt64, 无默认值) - -获取表 links 的结构... -表 links 的列: - - link_id (String, 无默认值) - - original_url (String, 无默认值) - - created_at (DateTime64(3), 无默认值) - - created_by (String, 无默认值) - - title (String, 无默认值) - - description (String, 无默认值) - - tags (Array(String), 无默认值) - - is_active (Bool, 默认值: true) - - expires_at (Nullable(DateTime), 无默认值) - - team_id (String, 默认值: '') - - project_id (String, 默认值: '') - -获取表 platform_distribution 的结构... -表 platform_distribution 的列: - - date (Date, 无默认值) - - utm_source (String, 无默认值) - - device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值) - - visits (UInt64, 无默认值) - - unique_visitors (UInt64, 无默认值) - -获取表 project_daily_stats 的结构... -表 project_daily_stats 的列: - - date (Date, 无默认值) - - project_id (String, 无默认值) - - total_clicks (UInt64, 无默认值) - - unique_visitors (UInt64, 无默认值) - - conversion_count (UInt64, 无默认值) - - links_used (UInt64, 无默认值) - - qr_scan_count (UInt64, 无默认值) - -获取表 projects 的结构... -表 projects 的列: - - project_id (String, 无默认值) - - team_id (String, 无默认值) - - name (String, 无默认值) - - created_at (DateTime, 无默认值) - - created_by (String, 无默认值) - - description (String, 默认值: '') - - is_archived (Bool, 默认值: false) - - links_count (UInt32, 默认值: 0) - - total_clicks (UInt64, 默认值: 0) - - last_updated (DateTime, 默认值: now()) - -获取表 qr_scans 的结构... -表 qr_scans 的列: - - scan_id (UUID, 默认值: generateUUIDv4()) - - qr_code_id (String, 无默认值) - - link_id (String, 无默认值) - - scan_time (DateTime64(3), 无默认值) - - visitor_id (String, 无默认值) - - location (String, 无默认值) - - device_type (Enum8('mobile' = 1, 'tablet' = 2, 'desktop' = 3, 'other' = 4), 无默认值) - - led_to_conversion (Bool, 默认值: false) - -获取表 qrcode_daily_stats 的结构... -表 qrcode_daily_stats 的列: - - date (Date, 无默认值) - - qr_code_id (String, 无默认值) - - total_scans (UInt64, 无默认值) - - unique_scanners (UInt64, 无默认值) - - conversions (UInt64, 无默认值) - - mobile_scans (UInt64, 无默认值) - - tablet_scans (UInt64, 无默认值) - - desktop_scans (UInt64, 无默认值) - - unique_locations (UInt64, 无默认值) - -获取表 qrcodes 的结构... -表 qrcodes 的列: - - qr_code_id (String, 无默认值) - - link_id (String, 无默认值) - - team_id (String, 无默认值) - - project_id (String, 默认值: '') - - name (String, 无默认值) - - description (String, 默认值: '') - - created_at (DateTime, 无默认值) - - created_by (String, 无默认值) - - updated_at (DateTime, 默认值: now()) - - qr_type (Enum8('standard' = 1, 'custom' = 2, 'dynamic' = 3), 默认值: 'standard') - - image_url (String, 默认值: '') - - design_config (String, 默认值: '{}') - - is_active (Bool, 默认值: true) - - total_scans (UInt64, 默认值: 0) - - unique_scanners (UInt32, 默认值: 0) - -获取表 sessions 的结构... -表 sessions 的列: - - session_id (String, 无默认值) - - visitor_id (String, 无默认值) - - link_id (String, 无默认值) - - started_at (DateTime64(3), 无默认值) - - last_activity (DateTime64(3), 无默认值) - - ended_at (Nullable(DateTime64(3)), 无默认值) - - duration_sec (UInt32, 默认值: 0) - - session_pages (UInt8, 默认值: 1) - - is_completed (Bool, 默认值: false) - -获取表 team_daily_stats 的结构... -表 team_daily_stats 的列: - - date (Date, 无默认值) - - team_id (String, 无默认值) - - total_clicks (UInt64, 无默认值) - - unique_visitors (UInt64, 无默认值) - - conversion_count (UInt64, 无默认值) - - links_used (UInt64, 无默认值) - - qr_scan_count (UInt64, 无默认值) - -获取表 team_members 的结构... -表 team_members 的列: - - team_id (String, 无默认值) - - user_id (String, 无默认值) - - role (Enum8('owner' = 1, 'admin' = 2, 'editor' = 3, 'viewer' = 4), 无默认值) - - joined_at (DateTime, 默认值: now()) - - invited_by (String, 无默认值) - - is_active (Bool, 默认值: true) - - last_active (DateTime, 默认值: now()) - -获取表 teams 的结构... -表 teams 的列: - - team_id (String, 无默认值) - - name (String, 无默认值) - - created_at (DateTime, 无默认值) - - created_by (String, 无默认值) - - description (String, 默认值: '') - - avatar_url (String, 默认值: '') - - is_active (Bool, 默认值: true) - - plan_type (Enum8('free' = 1, 'pro' = 2, 'enterprise' = 3), 无默认值) - - members_count (UInt32, 默认值: 1) - -ClickHouse数据库结构检查完成 diff --git a/scripts/db/sql/clickhouse/create_shorturl_table.sql b/scripts/db/sql/clickhouse/create_shorturl_table.sql new file mode 100644 index 0000000..ccd95c4 --- /dev/null +++ b/scripts/db/sql/clickhouse/create_shorturl_table.sql @@ -0,0 +1,46 @@ +-- 使用shorturl_analytics数据库 +USE shorturl_analytics; + +-- 删除已存在的shorturl表 +DROP TABLE IF EXISTS shorturl_analytics.shorturl; + +-- 创建shorturl表 +CREATE TABLE IF NOT EXISTS shorturl_analytics.shorturl ( + -- 短链接基本信息（来源于resources表） + id String COMMENT '资源ID (resources.id)', + external_id String COMMENT '外部ID (resources.external_id)', + type String COMMENT '类型，值为shorturl', + slug String COMMENT '短链接slug (存储在attributes中)', + original_url String COMMENT '原始URL (存储在attributes中)', + title String COMMENT '标题 (存储在attributes中)', + description String COMMENT '描述 (存储在attributes中)', + attributes String DEFAULT '{}' COMMENT '资源属性JSON', + schema_version Int32 COMMENT 'Schema版本', + -- 创建者信息 + creator_id String COMMENT '创建者ID (resources.creator_id)', + creator_email String COMMENT '创建者邮箱 (来自users表)', + creator_name String COMMENT '创建者名称 (来自users表)', + -- 时间信息 + created_at DateTime64(3) COMMENT '创建时间 (resources.created_at)', + updated_at DateTime64(3) COMMENT '更新时间 (resources.updated_at)', + deleted_at Nullable(DateTime64(3)) COMMENT '删除时间 (resources.deleted_at)', + -- 项目关联 (project_resources表) + projects String DEFAULT '[]' COMMENT '项目关联信息数组。结构: [{project_id: String, project_name: String, project_description: String, assigned_at: DateTime64}]', + -- 团队关联 (通过项目关联到团队) + teams String DEFAULT '[]' COMMENT '团队关联信息数组。结构: [{team_id: String, team_name: String, team_description: String, via_project_id: String}]', + -- 标签关联 (resource_tags表) + tags String DEFAULT '[]' COMMENT '标签关联信息数组。结构: [{tag_id: String, tag_name: String, tag_type: String, created_at: DateTime64}]', + -- QR码关联 (qr_code表) + qr_codes String DEFAULT '[]' COMMENT 'QR码信息数组。结构: [{qr_id: String, scan_count: Int32, url: String, template_name: String, created_at: DateTime64}]', + -- 渠道关联 (channel表) + channels String DEFAULT '[]' COMMENT '渠道信息数组。结构: [{channel_id: String, channel_name: String, channel_path: String, is_user_created: Boolean}]', + -- 收藏关联 (favorite表) + favorites String DEFAULT '[]' COMMENT '收藏信息数组。结构: [{favorite_id: String, user_id: String, user_name: String, created_at: DateTime64}]', + -- 自定义过期时间 (存储在attributes中) + expires_at Nullable(DateTime64(3)) COMMENT '过期时间', + -- 统计信息 (分析时聚合计算) + click_count UInt32 DEFAULT 0 COMMENT '点击次数', + unique_visitors UInt32 DEFAULT 0 COMMENT '唯一访问者数' +) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at) +ORDER BY + (id, created_at) SETTINGS index_granularity = 8192 COMMENT '用于存储所有shorturl类型资源的统一表，集成了相关联的项目、团队、标签、QR码、渠道和收藏信息'; \ No newline at end of file diff --git a/windmill/sync_shorturl_to_clickhouse.ts b/windmill/sync_shorturl_to_clickhouse.ts new file mode 100644 index 0000000..9268889 --- /dev/null +++ b/windmill/sync_shorturl_to_clickhouse.ts @@ -0,0 +1,600 @@ +// Windmill script to sync shorturl data from PostgreSQL to ClickHouse +// 作者: AI Assistant +// 创建日期: 2023-10-30 +// 描述: 此脚本从PostgreSQL数据库获取所有shorturl类型的资源及其关联数据，并同步到ClickHouse + +import { Pool } from "https://deno.land/x/postgres@v0.17.0/mod.ts"; +import { getResource, getVariable } from "https://deno.land/x/windmill@v1.183.0/mod.ts"; + +// 资源属性接口 +interface ResourceAttributes { + slug?: string; + original_url?: string; + originalUrl?: string; + title?: string; + description?: string; + expires_at?: string; + expiresAt?: string; + [key: string]: unknown; +} + +// ClickHouse配置接口 +interface ChConfig { + clickhouse_host: string; + clickhouse_port: number; + clickhouse_user: string; + clickhouse_password: string; + clickhouse_url: string; +} + +// PostgreSQL配置接口 +interface PgConfig { + host: string; + port: number; + user: string; + password: string; + dbname?: string; + [key: string]: unknown; +} + +// Windmill函数定义 +export async function main( + /** PostgreSQL和ClickHouse同步脚本 */ + params: { + /** 同步的资源数量限制，默认500 */ + limit?: number; + /** 是否包含已删除资源 */ + includeDeleted?: boolean; + /** 是否执行实际写入操作 */ + dryRun?: boolean; + /** 开始时间（ISO格式）*/ + startTime?: string; + /** 结束时间（ISO格式）*/ + endTime?: string; + } +) { + // 设置默认参数 + const limit = params.limit || 500; + const includeDeleted = params.includeDeleted || false; + const dryRun = params.dryRun || false; + const startTime = params.startTime ? new Date(params.startTime) : undefined; + const endTime = params.endTime ? new Date(params.endTime) : undefined; + + console.log(`开始同步PostgreSQL shorturl数据到ClickHouse`); + console.log(`参数: limit=${limit}, includeDeleted=${includeDeleted}, dryRun=${dryRun}`); + if (startTime) console.log(`开始时间: ${startTime.toISOString()}`); + if (endTime) console.log(`结束时间: ${endTime.toISOString()}`); + + // 获取数据库配置 + console.log("获取PostgreSQL数据库配置..."); + const pgConfig = await getResource('f/limq/postgresql') as PgConfig; + console.log(`数据库连接配置: host=${pgConfig.host}, port=${pgConfig.port}, database=${pgConfig.dbname || 'postgres'}, user=${pgConfig.user}`); + + let pgPool: Pool | null = null; + + try { + console.log("创建PostgreSQL连接池..."); + + pgPool = new Pool({ + hostname: pgConfig.host, + port: pgConfig.port, + user: pgConfig.user, + password: pgConfig.password, + database: pgConfig.dbname || 'postgres' + }, 3); + + console.log("PostgreSQL连接池创建完成，尝试连接..."); + + // 测试连接 + const client = await pgPool.connect(); + try { + console.log("连接成功，执行测试查询..."); + const testResult = await client.queryObject(`SELECT 1 AS test`); + console.log(`测试查询结果: ${JSON.stringify(testResult.rows)}`); + } finally { + client.release(); + } + + // 获取所有shorturl类型的资源 + const shorturls = await fetchShorturlResources(pgPool, { + limit, + includeDeleted, + startTime, + endTime, + }); + + console.log(`获取到 ${shorturls.length} 个shorturl资源`); + + if (shorturls.length === 0) { + return { synced: 0, message: "没有找到需要同步的shorturl资源" }; + } + + // 为每个资源获取关联数据 + const enrichedShorturls = await enrichShorturlData(pgPool, shorturls); + console.log(`已丰富 ${enrichedShorturls.length} 个shorturl资源的关联数据`); + + // 转换为ClickHouse格式 + const clickhouseData = formatForClickhouse(enrichedShorturls); + + if (!dryRun) { + // 写入ClickHouse + const inserted = await insertToClickhouse(clickhouseData); + console.log(`成功写入 ${inserted} 条记录到ClickHouse`); + return { synced: inserted, message: "同步完成" }; + } else { + console.log("Dry run模式 - 不执行实际写入"); + console.log(`将写入 ${clickhouseData.length} 条记录到ClickHouse`); + // 输出示例数据 + if (clickhouseData.length > 0) { + console.log("示例数据:"); + console.log(JSON.stringify(clickhouseData[0], null, 2)); + } + return { synced: 0, dryRun: true, sampleData: clickhouseData.slice(0, 1) }; + } + } catch (error: unknown) { + console.error(`同步过程中发生错误: ${(error as Error).message}`); + console.error(`错误类型: ${(error as Error).name}`); + if ((error as Error).stack) { + console.error(`错误堆栈: ${(error as Error).stack}`); + } + throw error; + } finally { + if (pgPool) { + await pgPool.end(); + console.log("PostgreSQL连接池已关闭"); + } + } +} + +// 从PostgreSQL获取所有shorturl资源 +async function fetchShorturlResources( + pgPool: Pool, + options: { + limit: number; + includeDeleted: boolean; + startTime?: Date; + endTime?: Date; + } +) { + let query = ` + SELECT + r.id, + r.external_id, + r.type, + r.attributes, + r.schema_version, + r.creator_id, + r.created_at, + r.updated_at, + r.deleted_at, + u.email as creator_email, + u.first_name as creator_first_name, + u.last_name as creator_last_name + FROM + limq.resources r + LEFT JOIN + limq.users u ON r.creator_id = u.id + WHERE + r.type = 'shorturl' + `; + + const params = []; + let paramCount = 1; + + if (!options.includeDeleted) { + query += ` AND r.deleted_at IS NULL`; + } + + if (options.startTime) { + query += ` AND r.created_at >= $${paramCount}`; + params.push(options.startTime); + paramCount++; + } + + if (options.endTime) { + query += ` AND r.created_at <= $${paramCount}`; + params.push(options.endTime); + paramCount++; + } + + query += ` ORDER BY r.created_at DESC LIMIT $${paramCount}`; + params.push(options.limit); + + const client = await pgPool.connect(); + try { + const result = await client.queryObject(query, params); + + // 添加调试日志 - 显示获取的数据样本 + if (result.rows.length > 0) { + console.log(`获取到 ${result.rows.length} 条shorturl记录`); + console.log(`第一条记录ID: ${result.rows[0].id}`); + console.log(`attributes类型: ${typeof result.rows[0].attributes}`); + console.log(`attributes内容示例: ${JSON.stringify(String(result.rows[0].attributes)).substring(0, 100)}...`); + } + + return result.rows; + } finally { + client.release(); + } +} + +// 为每个shorturl资源获取关联数据 +async function enrichShorturlData(pgPool: Pool, shorturls: Record[]) { + const client = await pgPool.connect(); + const enriched = []; + + try { + for (const shorturl of shorturls) { + // 1. 获取项目关联 + const projectsResult = await client.queryObject(` + SELECT + pr.resource_id, pr.project_id, + p.name as project_name, p.description as project_description, + pr.assigned_at + FROM + limq.project_resources pr + JOIN + limq.projects p ON pr.project_id = p.id + WHERE + pr.resource_id = $1 + `, [shorturl.id]); + + // 2. 获取团队关联（通过项目） + const teamIds = projectsResult.rows.map((p: Record) => p.project_id); + const teamsResult = teamIds.length > 0 ? await client.queryObject(` + SELECT + tp.team_id, tp.project_id, + t.name as team_name, t.description as team_description + FROM + limq.team_projects tp + JOIN + limq.teams t ON tp.team_id = t.id + WHERE + tp.project_id = ANY($1::uuid[]) + `, [teamIds]) : { rows: [] }; + + // 3. 获取标签关联 + const tagsResult = await client.queryObject(` + SELECT + rt.resource_id, rt.tag_id, rt.created_at, + t.name as tag_name, t.type as tag_type + FROM + limq.resource_tags rt + JOIN + limq.tags t ON rt.tag_id = t.id + WHERE + rt.resource_id = $1 + `, [shorturl.id]); + + // 4. 获取QR码关联 + const qrCodesResult = await client.queryObject(` + SELECT + id as qr_id, scan_count, url, template_name, created_at + FROM + limq.qr_code + WHERE + resource_id = $1 + `, [shorturl.id]); + + // 5. 获取渠道关联 + const channelsResult = await client.queryObject(` + SELECT + id as channel_id, name as channel_name, path as channel_path, + "isUserCreated" as is_user_created + FROM + limq.channel + WHERE + "shortUrlId" = $1 + `, [shorturl.id]); + + // 6. 获取收藏关联 + const favoritesResult = await client.queryObject(` + SELECT + f.id as favorite_id, f.user_id, f.created_at, + u.first_name, u.last_name + FROM + limq.favorite f + JOIN + limq.users u ON f.user_id = u.id + WHERE + f.favoritable_id = $1 AND f.favoritable_type = 'resource' + `, [shorturl.id]); + + // 调试日志 + console.log(`\n处理资源ID: ${shorturl.id}`); + console.log(`attributes类型: ${typeof shorturl.attributes}`); + + // 改进的attributes解析逻辑 + let attributes: ResourceAttributes = {}; + try { + if (typeof shorturl.attributes === 'string') { + // 如果是字符串，尝试解析为JSON + console.log(`尝试解析attributes字符串，长度: ${shorturl.attributes.length}`); + attributes = JSON.parse(shorturl.attributes); + } else if (typeof shorturl.attributes === 'object' && shorturl.attributes !== null) { + // 如果已经是对象，直接使用 + console.log('attributes已经是对象类型'); + attributes = shorturl.attributes as ResourceAttributes; + } else { + console.log(`无效的attributes类型: ${typeof shorturl.attributes}`); + attributes = {}; + } + } catch (err) { + const error = err as Error; + console.warn(`无法解析资源 ${shorturl.id} 的attributes JSON:`, error.message); + // 尝试进行更多原始数据分析 + if (typeof shorturl.attributes === 'string') { + console.log(`原始字符串前100字符: ${shorturl.attributes.substring(0, 100)}`); + } + attributes = {}; + } + + // 尝试从QR码获取数据 + let slugFromQr = ""; + const urlFromQr = ""; + + if (qrCodesResult.rows.length > 0 && qrCodesResult.rows[0].url) { + const qrUrl = qrCodesResult.rows[0].url as string; + console.log(`找到QR码URL: ${qrUrl}`); + + try { + const urlParts = qrUrl.split('/'); + slugFromQr = urlParts[urlParts.length - 1]; + console.log(`从QR码URL提取的slug: ${slugFromQr}`); + } catch (err) { + const error = err as Error; + console.log('无法从QR码URL提取slug:', error.message); + } + } + + // 日志输出实际字段值 + console.log(`提取字段 - name: ${attributes.name || 'N/A'}, slug: ${attributes.slug || 'N/A'}`); + console.log(`提取字段 - originalUrl: ${attributes.originalUrl || 'N/A'}, original_url: ${attributes.original_url || 'N/A'}`); + + // 整合所有数据 + const slug = attributes.slug || attributes.name || slugFromQr || ""; + const originalUrl = attributes.originalUrl || attributes.original_url || urlFromQr || ""; + + console.log(`最终使用的slug: ${slug}`); + console.log(`最终使用的originalUrl: ${originalUrl}`); + + enriched.push({ + ...shorturl, + attributes, + projects: projectsResult.rows, + teams: teamsResult.rows, + tags: tagsResult.rows, + qrCodes: qrCodesResult.rows, + channels: channelsResult.rows, + favorites: favoritesResult.rows, + // 从attributes中提取特定字段 - 使用改进的顺序和QR码备选 + slug, + originalUrl, + title: attributes.title || "", + description: attributes.description || "", + expiresAt: attributes.expires_at || attributes.expiresAt || null + }); + } + } finally { + client.release(); + } + + return enriched; +} + +// 将PostgreSQL数据格式化为ClickHouse格式 +function formatForClickhouse(shorturls: Record[]) { + // 将日期格式化为ClickHouse兼容的DateTime64(3)格式 + const formatDateTime = (date: Date | string | number | null | undefined): string | null => { + if (!date) return null; + // 转换为Date对象 + const dateObj = date instanceof Date ? date : new Date(date); + // 返回格式化的字符串: YYYY-MM-DD HH:MM:SS.SSS + return dateObj.toISOString().replace('T', ' ').replace('Z', ''); + }; + + console.log(`\n准备格式化 ${shorturls.length} 条记录为ClickHouse格式`); + + return shorturls.map(shorturl => { + // 调试日志：输出关键字段 + console.log(`处理资源: ${shorturl.id}`); + console.log(`slug: ${shorturl.slug || 'EMPTY'}`); + console.log(`originalUrl: ${shorturl.originalUrl || 'EMPTY'}`); + + // 记录attributes状态 + const attributesStr = JSON.stringify(shorturl.attributes || {}); + const attributesPrev = attributesStr.length > 100 ? + attributesStr.substring(0, 100) + '...' : + attributesStr; + console.log(`attributes: ${attributesPrev}`); + + const creatorName = [shorturl.creator_first_name, shorturl.creator_last_name] + .filter(Boolean) + .join(" "); + + // 格式化项目数据为JSON数组 + const projects = JSON.stringify((shorturl.projects as Record[]).map((p) => ({ + project_id: p.project_id, + project_name: p.project_name, + project_description: p.project_description, + assigned_at: p.assigned_at + }))); + + // 格式化团队数据为JSON数组 + const teams = JSON.stringify((shorturl.teams as Record[]).map((t) => ({ + team_id: t.team_id, + team_name: t.team_name, + team_description: t.team_description, + via_project_id: t.project_id + }))); + + // 格式化标签数据为JSON数组 + const tags = JSON.stringify((shorturl.tags as Record[]).map((t) => ({ + tag_id: t.tag_id, + tag_name: t.tag_name, + tag_type: t.tag_type, + created_at: t.created_at + }))); + + // 格式化QR码数据为JSON数组 + const qrCodes = JSON.stringify((shorturl.qrCodes as Record[]).map((q) => ({ + qr_id: q.qr_id, + scan_count: q.scan_count, + url: q.url, + template_name: q.template_name, + created_at: q.created_at + }))); + + // 格式化渠道数据为JSON数组 + const channels = JSON.stringify((shorturl.channels as Record[]).map((c) => ({ + channel_id: c.channel_id, + channel_name: c.channel_name, + channel_path: c.channel_path, + is_user_created: c.is_user_created + }))); + + // 格式化收藏数据为JSON数组 + const favorites = JSON.stringify((shorturl.favorites as Record[]).map((f) => ({ + favorite_id: f.favorite_id, + user_id: f.user_id, + user_name: `${f.first_name || ""} ${f.last_name || ""}`.trim(), + created_at: f.created_at + }))); + + // 统计信息（可通过events表聚合或在其他地方设置） + const clickCount = (shorturl.attributes as ResourceAttributes).click_count as number || 0; + const uniqueVisitors = 0; + + // 返回ClickHouse格式数据 + return { + id: shorturl.id, + external_id: shorturl.external_id || "", + type: shorturl.type, + slug: shorturl.slug || "", + original_url: shorturl.originalUrl || "", + title: shorturl.title || "", + description: shorturl.description || "", + attributes: JSON.stringify(shorturl.attributes || {}), + schema_version: shorturl.schema_version || 1, + creator_id: shorturl.creator_id || "", + creator_email: shorturl.creator_email || "", + creator_name: creatorName, + created_at: formatDateTime(shorturl.created_at as Date), + updated_at: formatDateTime(shorturl.updated_at as Date), + deleted_at: formatDateTime(shorturl.deleted_at as Date | null), + projects, + teams, + tags, + qr_codes: qrCodes, + channels, + favorites, + expires_at: formatDateTime(shorturl.expiresAt as Date | null), + click_count: clickCount, + unique_visitors: uniqueVisitors + }; + }); +} + +// 获取ClickHouse配置 +async function getClickHouseConfig(): Promise { + try { + // 使用getVariable而不是getResource获取ClickHouse配置 + const chConfigJson = await getVariable("f/shorturl_analytics/clickhouse"); + console.log("原始ClickHouse配置:", typeof chConfigJson); + + // 确保配置不为空 + if (!chConfigJson) { + throw new Error("未找到ClickHouse配置"); + } + + // 解析JSON字符串为对象 + let chConfig: ChConfig; + if (typeof chConfigJson === 'string') { + try { + chConfig = JSON.parse(chConfigJson); + } catch (parseError) { + console.error("解析JSON失败:", parseError); + throw new Error("ClickHouse配置不是有效的JSON"); + } + } else { + chConfig = chConfigJson as ChConfig; + } + + // 验证配置 + if (!chConfig.clickhouse_url && chConfig.clickhouse_host && chConfig.clickhouse_port) { + chConfig.clickhouse_url = `http://${chConfig.clickhouse_host}:${chConfig.clickhouse_port}`; + console.log(`已构建ClickHouse URL: ${chConfig.clickhouse_url}`); + } + + if (!chConfig.clickhouse_url) { + throw new Error("ClickHouse配置缺少URL"); + } + + return chConfig; + } catch (error) { + console.error("获取ClickHouse配置失败:", error); + throw error; + } +} + +// 写入数据到ClickHouse +async function insertToClickhouse(data: Record[]) { + if (data.length === 0) return 0; + + // 获取ClickHouse连接信息 + const chConfig = await getClickHouseConfig(); + + // 确保URL有效 + if (!chConfig.clickhouse_url) { + throw new Error("无效的ClickHouse URL: 未定义"); + } + + console.log(`准备写入数据到ClickHouse: ${chConfig.clickhouse_url}`); + + // 构建INSERT查询 + const columns = Object.keys(data[0]).join(", "); + + const query = ` + INSERT INTO shorturl_analytics.shorturl (${columns}) + FORMAT JSONEachRow + `; + + // 批量插入 + let inserted = 0; + const batchSize = 100; + + for (let i = 0; i < data.length; i += batchSize) { + const batch = data.slice(i, i + batchSize); + + // 使用JSONEachRow格式 + const rows = batch.map(row => JSON.stringify(row)).join('\n'); + + // 使用HTTP接口执行查询 + try { + console.log(`正在发送请求到: ${chConfig.clickhouse_url}`); + console.log(`认证信息: ${chConfig.clickhouse_user}:***`); + + const response = await fetch(chConfig.clickhouse_url, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": `Basic ${btoa(`${chConfig.clickhouse_user}:${chConfig.clickhouse_password}`)}` + }, + body: `${query}\n${rows}`, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`ClickHouse插入失败: ${errorText}`); + } + + inserted += batch.length; + console.log(`已插入 ${inserted}/${data.length} 条记录`); + } catch (error) { + console.error(`请求ClickHouse时出错:`, error); + throw error; + } + } + + return inserted; +} \ No newline at end of file