vitalitymailg/shorturl-analytics
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init ana page with apis

Browse Source
This commit is contained in:
William Tso
2025-03-21 12:08:37 +08:00
commit 271230fca7
71 changed files with 15699 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
scripts/db/sql/clickhouse/ch-query.sh Executable file
View File

@@ -0,0 +1,102 @@
#!/bin/bash
# 文件名: ch-query.sh
# 用途: 执行ClickHouse SQL查询的便捷脚本
# 连接参数
CH_HOST="localhost"
CH_PORT="9000"
CH_USER="admin"
CH_PASSWORD="your_secure_password"
# 基本查询函数
function ch_query() {
clickhouse client --host $CH_HOST --port $CH_PORT --user $CH_USER --password $CH_PASSWORD -q "$1"
}
# 显示帮助信息
function show_help() {
echo "ClickHouse 查询工具"
echo "用法: $0 [选项] [SQL查询]"
echo ""
echo "选项:"
echo " -t 显示所有表"
echo " -d 显示所有数据库"
echo " -s <表名> 显示表结构"
echo " -p <表名> 显示表样本数据(前10行)"
echo " -c <表名> 计算表中的记录数"
echo " -h, --help 显示此帮助信息"
echo " -q \"SQL查询\" 执行自定义SQL查询"
echo " -f <文件名> 执行SQL文件"
echo ""
echo "示例:"
echo " $0 -d # 显示所有数据库"
echo " $0 -t # 显示所有表"
echo " $0 -s limq.link_events # 显示link_events表结构"
echo " $0 -q \"SELECT * FROM limq.link_events LIMIT 5\" # 执行自定义查询"
}
# 没有参数时显示帮助
if [ $# -eq 0 ]; then
show_help
exit 0
fi
# 处理命令行参数
case "$1" in
-t)
ch_query "SHOW TABLES"
;;
-d)
ch_query "SHOW DATABASES"
;;
-s)
if [ -z "$2" ]; then
echo "错误: 需要提供表名"
exit 1
fi
ch_query "DESCRIBE TABLE $2"
;;
-p)
if [ -z "$2" ]; then
echo "错误: 需要提供表名"
exit 1
fi
ch_query "SELECT * FROM $2 LIMIT 10"
;;
-c)
if [ -z "$2" ]; then
echo "错误: 需要提供表名"
exit 1
fi
ch_query "SELECT COUNT(*) FROM $2"
;;
-q)
if [ -z "$2" ]; then
echo "错误: 需要提供SQL查询"
exit 1
fi
ch_query "$2"
;;
-f)
if [ -z "$2" ]; then
echo "错误: 需要提供SQL文件"
exit 1
fi
if [ ! -f "$2" ]; then
echo "错误: 文件 '$2' 不存在"
exit 1
fi
SQL=$(cat "$2")
ch_query "$SQL"
;;
-h|--help)
show_help
;;
*)
echo "未知选项: $1"
show_help
exit 1
;;
esac
exit 0

4
scripts/db/sql/clickhouse/clickhouse.md Normal file
View File

@@ -0,0 +1,4 @@
```bash
alias clickhouse-sql='clickhouse client --host localhost --port 9000 --user admin --password your_secure_password --database promote -q'
clickhouse-sql "SHOW TABLES"
```

170
scripts/db/sql/clickhouse/create_limq.sql Normal file
View File

@@ -0,0 +1,170 @@
-- 创建数据库
CREATE DATABASE IF NOT EXISTS limq;
-- 切换到limq数据库
USE limq;
-- 创建短链接访问事件表
CREATE TABLE IF NOT EXISTS limq.link_events (
event_id UUID DEFAULT generateUUIDv4(),
event_time DateTime64(3) DEFAULT now64(),
date Date DEFAULT toDate(event_time),
link_id String,
channel_id String,
visitor_id String,
session_id String,
event_type Enum8(
'click' = 1,
'redirect' = 2,
'conversion' = 3,
'error' = 4
),
-- 访问者信息
ip_address String,
country String,
city String,
-- 来源信息
referrer String,
utm_source String,
utm_medium String,
utm_campaign String,
-- 设备信息
user_agent String,
device_type Enum8(
'mobile' = 1,
'tablet' = 2,
'desktop' = 3,
'other' = 4
),
browser String,
os String,
-- 交互信息
time_spent_sec UInt32 DEFAULT 0,
is_bounce Boolean DEFAULT true,
-- QR码相关
is_qr_scan Boolean DEFAULT false,
qr_code_id String DEFAULT '',
-- 转化数据
conversion_type String DEFAULT '',
conversion_value Float64 DEFAULT 0,
-- 其他属性
custom_data String DEFAULT '{}'
) ENGINE = MergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, link_id, event_time) SETTINGS index_granularity = 8192;
-- 短链接维度表
CREATE TABLE IF NOT EXISTS limq.links (
link_id String,
original_url String,
created_at DateTime,
created_by String,
title String,
description String,
tags Array(String),
is_active Boolean DEFAULT true,
expires_at Nullable(DateTime),
team_id String DEFAULT '',
project_id String DEFAULT '',
PRIMARY KEY (link_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
link_id SETTINGS index_granularity = 8192;
-- 会话跟踪表
CREATE TABLE IF NOT EXISTS limq.sessions (
session_id String,
visitor_id String,
link_id String,
started_at DateTime64(3),
last_activity DateTime64(3),
ended_at Nullable(DateTime64(3)),
duration_sec UInt32 DEFAULT 0,
session_pages UInt8 DEFAULT 1,
is_completed Boolean DEFAULT false,
PRIMARY KEY (session_id)
) ENGINE = ReplacingMergeTree(last_activity)
ORDER BY
(session_id, link_id, visitor_id) SETTINGS index_granularity = 8192;
-- QR码统计表
CREATE TABLE IF NOT EXISTS limq.qr_scans (
scan_id UUID DEFAULT generateUUIDv4(),
qr_code_id String,
link_id String,
scan_time DateTime64(3),
visitor_id String,
location String,
device_type Enum8(
'mobile' = 1,
'tablet' = 2,
'desktop' = 3,
'other' = 4
),
led_to_conversion Boolean DEFAULT false,
PRIMARY KEY (scan_id)
) ENGINE = MergeTree() PARTITION BY toYYYYMM(scan_time)
ORDER BY
(qr_code_id, scan_time) SETTINGS index_granularity = 8192;
-- 每日链接汇总视图
CREATE MATERIALIZED VIEW limq.link_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
link_id,
count() AS total_clicks,
uniqExact(visitor_id) AS unique_visitors,
uniqExact(session_id) AS unique_sessions,
sum(time_spent_sec) AS total_time_spent,
avg(time_spent_sec) AS avg_time_spent,
countIf(is_bounce) AS bounce_count,
countIf(event_type = 'conversion') AS conversion_count,
uniqExact(referrer) AS unique_referrers,
countIf(device_type = 'mobile') AS mobile_count,
countIf(device_type = 'tablet') AS tablet_count,
countIf(device_type = 'desktop') AS desktop_count,
countIf(is_qr_scan) AS qr_scan_count,
sum(conversion_value) AS total_conversion_value
FROM
limq.link_events
GROUP BY
date,
link_id;
-- 每小时访问模式视图
CREATE MATERIALIZED VIEW limq.link_hourly_patterns ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, hour, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
toHour(event_time) AS hour,
link_id,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
GROUP BY
date,
hour,
link_id;
-- 平台分布视图
CREATE MATERIALIZED VIEW limq.platform_distribution ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, utm_source, device_type) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
utm_source,
device_type,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
WHERE
utm_source != ''
GROUP BY
date,
utm_source,
device_type;

146
scripts/db/sql/clickhouse/create_team_project_qrcode.sql Normal file
View File

@@ -0,0 +1,146 @@
-- 添加team、project和qrcode表到limq数据库
USE limq;
-- 团队表
CREATE TABLE IF NOT EXISTS limq.teams (
team_id String,
name String,
created_at DateTime,
created_by String,
description String DEFAULT '',
avatar_url String DEFAULT '',
is_active Boolean DEFAULT true,
plan_type Enum8(
'free' = 1,
'pro' = 2,
'enterprise' = 3
),
members_count UInt32 DEFAULT 1,
PRIMARY KEY (team_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
team_id SETTINGS index_granularity = 8192;
-- 项目表
CREATE TABLE IF NOT EXISTS limq.projects (
project_id String,
team_id String,
name String,
created_at DateTime,
created_by String,
description String DEFAULT '',
is_archived Boolean DEFAULT false,
links_count UInt32 DEFAULT 0,
total_clicks UInt64 DEFAULT 0,
last_updated DateTime DEFAULT now(),
PRIMARY KEY (project_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(project_id, team_id) SETTINGS index_granularity = 8192;
-- QR码表 (扩展现有的qr_scans表)
CREATE TABLE IF NOT EXISTS limq.qrcodes (
qr_code_id String,
link_id String,
team_id String,
project_id String DEFAULT '',
name String,
description String DEFAULT '',
created_at DateTime,
created_by String,
updated_at DateTime DEFAULT now(),
qr_type Enum8(
'standard' = 1,
'custom' = 2,
'dynamic' = 3
) DEFAULT 'standard',
image_url String DEFAULT '',
design_config String DEFAULT '{}',
is_active Boolean DEFAULT true,
total_scans UInt64 DEFAULT 0,
unique_scanners UInt32 DEFAULT 0,
PRIMARY KEY (qr_code_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(qr_code_id, link_id) SETTINGS index_granularity = 8192;
-- 团队成员表
CREATE TABLE IF NOT EXISTS limq.team_members (
team_id String,
user_id String,
role Enum8(
'owner' = 1,
'admin' = 2,
'editor' = 3,
'viewer' = 4
),
joined_at DateTime DEFAULT now(),
invited_by String,
is_active Boolean DEFAULT true,
last_active DateTime DEFAULT now(),
PRIMARY KEY (team_id, user_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
(team_id, user_id) SETTINGS index_granularity = 8192;
-- 团队每日统计视图
CREATE MATERIALIZED VIEW limq.team_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, team_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.team_id AS team_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.team_id != ''
GROUP BY
date,
l.team_id;
-- 项目每日统计视图
CREATE MATERIALIZED VIEW limq.project_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, project_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
l.project_id AS project_id,
count() AS total_clicks,
uniqExact(e.visitor_id) AS unique_visitors,
countIf(e.event_type = 'conversion') AS conversion_count,
uniqExact(e.link_id) AS links_used,
countIf(e.is_qr_scan) AS qr_scan_count
FROM
limq.link_events e
JOIN limq.links l ON e.link_id = l.link_id
WHERE
l.project_id != ''
GROUP BY
date,
l.project_id;
-- QR码每日统计视图
CREATE MATERIALIZED VIEW limq.qrcode_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, qr_code_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(scan_time) AS date,
qr_code_id,
count() AS total_scans,
uniqExact(visitor_id) AS unique_scanners,
countIf(led_to_conversion) AS conversions,
countIf(device_type = 'mobile') AS mobile_scans,
countIf(device_type = 'tablet') AS tablet_scans,
countIf(device_type = 'desktop') AS desktop_scans,
uniqExact(location) AS unique_locations
FROM
limq.qr_scans
GROUP BY
date,
qr_code_id;

997
scripts/db/sql/clickhouse/mock_link_data.sql Normal file
View File

@@ -0,0 +1,997 @@
-- 移动端点击访问事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 10:25:30',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-123',
's-456',
'click',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
45,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 11:32:21',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-124',
's-457',
'click',
'43.78.123.45',
'Japan',
'Tokyo',
'https://twitter.com',
'twitter',
'social',
'spring_promo',
'Mozilla/5.0 (Android 10)',
'mobile',
'Chrome',
'Android',
15,
true,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 14:15:45',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-125',
's-458',
'click',
'72.34.67.81',
'US',
'New York',
'https://www.facebook.com',
'facebook',
'social',
'crypto_ad',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
120,
false,
false,
'interact',
0
);
-- 桌面设备点击事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 08:45:12',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-126',
's-459',
'click',
'89.67.43.21',
'Germany',
'Berlin',
'https://www.reddit.com',
'reddit',
'referral',
'none',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Chrome',
'Windows',
300,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 16:20:33',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-127',
's-460',
'click',
'178.65.43.12',
'UK',
'London',
'https://www.linkedin.com',
'linkedin',
'social',
'biz_campaign',
'Mozilla/5.0 (Macintosh)',
'desktop',
'Safari',
'MacOS',
250,
false,
false,
'stay',
0
);
-- 平板设备点击事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 13:10:55',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-128',
's-461',
'click',
'156.78.34.12',
'Canada',
'Toronto',
'https://www.youtube.com',
'youtube',
'video',
'tutorial',
'Mozilla/5.0 (iPad)',
'tablet',
'Safari',
'iOS',
180,
false,
false,
'visit',
0
);
-- QR扫描访问事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 09:30:22',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_qr',
'v-129',
's-462',
'click',
'101.56.78.90',
'China',
'Beijing',
'direct',
'qr',
'print',
'offline_event',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
75,
false,
true,
'visit',
0
);
-- 转化事件
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 10:27:45',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-123',
's-456',
'conversion',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
120,
false,
false,
'signup',
50
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-15 08:52:18',
'2025-03-15',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-126',
's-459',
'conversion',
'89.67.43.21',
'Germany',
'Berlin',
'https://www.reddit.com',
'reddit',
'referral',
'none',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Chrome',
'Windows',
450,
false,
false,
'purchase',
150.75
);
-- 第二天的数据 (3/16)
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-16 11:15:30',
'2025-03-16',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-130',
's-463',
'click',
'178.91.45.67',
'France',
'Paris',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (Android 11)',
'mobile',
'Chrome',
'Android',
60,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-16 14:22:45',
'2025-03-16',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-131',
's-464',
'click',
'89.123.45.78',
'Spain',
'Madrid',
'https://www.instagram.com',
'instagram',
'social',
'influencer',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
90,
false,
false,
'interact',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-16 16:40:12',
'2025-03-16',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-131',
's-464',
'conversion',
'89.123.45.78',
'Spain',
'Madrid',
'https://www.instagram.com',
'instagram',
'social',
'influencer',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
200,
false,
false,
'subscription',
75.50
);
-- 第三天数据 (3/17)
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-17 09:10:22',
'2025-03-17',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-132',
's-465',
'click',
'45.67.89.123',
'US',
'Los Angeles',
'https://www.google.com',
'google',
'cpc',
'spring_sale',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Edge',
'Windows',
150,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-17 12:30:45',
'2025-03-17',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-133',
's-466',
'click',
'67.89.123.45',
'Brazil',
'Sao Paulo',
'https://www.yahoo.com',
'yahoo',
'organic',
'none',
'Mozilla/5.0 (iPad)',
'tablet',
'Safari',
'iOS',
120,
false,
false,
'stay',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-17 15:45:33',
'2025-03-17',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-132',
's-465',
'conversion',
'45.67.89.123',
'US',
'Los Angeles',
'https://www.google.com',
'google',
'cpc',
'spring_sale',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Edge',
'Windows',
300,
false,
false,
'purchase',
225.50
);
-- 添加一周前的数据 (对比期)
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-08 10:25:30',
'2025-03-08',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-140',
's-470',
'click',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
30,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-08 11:32:21',
'2025-03-08',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-141',
's-471',
'click',
'89.67.43.21',
'Germany',
'Berlin',
'https://www.reddit.com',
'reddit',
'referral',
'none',
'Mozilla/5.0 (Windows NT 10.0)',
'desktop',
'Chrome',
'Windows',
200,
false,
false,
'visit',
0
);
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
conversion_type,
conversion_value
)
VALUES
(
generateUUIDv4(),
'2025-03-08 13:10:55',
'2025-03-08',
'a71fcfe8-d293-4d6d-91c5-1528fa7f6294',
'ch_main',
'v-140',
's-470',
'conversion',
'103.45.67.89',
'China',
'Shanghai',
'https://www.google.com',
'google',
'organic',
'none',
'Mozilla/5.0 (iPhone)',
'mobile',
'Safari',
'iOS',
100,
false,
false,
'purchase',
100.00
);

193
scripts/db/sql/clickhouse/recreate_limq.sql Normal file
View File

@@ -0,0 +1,193 @@
-- 删除现有的物化视图（需要先删除视图，因为它们依赖于表）
DROP TABLE IF EXISTS limq.platform_distribution;
DROP TABLE IF EXISTS limq.link_hourly_patterns;
DROP TABLE IF EXISTS limq.link_daily_stats;
-- 删除现有的表
DROP TABLE IF EXISTS limq.qr_scans;
DROP TABLE IF EXISTS limq.sessions;
DROP TABLE IF EXISTS limq.link_events;
DROP TABLE IF EXISTS limq.links;
-- 创建数据库（如果不存在）
CREATE DATABASE IF NOT EXISTS limq;
-- 切换到limq数据库
USE limq;
-- 创建短链接访问事件表
CREATE TABLE IF NOT EXISTS limq.link_events (
event_id UUID DEFAULT generateUUIDv4(),
event_time DateTime64(3) DEFAULT now64(),
date Date DEFAULT toDate(event_time),
link_id String,
channel_id String,
visitor_id String,
session_id String,
event_type Enum8(
'click' = 1,
'redirect' = 2,
'conversion' = 3,
'error' = 4
),
-- 访问者信息
ip_address String,
country String,
city String,
-- 来源信息
referrer String,
utm_source String,
utm_medium String,
utm_campaign String,
-- 设备信息
user_agent String,
device_type Enum8(
'mobile' = 1,
'tablet' = 2,
'desktop' = 3,
'other' = 4
),
browser String,
os String,
-- 交互信息
time_spent_sec UInt32 DEFAULT 0,
is_bounce Boolean DEFAULT true,
-- QR码相关
is_qr_scan Boolean DEFAULT false,
qr_code_id String DEFAULT '',
-- 转化数据
conversion_type Enum8(
'visit' = 1,
'stay' = 2,
'interact' = 3,
'signup' = 4,
'subscription' = 5,
'purchase' = 6
) DEFAULT 'visit',
conversion_value Float64 DEFAULT 0,
-- 其他属性
custom_data String DEFAULT '{}'
) ENGINE = MergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, link_id, event_time) SETTINGS index_granularity = 8192;
-- 短链接维度表
CREATE TABLE IF NOT EXISTS limq.links (
link_id String,
original_url String,
created_at DateTime64(3),
created_by String,
title String,
description String,
tags Array(String),
is_active Boolean DEFAULT true,
expires_at Nullable(DateTime64(3)),
team_id String DEFAULT '',
project_id String DEFAULT '',
PRIMARY KEY (link_id)
) ENGINE = ReplacingMergeTree()
ORDER BY
link_id SETTINGS index_granularity = 8192;
-- 会话跟踪表
CREATE TABLE IF NOT EXISTS limq.sessions (
session_id String,
visitor_id String,
link_id String,
started_at DateTime64(3),
last_activity DateTime64(3),
ended_at Nullable(DateTime64(3)),
duration_sec UInt32 DEFAULT 0,
session_pages UInt8 DEFAULT 1,
is_completed Boolean DEFAULT false,
PRIMARY KEY (session_id)
) ENGINE = ReplacingMergeTree(last_activity)
ORDER BY
(session_id, link_id, visitor_id) SETTINGS index_granularity = 8192;
-- QR码统计表
CREATE TABLE IF NOT EXISTS limq.qr_scans (
scan_id UUID DEFAULT generateUUIDv4(),
qr_code_id String,
link_id String,
scan_time DateTime64(3),
visitor_id String,
location String,
device_type Enum8(
'mobile' = 1,
'tablet' = 2,
'desktop' = 3,
'other' = 4
),
led_to_conversion Boolean DEFAULT false,
PRIMARY KEY (scan_id)
) ENGINE = MergeTree() PARTITION BY toYYYYMM(scan_time)
ORDER BY
scan_id SETTINGS index_granularity = 8192;
-- 每日链接汇总视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_daily_stats ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
link_id,
count() AS total_clicks,
uniqExact(visitor_id) AS unique_visitors,
uniqExact(session_id) AS unique_sessions,
sum(time_spent_sec) AS total_time_spent,
avg(time_spent_sec) AS avg_time_spent,
countIf(is_bounce) AS bounce_count,
countIf(event_type = 'conversion') AS conversion_count,
uniqExact(referrer) AS unique_referrers,
countIf(device_type = 'mobile') AS mobile_count,
countIf(device_type = 'tablet') AS tablet_count,
countIf(device_type = 'desktop') AS desktop_count,
countIf(is_qr_scan) AS qr_scan_count,
sum(conversion_value) AS total_conversion_value
FROM
limq.link_events
GROUP BY
date,
link_id;
-- 每小时访问模式视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.link_hourly_patterns ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, hour, link_id) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
toHour(event_time) AS hour,
link_id,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
GROUP BY
date,
hour,
link_id;
-- 平台分布视图
CREATE MATERIALIZED VIEW IF NOT EXISTS limq.platform_distribution ENGINE = SummingMergeTree() PARTITION BY toYYYYMM(date)
ORDER BY
(date, utm_source, device_type) SETTINGS index_granularity = 8192 AS
SELECT
toDate(event_time) AS date,
utm_source,
device_type,
count() AS visits,
uniqExact(visitor_id) AS unique_visitors
FROM
limq.link_events
WHERE
utm_source != ''
GROUP BY
date,
utm_source,
device_type;

828
scripts/db/sql/clickhouse/seed-clickhouse-analytics.sql Normal file
View File

@@ -0,0 +1,828 @@
-- 清空现有数据（可选）
TRUNCATE TABLE IF EXISTS limq.link_events;
TRUNCATE TABLE IF EXISTS limq.link_daily_stats;
TRUNCATE TABLE IF EXISTS limq.link_hourly_patterns;
TRUNCATE TABLE IF EXISTS limq.links;
-- 使用固定的UUID值插入链接
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'11111111-1111-1111-1111-111111111111',
'https://example.com/page1',
now(),
'user-1',
'产品页面',
'我们的主要产品页面',
[ '产品',
'营销' ],
true
);
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'22222222-2222-2222-2222-222222222222',
'https://example.com/promo',
now(),
'user-1',
'促销活动',
'夏季特别促销活动',
[ '促销',
'活动' ],
true
);
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'33333333-3333-3333-3333-333333333333',
'https://example.com/blog',
now(),
'user-2',
'公司博客',
'公司新闻和更新',
[ '博客',
'内容' ],
true
);
INSERT INTO
limq.links (
link_id,
original_url,
created_at,
created_by,
title,
description,
tags,
is_active
)
VALUES
(
'44444444-4444-4444-4444-444444444444',
'https://example.com/signup',
now(),
'user-2',
'注册页面',
'新用户注册页面',
[ '转化',
'注册' ],
true
);
-- 为第一个链接创建500条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'11111111-1111-1111-1111-111111111111' AS link_id,
'channel-1' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 50 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 300 AS time_spent_sec,
rand() % 100 < 25 AS is_bounce,
rand() % 100 < 20 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 1.5 AS conversion_value,
'{}' AS custom_data
FROM
numbers(500);
-- 为第二个链接创建300条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'22222222-2222-2222-2222-222222222222' AS link_id,
'channel-1' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 40 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 300 AS time_spent_sec,
rand() % 100 < 25 AS is_bounce,
rand() % 100 < 15 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 2.5 AS conversion_value,
'{}' AS custom_data
FROM
numbers(300);
-- 为第三个链接创建200条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'33333333-3333-3333-3333-333333333333' AS link_id,
'channel-2' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 30 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 600 AS time_spent_sec,
rand() % 100 < 15 AS is_bounce,
rand() % 100 < 10 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 1.2 AS conversion_value,
'{}' AS custom_data
FROM
numbers(200);
-- 为第四个链接创建400条记录
INSERT INTO
limq.link_events (
event_id,
event_time,
date,
link_id,
channel_id,
visitor_id,
session_id,
event_type,
ip_address,
country,
city,
referrer,
utm_source,
utm_medium,
utm_campaign,
user_agent,
device_type,
browser,
os,
time_spent_sec,
is_bounce,
is_qr_scan,
qr_code_id,
conversion_type,
conversion_value,
custom_data
)
SELECT
generateUUIDv4() AS event_id,
subtractDays(now(), rand() % 30) AS event_time,
toDate(event_time) AS date,
'44444444-4444-4444-4444-444444444444' AS link_id,
'channel-2' AS channel_id,
concat('visitor-', toString(rand() % 100 + 1)) AS visitor_id,
concat('session-', toString(number % 60 + 1)) AS session_id,
multiIf(
rand() % 100 < 70,
'click',
rand() % 100 < 90,
'redirect',
rand() % 100 < 98,
'conversion',
'error'
) AS event_type,
concat('192.168.1.', toString(rand() % 255)) AS ip_address,
multiIf(
rand() % 100 < 60,
'China',
rand() % 100 < 85,
'US',
rand() % 100 < 95,
'Japan',
'Other'
) AS country,
multiIf(
rand() % 100 < 60,
'Beijing',
rand() % 100 < 85,
'New York',
rand() % 100 < 95,
'Tokyo',
'Other'
) AS city,
multiIf(
rand() % 100 < 30,
'https://google.com',
rand() % 100 < 50,
'https://facebook.com',
rand() % 100 < 65,
'https://twitter.com',
rand() % 100 < 75,
'https://instagram.com',
rand() % 100 < 85,
'https://linkedin.com',
rand() % 100 < 90,
'https://bing.com',
rand() % 100 < 95,
'https://baidu.com',
'direct'
) AS referrer,
multiIf(
rand() % 100 < 40,
'google',
rand() % 100 < 70,
'facebook',
rand() % 100 < 90,
'email',
'direct'
) AS utm_source,
multiIf(
rand() % 100 < 40,
'cpc',
rand() % 100 < 70,
'social',
rand() % 100 < 90,
'email',
'direct'
) AS utm_medium,
multiIf(
rand() % 100 < 40,
'summer_sale',
rand() % 100 < 70,
'product_launch',
rand() % 100 < 90,
'newsletter',
'brand'
) AS utm_campaign,
'Mozilla/5.0' AS user_agent,
multiIf(
rand() % 100 < 60,
'mobile',
rand() % 100 < 85,
'desktop',
rand() % 100 < 95,
'tablet',
'other'
) AS device_type,
multiIf(
rand() % 100 < 50,
'Chrome',
rand() % 100 < 80,
'Safari',
rand() % 100 < 95,
'Firefox',
'Edge'
) AS browser,
multiIf(
rand() % 100 < 50,
'iOS',
rand() % 100 < 90,
'Android',
'Windows'
) AS os,
rand() % 400 AS time_spent_sec,
rand() % 100 < 20 AS is_bounce,
rand() % 100 < 25 AS is_qr_scan,
concat('qr-', toString(rand() % 10 + 1)) AS qr_code_id,
multiIf(
rand() % 100 < 50,
'visit',
rand() % 100 < 70,
'stay',
rand() % 100 < 85,
'interact',
rand() % 100 < 93,
'signup',
rand() % 100 < 97,
'subscription',
'purchase'
) AS conversion_type,
rand() % 100 * 3.5 AS conversion_value,
'{}' AS custom_data
FROM
numbers(400);
-- 插入link_daily_stats表数据
INSERT INTO
limq.link_daily_stats (
date,
link_id,
total_clicks,
unique_visitors,
unique_sessions,
total_time_spent,
avg_time_spent,
bounce_count,
conversion_count,
unique_referrers,
mobile_count,
tablet_count,
desktop_count,
qr_scan_count,
total_conversion_value
)
SELECT
subtractDays(today(), number) AS date,
multiIf(
number % 4 = 0,
'11111111-1111-1111-1111-111111111111',
number % 4 = 1,
'22222222-2222-2222-2222-222222222222',
number % 4 = 2,
'33333333-3333-3333-3333-333333333333',
'44444444-4444-4444-4444-444444444444'
) AS link_id,
50 + rand() % 100 AS total_clicks,
30 + rand() % 50 AS unique_visitors,
20 + rand() % 40 AS unique_sessions,
(500 + rand() % 1000) * 60 AS total_time_spent,
(rand() % 10) * 60 + rand() % 60 AS avg_time_spent,
5 + rand() % 20 AS bounce_count,
rand() % 30 AS conversion_count,
3 + rand() % 8 AS unique_referrers,
20 + rand() % 40 AS mobile_count,
5 + rand() % 15 AS tablet_count,
15 + rand() % 30 AS desktop_count,
rand() % 10 AS qr_scan_count,
rand() % 1000 * 2.5 AS total_conversion_value
FROM
numbers(30)
WHERE
number < 30;
-- 插入link_hourly_patterns表数据
INSERT INTO
limq.link_hourly_patterns (date, hour, link_id, visits, unique_visitors)
SELECT
subtractDays(today(), number % 7) AS date,
number % 24 AS hour,
multiIf(
intDiv(number, 24) % 4 = 0,
'11111111-1111-1111-1111-111111111111',
intDiv(number, 24) % 4 = 1,
'22222222-2222-2222-2222-222222222222',
intDiv(number, 24) % 4 = 2,
'33333333-3333-3333-3333-333333333333',
'44444444-4444-4444-4444-444444444444'
) AS link_id,
5 + rand() % 20 AS visits,
3 + rand() % 10 AS unique_visitors
FROM
numbers(672) -- 7天 x 24小时 x 4个链接
WHERE
number < 672;
-- 显示数据行数，验证插入成功
SELECT
'link_events 表行数:' AS metric,
count() AS value
FROM
limq.link_events
UNION
ALL
SELECT
'link_daily_stats 表行数:',
count()
FROM
limq.link_daily_stats
UNION
ALL
SELECT
'link_hourly_patterns 表行数:',
count()
FROM
limq.link_hourly_patterns;

331
scripts/db/sql/postgres/pg-query.js Executable file
View File

@@ -0,0 +1,331 @@
#!/usr/bin/env node
// # 显示所有表
// ./pg-query.js -t
// # 显示表结构
// ./pg-query.js -d influencers
// # 显示样本数据，限制5行
// ./pg-query.js -s posts -l 5
// # 查看表记录数
// ./pg-query.js -c posts
// # 显示索引
// ./pg-query.js -i posts
// # 显示外键
// ./pg-query.js -f posts
// # 显示引用
// ./pg-query.js -r influencers
// # 执行自定义查询
// ./pg-query.js -q "SELECT * FROM influencers WHERE platform = 'Instagram' LIMIT 5"
// # 执行SQL文件
// ./pg-query.js -e schema.sql
const { Client } = require('pg');
const path = require('path');
const fs = require('fs');
const yargs = require('yargs/yargs');
const { hideBin } = require('yargs/helpers');
// 加载.env文件 - 使用正确的相对路径
require('dotenv').config({ path: path.resolve(__dirname, '../../../.env') });
// 显示连接信息(不含密码)以便调试
function getConnectionString() {
// 使用.env中的DATABASE_URL
const databaseUrl = process.env.DATABASE_URL;
if (!databaseUrl) {
console.error('错误: 未找到DATABASE_URL环境变量');
process.exit(1);
}
// 显示连接信息但隐藏密码
const sanitizedUrl = databaseUrl.replace(/:[^:@]+@/, ':***@');
console.log(`使用连接: ${sanitizedUrl}`);
return databaseUrl;
}
// 创建一个新的客户端
async function runQuery(query, params = []) {
const client = new Client({
connectionString: getConnectionString()
});
try {
await client.connect();
console.log('数据库连接成功');
const result = await client.query(query, params);
return result.rows;
} catch (err) {
console.error('查询执行错误:', err.message);
return null;
} finally {
await client.end();
}
}
// 显示所有表
async function showTables() {
const query = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name;";
const tables = await runQuery(query);
if (tables && tables.length > 0) {
console.log('数据库中的表:');
console.table(tables);
} else {
console.log('没有找到表或连接失败');
}
}
// 显示表结构
async function showTableStructure(tableName) {
const query = `
SELECT
column_name AS "列名",
data_type AS "数据类型",
CASE WHEN is_nullable = 'YES' THEN '允许为空' ELSE '不允许为空' END AS "是否可空",
column_default AS "默认值",
character_maximum_length AS "最大长度"
FROM
information_schema.columns
WHERE
table_schema = 'public' AND
table_name = $1
ORDER BY
ordinal_position;
`;
const columns = await runQuery(query, [tableName]);
if (columns && columns.length > 0) {
console.log(`表 ${tableName} 的结构:`);
console.table(columns);
} else {
console.log(`表 ${tableName} 不存在或连接失败`);
}
}
// 显示样本数据
async function showSampleData(tableName, limit = 10) {
const query = `SELECT * FROM "${tableName}" LIMIT ${limit};`;
const data = await runQuery(query);
if (data && data.length > 0) {
console.log(`表 ${tableName} 的样本数据 (${limit} 行):`);
console.table(data);
} else {
console.log(`表 ${tableName} 为空或不存在`);
}
}
// 显示记录计数
async function showRecordCount(tableName) {
const query = `SELECT COUNT(*) AS "记录数" FROM "${tableName}";`;
const count = await runQuery(query);
if (count) {
console.log(`表 ${tableName} 的记录数:`);
console.table(count);
} else {
console.log(`表 ${tableName} 不存在或连接失败`);
}
}
// 显示索引信息
async function showIndexes(tableName) {
const query = `
SELECT
indexname AS "索引名称",
indexdef AS "索引定义"
FROM
pg_indexes
WHERE
tablename = $1
ORDER BY
indexname;
`;
const indexes = await runQuery(query, [tableName]);
if (indexes && indexes.length > 0) {
console.log(`表 ${tableName} 的索引:`);
console.table(indexes);
} else {
console.log(`表 ${tableName} 没有索引或不存在`);
}
}
// 显示外键
async function showForeignKeys(tableName) {
const query = `
SELECT
conname AS "外键名称",
pg_get_constraintdef(oid) AS "外键定义"
FROM
pg_constraint
WHERE
conrelid = $1::regclass AND contype = 'f';
`;
const foreignKeys = await runQuery(query, [tableName]);
if (foreignKeys && foreignKeys.length > 0) {
console.log(`表 ${tableName} 的外键:`);
console.table(foreignKeys);
} else {
console.log(`表 ${tableName} 没有外键或不存在`);
}
}
// 显示引用当前表的外键
async function showReferencingKeys(tableName) {
const query = `
SELECT
c.conname AS "外键名称",
t.relname AS "引用表",
pg_get_constraintdef(c.oid) AS "外键定义"
FROM
pg_constraint c
JOIN
pg_class t ON c.conrelid = t.oid
WHERE
c.confrelid = $1::regclass AND c.contype = 'f';
`;
const referencingKeys = await runQuery(query, [tableName]);
if (referencingKeys && referencingKeys.length > 0) {
console.log(`引用表 ${tableName} 的外键关系:`);
console.table(referencingKeys);
} else {
console.log(`没有找到引用表 ${tableName} 的外键关系`);
}
}
// 执行自定义查询
async function executeQuery(query) {
const result = await runQuery(query);
if (result) {
console.log('查询结果:');
console.table(result);
} else {
console.log('查询执行失败或无结果');
}
}
// 执行SQL文件
async function executeSqlFile(filename) {
try {
const sql = fs.readFileSync(filename, 'utf8');
console.log(`执行SQL文件: ${filename}`);
await executeQuery(sql);
} catch (err) {
console.error(`执行SQL文件失败: ${err.message}`);
}
}
// 主函数
async function main() {
try {
const argv = yargs(hideBin(process.argv))
.usage('PostgreSQL 查询工具\n\n用法: $0 [选项]')
.option('t', {
alias: 'tables',
describe: '显示所有表',
type: 'boolean'
})
.option('d', {
alias: 'describe',
describe: '显示表结构',
type: 'string'
})
.option('s', {
alias: 'sample',
describe: '显示表样本数据',
type: 'string'
})
.option('l', {
alias: 'limit',
describe: '样本数据行数限制',
type: 'number',
default: 10
})
.option('c', {
alias: 'count',
describe: '计算表中的记录数',
type: 'string'
})
.option('i', {
alias: 'indexes',
describe: '显示表索引',
type: 'string'
})
.option('f', {
alias: 'foreign-keys',
describe: '显示表外键关系',
type: 'string'
})
.option('r', {
alias: 'references',
describe: '显示引用此表的外键',
type: 'string'
})
.option('q', {
alias: 'query',
describe: '执行自定义SQL查询',
type: 'string'
})
.option('e', {
alias: 'execute-file',
describe: '执行SQL文件',
type: 'string'
})
.example('$0 -t', '显示所有表')
.example('$0 -d influencers', '显示influencers表结构')
.example('$0 -s posts -l 5', '显示posts表前5行数据')
.epilog('更多信息请访问项目文档')
.help()
.alias('h', 'help')
.argv;
if (argv.tables) {
await showTables();
} else if (argv.describe) {
await showTableStructure(argv.describe);
} else if (argv.sample) {
await showSampleData(argv.sample, argv.limit);
} else if (argv.count) {
await showRecordCount(argv.count);
} else if (argv.indexes) {
await showIndexes(argv.indexes);
} else if (argv.foreignKeys) {
await showForeignKeys(argv.foreignKeys);
} else if (argv.references) {
await showReferencingKeys(argv.references);
} else if (argv.query) {
await executeQuery(argv.query);
} else if (argv.executeFile) {
await executeSqlFile(argv.executeFile);
} else {
yargs(hideBin(process.argv)).showHelp();
}
} catch (err) {
console.error('程序执行错误:', err.message);
process.exit(1);
}
}
// 执行主函数
main().catch(err => {
console.error('程序执行错误:', err);
process.exit(1);
});