From a25478e73864518647fc8d4ff55fe1580e771d43 Mon Sep 17 00:00:00 2001 From: William Tso Date: Wed, 12 Mar 2025 23:31:42 +0800 Subject: [PATCH] insert data to clickhouse test succss --- backend/.env | 2 +- backend/README.md | 282 ----------- backend/package.json | 3 +- backend/pnpm-lock.yaml | 514 ++++++++++++++++++- backend/src/config/index.ts | 1 + backend/src/services/syncService.ts | 740 ++-------------------------- backend/src/utils/clickhouse.ts | 12 +- 7 files changed, 568 insertions(+), 986 deletions(-) delete mode 100644 backend/README.md diff --git a/backend/.env b/backend/.env index f3bf938..c2439a4 100644 --- a/backend/.env +++ b/backend/.env @@ -17,7 +17,7 @@ CLICKHOUSE_PORT=8123 CLICKHOUSE_USER=admin CLICKHOUSE_PASSWORD=your_secure_password CLICKHOUSE_DATABASE=promote - +CLICKHOUSE_URL=http://localhost:8123 # BullMQ Configuration BULL_REDIS_HOST="localhost" BULL_REDIS_PORT="6379" diff --git a/backend/README.md b/backend/README.md deleted file mode 100644 index a3772bf..0000000 --- a/backend/README.md +++ /dev/null @@ -1,282 +0,0 @@ -# Promote Backend API - -Backend API for the Promote platform, built with Hono.js, Supabase, ClickHouse, Redis, and BullMQ. This platform facilitates influencer marketing campaigns management and analytics tracking. - -## 功能概述 - -- **项目管理**: 创建和管理营销项目 -- **KOL管理**: 跟踪和管理网红账号 -- **帖子跟踪**: 监控营销内容表现 -- **分析跟踪**: 实时追踪视图、点赞和关注者数据 -- **用户认证**: 基于JWT的安全认证 -- **数据缓存**: 使用Redis优化API响应时间 -- **后台任务**: 使用BullMQ处理异步任务 - -## 技术栈 - -- **框架**: [Hono.js](https://honojs.dev/) - 轻量、高性能的Web框架 -- **认证**: [Supabase Auth](https://supabase.com/docs/guides/auth) + JWT - 安全的用户认证 -- **数据库**: - - [Supabase (PostgreSQL)](https://supabase.com/) - 存储关系型数据 - - [ClickHouse](https://clickhouse.com/) - 分析事件数据的列式数据库 -- **缓存**: [Redis](https://redis.io/) - 高性能内存数据存储 -- **任务队列**: [BullMQ](https://docs.bullmq.io/) - 基于Redis的分布式任务队列 - -## 数据库结构 - -### PostgreSQL数据库 - 关系型业务数据 - -#### 主要表 - -**projects** - 营销项目表 -- `id` (uuid, PK): 项目唯一标识 -- `name` (text): 项目名称 -- `description` (text): 项目描述 -- `created_by` (uuid): 创建者ID -- `created_at`, `updated_at`: 时间戳 - -**influencers** - 网红表 -- `influencer_id` (uuid, PK): 网红唯一标识 -- `name` (text): 网红名称 -- `platform` (text): 所属平台(如youtube, instagram等) -- `profile_url` (text): 网红主页链接 -- `external_id` (text): 外部平台ID -- `followers_count` (integer): 粉丝数 -- `video_count` (integer): 视频数量 -- `platform_count` (integer): 平台数量 -- `created_at`, `updated_at`: 时间戳 - -**project_influencers** - 项目与网红关联表 -- `id` (uuid, PK): 关联记录ID -- `project_id` (uuid, FK): 关联的项目ID -- `influencer_id` (uuid, FK): 关联的网红ID -- `created_at`, `updated_at`: 时间戳 - -**posts** - 帖子表 -- `post_id` (uuid, PK): 帖子唯一标识 -- `influencer_id` (uuid, FK): 发布者ID -- `platform` (text): 发布平台 -- `post_url` (text): 帖子链接 -- `title` (text): 帖子标题 -- `description` (text): 帖子描述 -- `published_at`: 发布时间 -- `created_at`, `updated_at`: 时间戳 - -**其他表** -- `comments`: 评论数据 -- `project_comments`: 项目评论 -- `user_profiles`: 用户资料 - -### ClickHouse数据库 - 事件分析数据 - -#### 事件表 - -**events** - 通用事件表 -- `event_id` (UUID): 事件唯一标识 -- `user_id` (String): 用户ID -- `event_type` (String): 事件类型 -- `value` (Float64): 事件值 -- `timestamp` (DateTime): 事件时间 - -**follower_events** - 关注/取关事件表 -- `follower_id` (String): 关注者ID -- `followed_id` (String): 被关注者ID -- `timestamp` (DateTime): 事件时间 -- `action` (Enum): 'follow'或'unfollow' - -**like_events** - 点赞/取消点赞事件表 -- `user_id` (String): 用户ID -- `content_id` (String): 内容ID -- `timestamp` (DateTime): 事件时间 -- `action` (Enum): 'like'或'unlike' - -**view_events** - 内容查看事件表 -- `user_id` (String): 用户ID -- `content_id` (String): 内容ID -- `timestamp` (DateTime): 查看时间 -- `ip` (String): IP地址 -- `user_agent` (String): 用户代理 - -## 环境要求 - -- Node.js 18+ -- pnpm -- Redis -- ClickHouse -- Supabase账户 - -## 安装步骤 - -1. 克隆仓库 - -```bash -git clone -cd promote -``` - -2. 安装依赖 - -```bash -cd backend -pnpm install -``` - -3. 环境配置 - -创建`.env`文件，参考`.env.example` - -```env -# Supabase配置 -DATABASE_URL=postgres://postgres:password@localhost:5432/promote -SUPABASE_URL=your_supabase_url -SUPABASE_KEY=your_supabase_key - -# ClickHouse配置 -CLICKHOUSE_HOST=localhost -CLICKHOUSE_PORT=8123 -CLICKHOUSE_USER=default -CLICKHOUSE_PASSWORD= -CLICKHOUSE_DATABASE=promote - -# Redis配置 -REDIS_URL=redis://localhost:6379 -``` - -4. 启动开发服务器 - -```bash -pnpm dev -``` - -## 数据库检查工具 - -项目包含数据库结构检查工具，位于`backend/scripts/db-inspector`目录： - -```bash -# 一键运行所有数据库检查 -./backend/scripts/db-inspector/run-all.sh - -# 单独运行PostgreSQL检查 -node backend/scripts/db-inspector/postgres-schema.js - -# 单独运行ClickHouse检查 -node backend/scripts/db-inspector/clickhouse-schema.js -``` - -检查结果保存在`backend/db-reports`目录。 - -## API端点 - -### 认证 - -- `POST /api/auth/register` - 注册新用户 -- `POST /api/auth/login` - 用户登录 -- `GET /api/auth/verify` - 验证Token - -### 项目 - -- `GET /api/projects` - 获取所有项目 -- `GET /api/projects/:id` - 获取单个项目 -- `POST /api/projects` - 创建新项目 -- `PUT /api/projects/:id` - 更新项目 -- `DELETE /api/projects/:id` - 删除项目 -- `GET /api/projects/:id/influencers` - 获取项目关联的网红 - -### 网红 - -- `GET /api/influencers` - 获取所有网红 -- `GET /api/influencers/:id` - 获取单个网红信息 -- `POST /api/influencers` - 添加新网红 -- `PUT /api/influencers/:id` - 更新网红信息 -- `DELETE /api/influencers/:id` - 删除网红 -- `GET /api/influencers/:id/posts` - 获取网红的帖子 - -### 帖子 - -- `GET /api/posts` - 获取所有帖子 -- `GET /api/posts/:id` - 获取单个帖子 -- `POST /api/posts` - 创建新帖子 -- `PUT /api/posts/:id` - 更新帖子 -- `DELETE /api/posts/:id` - 删除帖子 - -### 分析 - -- `POST /api/analytics/view` - 记录内容查看事件 -- `POST /api/analytics/like` - 记录点赞/取消点赞事件 -- `POST /api/analytics/follow` - 记录关注/取消关注事件 -- `GET /api/analytics/content/:id` - 获取内容分析 -- `GET /api/analytics/user/:id` - 获取用户分析 -- `GET /api/analytics/project/:id` - 获取项目分析 - -## 开发 - -### 构建项目 - -```bash -pnpm build -``` - -### 启动生产服务器 - -```bash -pnpm start -``` - -### Linting - -```bash -pnpm lint -``` - -### 测试 - -```bash -pnpm test -``` - -## 项目结构 - -``` -backend/ -├── db-reports/ # 数据库结构检查报告 -├── scripts/ # 脚本工具 -│ └── db-inspector/ # 数据库检查工具 -├── src/ -│ ├── config/ # 配置文件 -│ ├── controllers/ # 路由控制器 -│ ├── middlewares/ # 中间件函数 -│ ├── models/ # 数据模型 -│ ├── routes/ # API路由 -│ ├── services/ # 业务逻辑 -│ │ ├── analytics/ # 分析服务 -│ │ ├── auth/ # 认证服务 -│ │ ├── influencer/ # 网红管理服务 -│ │ ├── post/ # 帖子服务 -│ │ └── project/ # 项目服务 -│ ├── utils/ # 工具函数 -│ └── index.ts # 入口点 -├── .env # 环境变量 -├── package.json # 依赖和脚本 -└── tsconfig.json # TypeScript配置 -``` - -## 数据流程 - -1. **用户认证流程** - - 用户通过API注册/登录 - - 验证凭据并生成JWT令牌 - - 令牌用于后续请求验证 - -2. **内容创建流程** - - 创建项目 - - 添加网红到项目 - - 跟踪网红发布的帖子 - -3. **分析跟踪流程** - - 通过API端点记录事件 - - 事件写入ClickHouse - - 通过查询分析数据 - -## 许可 - -本项目基于ISC许可证开源。 \ No newline at end of file diff --git a/backend/package.json b/backend/package.json index d3be60d..e640378 100644 --- a/backend/package.json +++ b/backend/package.json @@ -20,7 +20,7 @@ "license": "ISC", "packageManager": "pnpm@9.15.4+sha512.b2dc20e2fc72b3e18848459b37359a32064663e5627a51e4c74b2c29dd8e8e0491483c3abb40789cfd578bf362fb6ba8261b05f0387d76792ed6e23ea3b1b6a0", "dependencies": { - "@clickhouse/client": "^0.2.10", + "@clickhouse/client": "^1.10.1", "@hono/node-server": "^1.13.8", "@hono/swagger-ui": "^0.5.1", "@supabase/supabase-js": "^2.49.1", @@ -34,7 +34,6 @@ "yargs": "^17.7.2" }, "devDependencies": { - "@clickhouse/client": "^1.10.1", "@supabase/supabase-js": "^2.49.1", "@types/axios": "^0.14.4", "@types/dotenv": "^8.2.3", diff --git a/backend/pnpm-lock.yaml b/backend/pnpm-lock.yaml index 66b5507..98dc0ec 100644 --- a/backend/pnpm-lock.yaml +++ b/backend/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@clickhouse/client': - specifier: ^0.2.10 - version: 0.2.10 + specifier: ^1.10.1 + version: 1.10.1 '@hono/node-server': specifier: ^1.13.8 version: 1.13.8(hono@4.7.4) @@ -32,22 +32,46 @@ importers: jsonwebtoken: specifier: ^9.0.2 version: 9.0.2 + pg: + specifier: ^8.14.0 + version: 8.14.0 redis: specifier: ^4.7.0 version: 4.7.0 + uuid: + specifier: ^11.1.0 + version: 11.1.0 + yargs: + specifier: ^17.7.2 + version: 17.7.2 devDependencies: + '@types/axios': + specifier: ^0.14.4 + version: 0.14.4 + '@types/dotenv': + specifier: ^8.2.3 + version: 8.2.3 '@types/jsonwebtoken': specifier: ^9.0.6 version: 9.0.9 '@types/node': specifier: ^20.11.30 version: 20.17.23 + '@types/pg': + specifier: ^8.11.11 + version: 8.11.11 + '@types/uuid': + specifier: ^10.0.0 + version: 10.0.0 '@typescript-eslint/eslint-plugin': specifier: ^7.4.0 version: 7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.8.2))(eslint@8.57.1)(typescript@5.8.2) '@typescript-eslint/parser': specifier: ^7.4.0 version: 7.18.0(eslint@8.57.1)(typescript@5.8.2) + axios: + specifier: ^1.8.2 + version: 1.8.3 eslint: specifier: ^8.57.0 version: 8.57.1 @@ -63,11 +87,11 @@ importers: packages: - '@clickhouse/client-common@0.2.10': - resolution: {integrity: sha512-BvTY0IXS96y9RUeNCpKL4HUzHmY80L0lDcGN0lmUD6zjOqYMn78+xyHYJ/AIAX7JQsc+/KwFt2soZutQTKxoGQ==} + '@clickhouse/client-common@1.10.1': + resolution: {integrity: sha512-Duh3cign2ChvXABpjVj9Hkz5y20Zf48OE0Y50S4qBVPdhI81S4Rh4MI/bEwvwMnzHubSkiEQ+VhC5HzV8ybnpg==} - '@clickhouse/client@0.2.10': - resolution: {integrity: sha512-ZwBgzjEAFN/ogS0ym5KHVbR7Hx/oYCX01qGp2baEyfN2HM73kf/7Vp3GvMHWRy+zUXISONEtFv7UTViOXnmFrg==} + '@clickhouse/client@1.10.1': + resolution: {integrity: sha512-Ot/6l4hFALK6NtZDS2UegukfRXWkkftWHCnzKUwanpOQ3Jd+RVKx5dxQreeBG5XcRjt1xyf5904PFjbCnaulXg==} engines: {node: '>=16'} '@esbuild/aix-ppc64@0.21.5': @@ -601,6 +625,14 @@ packages: '@supabase/supabase-js@2.49.1': resolution: {integrity: sha512-lKaptKQB5/juEF5+jzmBeZlz69MdHZuxf+0f50NwhL+IE//m4ZnOeWlsKRjjsM0fVayZiQKqLvYdBn0RLkhGiQ==} + '@types/axios@0.14.4': + resolution: {integrity: sha512-9JgOaunvQdsQ/qW2OPmE5+hCeUB52lQSolecrFrthct55QekhmXEwT203s20RL+UHtCQc15y3VXpby9E7Kkh/g==} + deprecated: This is a stub types definition. axios provides its own type definitions, so you do not need this installed. + + '@types/dotenv@8.2.3': + resolution: {integrity: sha512-g2FXjlDX/cYuc5CiQvyU/6kkbP1JtmGzh0obW50zD7OKeILVL0NSpPWLXVfqoAGQjom2/SLLx9zHq0KXvD6mbw==} + deprecated: This is a stub types definition. dotenv provides its own type definitions, so you do not need this installed. + '@types/estree@1.0.6': resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} @@ -613,9 +645,15 @@ packages: '@types/node@20.17.23': resolution: {integrity: sha512-8PCGZ1ZJbEZuYNTMqywO+Sj4vSKjSjT6Ua+6RFOYlEvIvKQABPtrNkoVSLSKDb4obYcMhspVKmsw8Cm10NFRUg==} + '@types/pg@8.11.11': + resolution: {integrity: sha512-kGT1qKM8wJQ5qlawUrEkXgvMSXoV213KfMGXcwfDwUIfUHXqXYXOfS1nE1LINRJVVVx5wCm70XnFlMHaIcQAfw==} + '@types/phoenix@1.6.6': resolution: {integrity: sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A==} + '@types/uuid@10.0.0': + resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} + '@types/ws@8.18.0': resolution: {integrity: sha512-8svvI3hMyvN0kKCJMvTJP/x6Y/EoQbepff882wL+Sn5QsXb3etnamgrJq4isrBxSJj5L2AuXcI0+bgkoAXGUJw==} @@ -734,6 +772,12 @@ packages: assertion-error@1.1.0: resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.8.3: + resolution: {integrity: sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==} + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -757,6 +801,10 @@ packages: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -772,6 +820,10 @@ packages: check-error@1.0.3: resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} @@ -783,6 +835,10 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -813,6 +869,10 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + denque@2.1.0: resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} engines: {node: '>=0.10'} @@ -837,9 +897,32 @@ packages: resolution: {integrity: sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==} engines: {node: '>=12'} + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + ecdsa-sig-formatter@1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esbuild@0.21.5: resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} engines: {node: '>=12'} @@ -850,6 +933,10 @@ packages: engines: {node: '>=18'} hasBin: true + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -930,6 +1017,19 @@ packages: flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + follow-redirects@1.15.9: + resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.2: + resolution: {integrity: sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==} + engines: {node: '>= 6'} + fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -938,13 +1038,28 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + generic-pool@3.9.0: resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} engines: {node: '>= 4'} + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + get-func-name@2.0.2: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + get-stream@8.0.1: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} @@ -972,6 +1087,10 @@ packages: resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} engines: {node: '>=10'} + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} @@ -979,6 +1098,18 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + hono@4.7.4: resolution: {integrity: sha512-Pst8FuGqz3L7tFF+u9Pu70eI0xa5S3LPUmrNd5Jm8nTHze9FxLTK9Kaj5g/k4UcwuJSXTP65SyHOPLrffpcAJg==} engines: {node: '>=16.9.0'} @@ -1014,6 +1145,10 @@ packages: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -1114,6 +1249,10 @@ packages: magic-string@0.30.17: resolution: {integrity: sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==} + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} @@ -1125,6 +1264,14 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + mimic-fn@4.0.0: resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} engines: {node: '>=12'} @@ -1168,6 +1315,9 @@ packages: resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -1224,6 +1374,48 @@ packages: pathval@1.1.1: resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} + pg-cloudflare@1.1.1: + resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} + + pg-connection-string@2.7.0: + resolution: {integrity: sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + + pg-pool@3.8.0: + resolution: {integrity: sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.8.0: + resolution: {integrity: sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg-types@4.0.2: + resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} + engines: {node: '>=10'} + + pg@8.14.0: + resolution: {integrity: sha512-nXbVpyoaXVmdqlKEzToFf37qzyeeh7mbiXsnoWvstSqohj88yaa/I/Rq/HEVn2QPSZEuLIJa/jSpRDyzjEx4FQ==} + engines: {node: '>= 8.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -1238,6 +1430,41 @@ packages: resolution: {integrity: sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-array@3.0.4: + resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==} + engines: {node: '>=12'} + + postgres-bytea@1.0.0: + resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} + engines: {node: '>=0.10.0'} + + postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + + postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -1246,6 +1473,9 @@ packages: resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -1267,6 +1497,10 @@ packages: redis@4.7.0: resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==} + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -1322,6 +1556,10 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -1331,6 +1569,10 @@ packages: std-env@3.8.1: resolution: {integrity: sha512-vj5lIj3Mwf9D79hBkltk5qmkFI+biIKWS2IBxEyEU3AX1tUf7AoL8nSazCOiiqQsGKIq01SClsKEzweu34uwvA==} + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -1411,6 +1653,10 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} hasBin: true @@ -1496,6 +1742,10 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} @@ -1511,9 +1761,25 @@ packages: utf-8-validate: optional: true + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -1524,11 +1790,11 @@ packages: snapshots: - '@clickhouse/client-common@0.2.10': {} + '@clickhouse/client-common@1.10.1': {} - '@clickhouse/client@0.2.10': + '@clickhouse/client@1.10.1': dependencies: - '@clickhouse/client-common': 0.2.10 + '@clickhouse/client-common': 1.10.1 '@esbuild/aix-ppc64@0.21.5': optional: true @@ -1882,6 +2148,16 @@ snapshots: - bufferutil - utf-8-validate + '@types/axios@0.14.4': + dependencies: + axios: 1.8.3 + transitivePeerDependencies: + - debug + + '@types/dotenv@8.2.3': + dependencies: + dotenv: 16.4.7 + '@types/estree@1.0.6': {} '@types/jsonwebtoken@9.0.9': @@ -1895,8 +2171,16 @@ snapshots: dependencies: undici-types: 6.19.8 + '@types/pg@8.11.11': + dependencies: + '@types/node': 20.17.23 + pg-protocol: 1.8.0 + pg-types: 4.0.2 + '@types/phoenix@1.6.6': {} + '@types/uuid@10.0.0': {} + '@types/ws@8.18.0': dependencies: '@types/node': 20.17.23 @@ -2044,6 +2328,16 @@ snapshots: assertion-error@1.1.0: {} + asynckit@0.4.0: {} + + axios@1.8.3: + dependencies: + follow-redirects: 1.15.9 + form-data: 4.0.2 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + balanced-match@1.0.2: {} brace-expansion@1.1.11: @@ -2075,6 +2369,11 @@ snapshots: cac@6.7.14: {} + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + callsites@3.1.0: {} chai@4.5.0: @@ -2096,6 +2395,12 @@ snapshots: dependencies: get-func-name: 2.0.2 + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + cluster-key-slot@1.1.2: {} color-convert@2.0.1: @@ -2104,6 +2409,10 @@ snapshots: color-name@1.1.4: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + concat-map@0.0.1: {} confbox@0.1.8: {} @@ -2128,6 +2437,8 @@ snapshots: deep-is@0.1.4: {} + delayed-stream@1.0.0: {} + denque@2.1.0: {} detect-libc@2.0.3: @@ -2145,10 +2456,33 @@ snapshots: dotenv@16.4.7: {} + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + ecdsa-sig-formatter@1.0.11: dependencies: safe-buffer: 5.2.1 + emoji-regex@8.0.0: {} + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + esbuild@0.21.5: optionalDependencies: '@esbuild/aix-ppc64': 0.21.5 @@ -2203,6 +2537,8 @@ snapshots: '@esbuild/win32-ia32': 0.25.0 '@esbuild/win32-x64': 0.25.0 + escalade@3.2.0: {} + escape-string-regexp@4.0.0: {} eslint-scope@7.2.2: @@ -2328,15 +2664,46 @@ snapshots: flatted@3.3.3: {} + follow-redirects@1.15.9: {} + + form-data@4.0.2: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + mime-types: 2.1.35 + fs.realpath@1.0.0: {} fsevents@2.3.3: optional: true + function-bind@1.1.2: {} + generic-pool@3.9.0: {} + get-caller-file@2.0.5: {} + get-func-name@2.0.2: {} + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + get-stream@8.0.1: {} get-tsconfig@4.10.0: @@ -2373,10 +2740,22 @@ snapshots: merge2: 1.4.1 slash: 3.0.0 + gopd@1.2.0: {} + graphemer@1.4.0: {} has-flag@4.0.0: {} + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + hono@4.7.4: {} human-signals@5.0.0: {} @@ -2413,6 +2792,8 @@ snapshots: is-extglob@2.1.1: {} + is-fullwidth-code-point@3.0.0: {} + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 @@ -2509,6 +2890,8 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.0 + math-intrinsics@1.1.0: {} + merge-stream@2.0.0: {} merge2@1.4.1: {} @@ -2518,6 +2901,12 @@ snapshots: braces: 3.0.3 picomatch: 2.3.1 + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + mimic-fn@4.0.0: {} minimatch@3.1.2: @@ -2568,6 +2957,8 @@ snapshots: dependencies: path-key: 4.0.0 + obuf@1.1.2: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -2617,6 +3008,53 @@ snapshots: pathval@1.1.1: {} + pg-cloudflare@1.1.1: + optional: true + + pg-connection-string@2.7.0: {} + + pg-int8@1.0.1: {} + + pg-numeric@1.0.2: {} + + pg-pool@3.8.0(pg@8.14.0): + dependencies: + pg: 8.14.0 + + pg-protocol@1.8.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.0 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg-types@4.0.2: + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.4 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + + pg@8.14.0: + dependencies: + pg-connection-string: 2.7.0 + pg-pool: 3.8.0(pg@8.14.0) + pg-protocol: 1.8.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.1.1 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.1.1: {} picomatch@2.3.1: {} @@ -2633,6 +3071,28 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-array@3.0.4: {} + + postgres-bytea@1.0.0: {} + + postgres-bytea@3.0.0: + dependencies: + obuf: 1.1.2 + + postgres-date@1.0.7: {} + + postgres-date@2.1.0: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + postgres-interval@3.0.0: {} + + postgres-range@1.1.4: {} + prelude-ls@1.2.1: {} pretty-format@29.7.0: @@ -2641,6 +3101,8 @@ snapshots: ansi-styles: 5.2.0 react-is: 18.3.1 + proxy-from-env@1.1.0: {} + punycode@2.3.1: {} queue-microtask@1.2.3: {} @@ -2662,6 +3124,8 @@ snapshots: '@redis/search': 1.2.0(@redis/client@1.6.0) '@redis/time-series': 1.1.0(@redis/client@1.6.0) + require-directory@2.1.1: {} + resolve-from@4.0.0: {} resolve-pkg-maps@1.0.0: {} @@ -2719,12 +3183,20 @@ snapshots: source-map-js@1.2.1: {} + split2@4.2.0: {} + stackback@0.0.2: {} standard-as-callback@2.1.0: {} std-env@3.8.1: {} + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -2786,6 +3258,8 @@ snapshots: dependencies: punycode: 2.3.1 + uuid@11.1.0: {} + uuid@9.0.1: {} vite-node@1.6.1(@types/node@20.17.23): @@ -2867,12 +3341,34 @@ snapshots: word-wrap@1.2.5: {} + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrappy@1.0.2: {} ws@8.18.1: {} + xtend@4.0.2: {} + + y18n@5.0.8: {} + yallist@4.0.0: {} + yargs-parser@21.1.1: {} + + yargs@17.7.2: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + yocto-queue@0.1.0: {} yocto-queue@1.1.1: {} diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 2369e86..4b818d3 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -25,6 +25,7 @@ export const config = { clickhouse: { host: process.env.CLICKHOUSE_HOST || 'localhost', port: process.env.CLICKHOUSE_PORT || '8123', + url: process.env.CLICKHOUSE_URL || 'http://localhost:8123', user: process.env.CLICKHOUSE_USER || 'admin', password: process.env.CLICKHOUSE_PASSWORD || 'your_secure_password', database: process.env.CLICKHOUSE_DATABASE || 'promote', diff --git a/backend/src/services/syncService.ts b/backend/src/services/syncService.ts index ab5ed2f..dd3fa30 100644 --- a/backend/src/services/syncService.ts +++ b/backend/src/services/syncService.ts @@ -1,707 +1,67 @@ -import { Pool } from 'pg'; -import supabase from '../utils/supabase'; -import clickhouse from '../utils/clickhouse'; -import config from '../config'; import { randomUUID } from 'crypto'; +import clickhouse from '../utils/clickhouse'; -// Define types for better type safety -interface PostRecord { - post_id: string; - influencer_id: string; - platform: string; - project_id?: string; - title?: string; - description?: string; - published_at: string; - created_at: string; - influencer_name?: string; - followers_count?: number; -} - -interface CommentRecord { - comment_id: string; - post_id: string; - user_id?: string; - content: string; - sentiment_score?: number; - created_at: string; - influencer_id: string; - platform: string; - project_id?: string; -} - -interface InfluencerRecord { - influencer_id: string; - name: string; - platform: string; - followers_count: number; - video_count: number; - updated_at: string; -} - -interface ProjectRecord { - id: string; - name: string; - description?: string; - created_at: string; -} - -interface SyncStats { +/** + * 简单的同步函数，只插入一条测试数据到ClickHouse + */ +export async function syncAllData(fromTimestamp: string): Promise<{ success: boolean; - timestamp: string; - duration: number; // milliseconds - posts_synced: number; - comments_synced: number; - influencer_changes_synced: number; - projects_synced: number; + message: string; + posts?: number; + comments?: number; + influencer_changes?: number; + projects?: number; errors: string[]; -} - -// Initialize PostgreSQL client -const pgPool = new Pool({ - connectionString: process.env.DATABASE_URL || 'postgresql://postgres:postgres@localhost:5432/promote', -}); - -// Batch size -const BATCH_SIZE = 100; - -/** - * Submits sync stats to ClickHouse - * @param stats Sync stats - */ -async function recordSyncStats(stats: SyncStats): Promise { - try { - // 首先检查表是否存在，如果不存在则创建 - await clickhouse.query({ - query: ` - CREATE TABLE IF NOT EXISTS ${config.clickhouse.database}.sync_logs ( - timestamp DateTime, - duration_ms UInt32, - posts_synced UInt32, - comments_synced UInt32, - influencer_changes_synced UInt32, - projects_synced UInt32, - success UInt8, - error_messages String - ) ENGINE = MergeTree() - ORDER BY (timestamp) - ` - }); - - // 构建INSERT语句 - const insertQuery = ` - INSERT INTO ${config.clickhouse.database}.sync_logs - (timestamp, duration_ms, posts_synced, comments_synced, influencer_changes_synced, - projects_synced, success, error_messages) - VALUES ('${stats.timestamp}', ${stats.duration}, ${stats.posts_synced}, - ${stats.comments_synced}, ${stats.influencer_changes_synced}, - ${stats.projects_synced}, ${stats.success ? 1 : 0}, '${stats.errors.join('; ').replace(/'/g, "\\'")}')` - - console.log('[DEBUG] 要执行的同步统计插入语句:', insertQuery); - - // 注释掉实际执行的代码 - // await clickhouse.query({ - // query: insertQuery - // }); - } catch (error) { - console.error('Failed to record sync stats:', error); - } -} - -/** - * 转义ClickHouse字符串中的特殊字符 - */ -function escapeClickHouseString(str: string): string { - if (!str) return ''; - return str.replace(/'/g, "\\'"); -} - -/** - * Syncs new posts from PostgreSQL to ClickHouse - * @param lastSyncTimestamp The timestamp of the last sync - */ -export async function syncNewPosts(lastSyncTimestamp: string): Promise { - try { - // Get new posts from PostgreSQL - const query = ` - SELECT - p.post_id, - p.influencer_id, - p.platform, - p.project_id, - p.title, - p.description, - p.published_at, - p.created_at, - i.name as influencer_name, - i.followers_count - FROM posts p - JOIN influencers i ON p.influencer_id = i.influencer_id - WHERE p.created_at > $1 - ORDER BY p.created_at - `; - - const { rows: posts } = await pgPool.query(query, [lastSyncTimestamp]); - - if (posts.length === 0) { - console.log('No new posts to sync'); - return 0; - } - - console.log(`Found ${posts.length} new posts to sync`); - - let syncedCount = 0; - - // Batch processing to avoid processing too much data at once - for (let i = 0; i < posts.length; i += BATCH_SIZE) { - const batch = posts.slice(i, i + BATCH_SIZE); - - try { - // 准备批量插入的值部分 - const values = batch.map(post => { - const eventId = randomUUID(); - const timestamp = new Date(post.created_at).toISOString(); - const date = timestamp.split('T')[0]; - const hour = new Date(post.created_at).getHours(); - const contentType = determineContentType(post.title || '', post.description || ''); - const keywords = JSON.stringify(extractKeywords(post.title || '')); - - return `('${eventId}', '${timestamp}', '${date}', ${hour}, '', '${post.influencer_id}', '${post.post_id}', '${post.project_id || ''}', 'impression', 'exposure', '${escapeClickHouseString(post.platform)}', '${contentType}', 'approved', 'neutral', '', ${keywords}, 1.0, ${post.followers_count || 0}, 0, 0, 0, 0, '', '', '', '', '', '', '')`; - }).join(', '); - - // 构建完整插入查询 - const insertQuery = ` - INSERT INTO ${config.clickhouse.database}.events - (event_id, timestamp, date, hour, user_id, influencer_id, content_id, project_id, - event_type, funnel_stage, platform, content_type, content_status, sentiment, - comment_text, keywords, interaction_value, followers_count, followers_change, - likes_count, likes_change, views_count, ip, user_agent, device_type, referrer, - geo_country, geo_city, session_id) - VALUES ${values}`; - - console.log(`[DEBUG] 批次 ${i / BATCH_SIZE + 1} 帖子插入语句 (前500字符): ${insertQuery.substring(0, 500)}...`); - - // 看看values的值 - if (batch.length > 0) { - console.log(`[DEBUG] 第一条帖子数据值: ${values.split('),')[0]})`); - } - - // 注释掉实际执行的代码 - // await clickhouse.query({ - // query: insertQuery - // }); - - syncedCount += batch.length; - console.log(`[DEBUG] 模拟同步批次 ${batch.length} 帖子 (${syncedCount}/${posts.length})`); - } catch (error) { - console.error(`Error syncing post batch ${i / BATCH_SIZE + 1}:`, error); - } - } - - console.log(`[DEBUG] 模拟成功同步 ${syncedCount} 帖子到 ClickHouse`); - return syncedCount; - } catch (error) { - console.error('Error syncing new posts:', error); - throw error; - } -} - -/** - * Syncs new comments from PostgreSQL to ClickHouse - * @param lastSyncTimestamp The timestamp of the last sync - */ -export async function syncComments(lastSyncTimestamp: string): Promise { - try { - // Get new comments from PostgreSQL - const query = ` - SELECT - c.comment_id, - c.post_id, - c.user_id, - c.content, - c.sentiment_score, - c.created_at, - p.influencer_id, - p.platform, - p.project_id - FROM comments c - JOIN posts p ON c.post_id = p.post_id - WHERE c.created_at > $1 - ORDER BY c.created_at - `; - - const { rows: comments } = await pgPool.query(query, [lastSyncTimestamp]); - - if (comments.length === 0) { - console.log('No new comments to sync'); - return 0; - } - - console.log(`Found ${comments.length} new comments to sync`); - - let syncedCount = 0; - - // Batch processing to avoid processing too much data at once - for (let i = 0; i < comments.length; i += BATCH_SIZE) { - const batch = comments.slice(i, i + BATCH_SIZE); - - try { - // 准备批量插入的值部分 - const values = batch.map(comment => { - const eventId = randomUUID(); - const timestamp = new Date(comment.created_at).toISOString(); - const date = timestamp.split('T')[0]; - const hour = new Date(comment.created_at).getHours(); - const sentiment = determineSentiment(comment.sentiment_score || 0); - const keywords = JSON.stringify(extractKeywords(comment.content)); - const escapedComment = escapeClickHouseString(comment.content); - - return `('${eventId}', '${timestamp}', '${date}', ${hour}, '${comment.user_id || ''}', '${comment.influencer_id}', '${comment.post_id}', '${comment.project_id || ''}', 'comment', 'consideration', '${escapeClickHouseString(comment.platform)}', 'text', 'approved', '${sentiment}', '${escapedComment}', ${keywords}, 3.0, 0, 0, 0, 0, 0, '', '', '', '', '', '', '')`; - }).join(', '); - - // 构建完整插入查询 - const insertQuery = ` - INSERT INTO ${config.clickhouse.database}.events - (event_id, timestamp, date, hour, user_id, influencer_id, content_id, project_id, - event_type, funnel_stage, platform, content_type, content_status, sentiment, - comment_text, keywords, interaction_value, followers_count, followers_change, - likes_count, likes_change, views_count, ip, user_agent, device_type, referrer, - geo_country, geo_city, session_id) - VALUES ${values}`; - - console.log(`[DEBUG] 批次 ${i / BATCH_SIZE + 1} 评论插入语句 (前500字符): ${insertQuery.substring(0, 500)}...`); - - // 看看values的值 - if (batch.length > 0) { - console.log(`[DEBUG] 第一条评论数据值: ${values.split('),')[0]})`); - } - - // 注释掉实际执行的代码 - // await clickhouse.query({ - // query: insertQuery - // }); - - syncedCount += batch.length; - console.log(`[DEBUG] 模拟同步批次 ${batch.length} 评论 (${syncedCount}/${comments.length})`); - } catch (error) { - console.error(`Error syncing comment batch ${i / BATCH_SIZE + 1}:`, error); - } - } - - console.log(`[DEBUG] 模拟成功同步 ${syncedCount} 评论到 ClickHouse`); - return syncedCount; - } catch (error) { - console.error('Error syncing new comments:', error); - throw error; - } -} - -/** - * Syncs project information from PostgreSQL to ClickHouse - * @param lastSyncTimestamp The timestamp of the last sync - */ -export async function syncProjects(lastSyncTimestamp: string): Promise { - try { - // Get new projects and updated projects from PostgreSQL - const query = ` - SELECT - id, - name, - description, - created_at - FROM projects - WHERE created_at > $1 OR updated_at > $1 - ORDER BY created_at - `; - - const { rows: projects } = await pgPool.query(query, [lastSyncTimestamp]); - - if (projects.length === 0) { - console.log('No new projects to sync'); - return 0; - } - - console.log(`Found ${projects.length} projects to sync`); - - let syncedCount = 0; - - // Batch processing - for (let i = 0; i < projects.length; i += BATCH_SIZE) { - const batch = projects.slice(i, i + BATCH_SIZE); - - try { - // 准备批量插入的值部分 - const values = batch.map(project => { - const eventId = randomUUID(); - const timestamp = new Date(project.created_at).toISOString(); - const date = timestamp.split('T')[0]; - const hour = new Date(project.created_at).getHours(); - const keywords = JSON.stringify(extractKeywords(project.name + ' ' + (project.description || ''))); - const escapedDesc = escapeClickHouseString(project.description || ''); - - return `('${eventId}', '${timestamp}', '${date}', ${hour}, '', '', '', '${project.id}', 'project_update', 'interest', 'internal', 'text', 'approved', 'neutral', '${escapedDesc}', ${keywords}, 5.0, 0, 0, 0, 0, 0, '', '', '', '', '', '', '')`; - }).join(', '); - - // 构建完整插入查询 - const insertQuery = ` - INSERT INTO ${config.clickhouse.database}.events - (event_id, timestamp, date, hour, user_id, influencer_id, content_id, project_id, - event_type, funnel_stage, platform, content_type, content_status, sentiment, - comment_text, keywords, interaction_value, followers_count, followers_change, - likes_count, likes_change, views_count, ip, user_agent, device_type, referrer, - geo_country, geo_city, session_id) - VALUES ${values}`; - - console.log(`[DEBUG] 批次 ${i / BATCH_SIZE + 1} 项目插入语句 (前500字符): ${insertQuery.substring(0, 500)}...`); - - // 看看values的值 - if (batch.length > 0) { - console.log(`[DEBUG] 第一条项目数据值: ${values.split('),')[0]})`); - } - - // 注释掉实际执行的代码 - // await clickhouse.query({ - // query: insertQuery - // }); - - syncedCount += batch.length; - console.log(`[DEBUG] 模拟同步批次 ${batch.length} 项目 (${syncedCount}/${projects.length})`); - } catch (error) { - console.error(`Error syncing project batch ${i / BATCH_SIZE + 1}:`, error); - } - } - - console.log(`[DEBUG] 模拟成功同步 ${syncedCount} 项目到 ClickHouse`); - return syncedCount; - } catch (error) { - console.error('Error syncing projects:', error); - throw error; - } -} - -/** - * Syncs influencer metric changes from PostgreSQL to ClickHouse - * @param lastSyncTimestamp The timestamp of the last sync - */ -export async function syncInfluencerChanges(lastSyncTimestamp: string): Promise { - try { - // Get influencers with updated metrics - const query = ` - SELECT - i.influencer_id, - i.name, - i.platform, - i.followers_count, - i.video_count, - i.updated_at - FROM influencers i - WHERE i.updated_at > $1 - ORDER BY i.updated_at - `; - - const { rows: influencers } = await pgPool.query(query, [lastSyncTimestamp]); - - if (influencers.length === 0) { - console.log('No influencer changes to sync'); - return 0; - } - - console.log(`Found ${influencers.length} influencer changes to sync`); - - let syncedCount = 0; - let batchEvents: string[] = []; - - // 从ClickHouse获取所有相关的影响者的最新一条记录 - if (influencers.length > 0) { - try { - const influencerIds = influencers.map(i => `'${i.influencer_id}'`).join(','); - const result = await clickhouse.query({ - query: ` - SELECT - influencer_id AS id, - followers_count, - max(timestamp) AS last_update - FROM ${config.clickhouse.database}.events - WHERE influencer_id IN (${influencerIds}) - AND event_type IN ('follow', 'unfollow', 'impression') - GROUP BY influencer_id, followers_count - ORDER BY last_update DESC - `, - format: 'JSONEachRow' - }); - - // 将结果转换为对象，以便快速查找 - const prevMetricsMap = new Map(); - - // 获取结果中的数据 - try { - // 尝试解析结果 - if ('rows' in result) { - // 如果结果有rows属性，直接使用 - for (const record of result.rows as any[]) { - if (!prevMetricsMap.has(record.id) || - new Date(record.last_update) > new Date(prevMetricsMap.get(record.id)!.last_update)) { - prevMetricsMap.set(record.id, record); - } - } - } else { - // 否则尝试转换结果为JSON - // 使用同步方法处理结果，避免使用text()方法 - const rows: any[] = []; - try { - // 检查是否有替代方法 - if (typeof result.json === 'function') { - const jsonData = await result.json(); - if (Array.isArray(jsonData)) { - rows.push(...jsonData); - } - } else { - // 假设结果是ResultSet或类似结构 - console.log('Warning: Using fallback method to process query results'); - // 无法直接处理结果，使用空数组继续 - } - } catch (parseError) { - console.error('Error parsing ClickHouse result:', parseError); - } - - for (const record of rows) { - const typedRecord = record as { id: string; followers_count: number; last_update: string }; - if (!prevMetricsMap.has(typedRecord.id) || - new Date(typedRecord.last_update) > new Date(prevMetricsMap.get(typedRecord.id)!.last_update)) { - prevMetricsMap.set(typedRecord.id, typedRecord); - } - } - } - } catch (e) { - console.error('Error processing ClickHouse result:', e); - } - - // 处理每个影响者的变化 - for (const influencer of influencers) { - try { - // 获取之前的指标 - const prevMetrics = prevMetricsMap.get(influencer.influencer_id); - const prevFollowersCount = prevMetrics ? Number(prevMetrics.followers_count) || 0 : 0; - - // 计算粉丝变化 - const followersChange = influencer.followers_count - prevFollowersCount; - - // 只有在有实际变化时才创建事件 - if (followersChange !== 0) { - const eventId = randomUUID(); - const timestamp = new Date(influencer.updated_at).toISOString(); - const date = timestamp.split('T')[0]; - const hour = new Date(influencer.updated_at).getHours(); - const eventType = followersChange > 0 ? 'follow' : 'unfollow'; - - batchEvents.push(`('${eventId}', '${timestamp}', '${date}', ${hour}, '', '${influencer.influencer_id}', '', '', '${eventType}', 'interest', '${escapeClickHouseString(influencer.platform)}', 'text', 'approved', 'neutral', '', '[]', 2.0, ${influencer.followers_count}, ${followersChange}, 0, 0, 0, '', '', '', '', '', '', '')`); - - syncedCount++; - } - } catch (error) { - console.error(`Error processing influencer ${influencer.influencer_id}:`, error); - // 继续处理下一个影响者 - } - } - } catch (error) { - console.error('Error querying previous metrics:', error); - } - } - - // 如果有要插入的事件，批量插入 - if (batchEvents.length > 0) { - try { - // 构建完整插入查询 - const insertQuery = ` - INSERT INTO ${config.clickhouse.database}.events - (event_id, timestamp, date, hour, user_id, influencer_id, content_id, project_id, - event_type, funnel_stage, platform, content_type, content_status, sentiment, - comment_text, keywords, interaction_value, followers_count, followers_change, - likes_count, likes_change, views_count, ip, user_agent, device_type, referrer, - geo_country, geo_city, session_id) - VALUES ${batchEvents.join(', ')}`; - - console.log(`[DEBUG] KOL变化插入语句 (前500字符): ${insertQuery.substring(0, 500)}...`); - - // 看看values的值 - if (batchEvents.length > 0) { - console.log(`[DEBUG] 第一条KOL变化数据值: ${batchEvents[0]}`); - } - - // 注释掉实际执行的代码 - // await clickhouse.query({ - // query: insertQuery - // }); - - console.log(`[DEBUG] 模拟同步 ${batchEvents.length} KOL变化`); - } catch (error) { - console.error(`Error syncing influencer batch:`, error); - syncedCount = 0; // 失败时重置同步计数 - } - } else { - console.log('No follower changes detected, skipping influencer sync'); - } - - console.log(`[DEBUG] 模拟成功同步 ${syncedCount} KOL变化到 ClickHouse`); - return syncedCount; - } catch (error) { - console.error('Error syncing influencer changes:', error); - throw error; - } -} - -/** - * Syncs all data from PostgreSQL to ClickHouse - * @param lastSyncTimestamp The timestamp of the last sync - */ -export async function syncAllData(lastSyncTimestamp: string): Promise<{ - posts: number; - comments: number; - influencer_changes: number; - projects: number; - success: boolean; - errors: string[]; - duration: number; }> { - const startTime = Date.now(); + console.log(`开始同步数据，时间范围: ${fromTimestamp} - 现在`); const errors: string[] = []; - let postsCount = 0; - let commentsCount = 0; - let influencerChangesCount = 0; - let projectsCount = 0; - let success = true; - + try { - // Sync new posts - try { - postsCount = await syncNewPosts(lastSyncTimestamp); - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - errors.push(`Posts sync error: ${errorMessage}`); - success = false; - } + // 使用insert方法并仅提供必要字段，让ClickHouse为其他字段使用默认值 + await clickhouse.insert({ + table: 'events', + values: [{ + // 让ClickHouse自动生成event_id、timestamp、date和hour + user_id: 'test-user-123', + influencer_id: 'influencer-456', + content_id: 'content-789', + project_id: 'project-abc', + event_type: 'comment', + funnel_stage: 'consideration', + platform: 'instagram', + content_type: 'text', + content_status: 'approved', + sentiment: 'positive', + comment_text: '测试数据 - ClickHouse同步测试' + }], + format: 'JSONEachRow' // 使用JSONEachRow格式 + }); - // Sync new comments - try { - commentsCount = await syncComments(lastSyncTimestamp); - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - errors.push(`Comments sync error: ${errorMessage}`); - success = false; - } + console.log('数据插入成功'); - // Sync influencer changes - try { - influencerChangesCount = await syncInfluencerChanges(lastSyncTimestamp); - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - errors.push(`Influencer changes sync error: ${errorMessage}`); - success = false; - } + // 只计算了一条评论 + const comments = 1; - // Sync projects - try { - projectsCount = await syncProjects(lastSyncTimestamp); - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - errors.push(`Projects sync error: ${errorMessage}`); - success = false; - } - - // Record sync stats - const endTime = Date.now(); - const duration = endTime - startTime; - const syncStats: SyncStats = { - success, - timestamp: new Date().toISOString(), - duration, - posts_synced: postsCount, - comments_synced: commentsCount, - influencer_changes_synced: influencerChangesCount, - projects_synced: projectsCount, + return { + success: true, + message: '测试数据插入成功', + comments, + posts: 0, + influencer_changes: 0, + projects: 0, errors }; - - await recordSyncStats(syncStats); - + } catch (err: any) { + console.error('数据插入失败:', err.message); + errors.push(err.message); return { - posts: postsCount, - comments: commentsCount, - influencer_changes: influencerChangesCount, - projects: projectsCount, - success, - errors, - duration - }; - } catch (error: unknown) { - console.error('Error in syncAllData:', error); - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - return { - posts: postsCount, - comments: commentsCount, - influencer_changes: influencerChangesCount, - projects: projectsCount, success: false, - errors: [...errors, `General sync error: ${errorMessage}`], - duration: Date.now() - startTime + message: `插入失败: ${err.message}`, + comments: 0, + posts: 0, + influencer_changes: 0, + projects: 0, + errors }; } } - -/** - * Helper function to determine content type based on title/description - */ -function determineContentType(title: string, description: string = ''): string { - const text = (title + ' ' + description).toLowerCase(); - - if (text.includes('video') || text.includes('watch') || text.includes('视频')) return 'video'; - if (text.includes('image') || text.includes('photo') || text.includes('pic') || text.includes('图片')) return 'image'; - if (text.includes('story') || text.includes('故事')) return 'story'; - if (text.includes('reel') || text.includes('短视频')) return 'reel'; - if (text.includes('live') || text.includes('直播')) return 'live'; - - // Default - return 'text'; -} - -/** - * Helper function to determine sentiment from score - */ -function determineSentiment(score: number): string { - if (!score && score !== 0) return 'neutral'; - - if (score > 0.3) return 'positive'; - if (score < -0.3) return 'negative'; - return 'neutral'; -} - -/** - * Helper function to extract keywords from text - */ -function extractKeywords(text: string): string[] { - if (!text) return []; - - // Convert to lowercase - const lower = text.toLowerCase(); - - // Remove special characters and split into words - const words = lower.replace(/[^\w\s]/g, ' ').split(/\s+/); - - // Filter out common words (simple stop words list) - const stopWords = new Set([ - 'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'with', - 'about', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', - 'had', 'do', 'does', 'did', 'i', 'you', 'he', 'she', 'it', 'we', 'they', - 'this', 'that', 'these', 'those', 'of', 'by', 'from', 'as', 'if', 'then', - 'than', 'so', 'what', 'when', 'where', 'how', 'all', 'any', 'both', 'each', - '我', '你', '他', '她', '它', '们', '的', '和', '是', '在', '了', '有', '就', - '都', '而', '及', '与', '这', '那', '不', '但', '如', '要', '可以', '会' - ]); - - const keywords = words - .filter(word => word.length > 2) // Filter out short words - .filter(word => !stopWords.has(word)) // Filter out stop words - .slice(0, 10); // Limit to 10 keywords - - return [...new Set(keywords)]; // Remove duplicates -} \ No newline at end of file diff --git a/backend/src/utils/clickhouse.ts b/backend/src/utils/clickhouse.ts index 8cfad49..dacf8e9 100644 --- a/backend/src/utils/clickhouse.ts +++ b/backend/src/utils/clickhouse.ts @@ -5,10 +5,14 @@ import config from '../config'; const createClickHouseClient = () => { try { return createClient({ - host: `http://${config.clickhouse.host}:${config.clickhouse.port}`, + url: `http://${config.clickhouse.host}:${config.clickhouse.port}`, username: config.clickhouse.user, password: config.clickhouse.password, database: config.clickhouse.database, + request_timeout: 30000, + keep_alive: { + enabled: true, + }, }); } catch (error) { console.error('Error creating ClickHouse client:', error); @@ -18,6 +22,10 @@ const createClickHouseClient = () => { console.log('ClickHouse query (mock):', query, values); return { rows: [] }; }, + insert: async ({ table, values, format }: { table: string; values: any[]; format?: string }) => { + console.log('ClickHouse insert (mock):', { table, values, format }); + return { rows: [] }; + }, close: async () => { console.log('ClickHouse connection closed (mock)'); } @@ -28,4 +36,4 @@ const createClickHouseClient = () => { const clickhouse = createClickHouseClient(); -export default clickhouse; \ No newline at end of file +export default clickhouse; \ No newline at end of file