kol overview

This commit is contained in:
2025-03-13 18:17:27 +08:00
parent f5c660217a
commit 6d29a208f1
7 changed files with 806 additions and 1 deletions

View File

@@ -37,6 +37,7 @@
"@supabase/supabase-js": "^2.49.1", "@supabase/supabase-js": "^2.49.1",
"@types/axios": "^0.14.4", "@types/axios": "^0.14.4",
"@types/dotenv": "^8.2.3", "@types/dotenv": "^8.2.3",
"@types/express": "^5.0.0",
"@types/jsonwebtoken": "^9.0.6", "@types/jsonwebtoken": "^9.0.6",
"@types/node": "^20.11.30", "@types/node": "^20.11.30",
"@types/pg": "^8.11.11", "@types/pg": "^8.11.11",
@@ -52,4 +53,4 @@
"uuid": "^11.1.0", "uuid": "^11.1.0",
"vitest": "^1.4.0" "vitest": "^1.4.0"
} }
} }

View File

@@ -0,0 +1,119 @@
import { Context } from 'hono';
import { analyticsService } from '../services/analyticsService';
import { logger } from '../utils/logger';
/**
* Controller for analytics endpoints
*/
export class AnalyticsController {
/**
* Get KOL performance overview
* Returns card-style layout showing key performance metrics for each KOL
*
* @param c Hono Context
* @returns Response with KOL performance data
*/
async getKolOverview(c: Context) {
const requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 10)}`;
const startTime = Date.now();
try {
// Get query parameters for time range filtering
const timeRange = c.req.query('timeRange') || '30'; // Default to 30 days
const projectId = c.req.query('projectId'); // Optional project filter
const sortBy = c.req.query('sortBy') || 'followers_change'; // Default sort by followers change
const sortOrder = c.req.query('sortOrder') || 'desc'; // Default to descending order
const limit = parseInt(c.req.query('limit') || '20', 10); // Default limit to 20 KOLs
const offset = parseInt(c.req.query('offset') || '0', 10); // Default offset to 0
const debug = c.req.query('debug') || 'false'; // Default debug to false
logger.info(`[${requestId}] KOL overview request received`, {
timeRange,
projectId,
sortBy,
sortOrder,
limit,
offset,
debug,
userAgent: c.req.header('user-agent'),
ip: c.req.header('x-forwarded-for') || 'unknown'
});
// Validate time range
if (!['7', '30', '90'].includes(timeRange)) {
logger.warn(`[${requestId}] Invalid timeRange: ${timeRange}`);
return c.json({
success: false,
error: 'Invalid timeRange. Must be 7, 30, or 90.'
}, 400);
}
// Validate sort order
if (!['asc', 'desc'].includes(sortOrder)) {
logger.warn(`[${requestId}] Invalid sortOrder: ${sortOrder}`);
return c.json({
success: false,
error: 'Invalid sortOrder. Must be asc or desc.'
}, 400);
}
// Validate sort field
const validSortFields = ['followers_change', 'likes_change', 'follows_change', 'followers_count'];
if (!validSortFields.includes(sortBy)) {
logger.warn(`[${requestId}] Invalid sortBy: ${sortBy}`);
return c.json({
success: false,
error: `Invalid sortBy. Must be one of: ${validSortFields.join(', ')}`
}, 400);
}
// Get KOL overview data from the service
const data = await analyticsService.getKolPerformanceOverview(
parseInt(timeRange, 10),
projectId,
sortBy,
sortOrder,
limit,
offset
);
// Debug mode - log additional event data
if (debug.toLowerCase() === 'true' && process.env.NODE_ENV !== 'production') {
await analyticsService.debugEventData();
}
// Log successful response
const duration = Date.now() - startTime;
logger.info(`[${requestId}] KOL overview response sent successfully`, {
duration,
resultCount: data.kols.length,
totalRecords: data.total
});
// Return the data
return c.json({
success: true,
data: data.kols,
pagination: {
limit,
offset,
total: data.total
}
});
} catch (error) {
// Log error
const duration = Date.now() - startTime;
logger.error(`[${requestId}] Error fetching KOL overview (${duration}ms)`, error);
// Return error response
return c.json({
success: false,
error: 'Failed to fetch KOL overview data',
message: error instanceof Error ? error.message : 'Unknown error'
}, 500);
}
}
}
// Export singleton instance
export const analyticsController = new AnalyticsController();

View File

@@ -10,6 +10,7 @@ import projectCommentsRouter from './routes/projectComments';
import commentsRouter from './routes/comments'; import commentsRouter from './routes/comments';
import influencersRouter from './routes/influencers'; import influencersRouter from './routes/influencers';
import projectsRouter from './routes/projects'; import projectsRouter from './routes/projects';
import analyticsRouter from './routes/analytics';
import { connectRedis } from './utils/redis'; import { connectRedis } from './utils/redis';
import { initWorkers } from './utils/queue'; import { initWorkers } from './utils/queue';
import { createSwaggerUI } from './swagger'; import { createSwaggerUI } from './swagger';
@@ -45,6 +46,7 @@ app.route('/api/project-comments', projectCommentsRouter);
app.route('/api/comments', commentsRouter); app.route('/api/comments', commentsRouter);
app.route('/api/influencers', influencersRouter); app.route('/api/influencers', influencersRouter);
app.route('/api/projects', projectsRouter); app.route('/api/projects', projectsRouter);
app.route('/api/analytics', analyticsRouter);
// Swagger UI // Swagger UI
const swaggerApp = createSwaggerUI(); const swaggerApp = createSwaggerUI();

View File

@@ -0,0 +1,29 @@
import { Hono } from 'hono';
import { analyticsController } from '../controllers/analyticsController';
import { logger } from '../utils/logger';
// Create analytics router
const analyticsRouter = new Hono();
// Log all analytics requests
analyticsRouter.use('*', async (c, next) => {
const startTime = Date.now();
const path = c.req.path;
const method = c.req.method;
logger.info(`Analytics API request: ${method} ${path}`, {
query: c.req.query(),
userAgent: c.req.header('user-agent'),
referer: c.req.header('referer')
});
await next();
const duration = Date.now() - startTime;
logger.info(`Analytics API response: ${method} ${path} completed in ${duration}ms`);
});
// KOL performance overview endpoint
analyticsRouter.get('/kol-overview', (c) => analyticsController.getKolOverview(c));
export default analyticsRouter;

View File

@@ -0,0 +1,374 @@
import clickhouse from '../utils/clickhouse';
import { logger } from '../utils/logger';
import { ResultSet } from '@clickhouse/client';
/**
* Represents a KOL's performance overview data
*/
export interface KolPerformanceData {
influencer_id: string;
name: string;
platform: string;
profile_url: string;
followers_count: number;
followers_change: number;
followers_change_percentage: number | null;
likes_change: number;
likes_change_percentage: number | null;
follows_change: number;
follows_change_percentage: number | null;
}
/**
* Response structure for KOL performance overview
*/
export interface KolPerformanceResponse {
kols: KolPerformanceData[];
total: number;
}
/**
* Analytics service for KOL performance data
*/
export class AnalyticsService {
/**
* Get KOL performance overview from ClickHouse
* @param timeRange Number of days to look back (7, 30, or 90)
* @param projectId Optional project ID to filter by
* @param sortBy Field to sort by (followers_change, likes_change, follows_change)
* @param sortOrder Sort order (asc or desc)
* @param limit Number of KOLs to return
* @param offset Offset for pagination
* @returns KOL performance overview data
*/
async getKolPerformanceOverview(
timeRange: number,
projectId?: string,
sortBy: string = 'followers_change',
sortOrder: string = 'desc',
limit: number = 20,
offset: number = 0
): Promise<KolPerformanceResponse> {
const startTime = Date.now();
logger.info('Fetching KOL performance overview', { timeRange, projectId, sortBy, sortOrder, limit, offset });
try {
// First let's check if we have any data in the tables
await this.checkDataExistence();
// Calculate date for time range
const currentDate = new Date();
const pastDate = new Date();
pastDate.setDate(currentDate.getDate() - timeRange);
// Format dates for SQL
const currentDateStr = this.formatDateForClickhouse(currentDate);
const pastDateStr = this.formatDateForClickhouse(pastDate);
// Build project filter condition
const projectFilter = projectId ? `AND project_id = '${projectId}'` : '';
// First get the influencers
const influencersQuery = `
SELECT
influencer_id,
name,
platform,
profile_url,
followers_count
FROM
influencers
WHERE
1=1 ${projectFilter}
ORDER BY
followers_count DESC
LIMIT ${limit}
OFFSET ${offset}
`;
// Query to get total count for pagination
const countQuery = `
SELECT
COUNT(*) as total
FROM
influencers
WHERE
1=1 ${projectFilter}
`;
logger.debug('Executing ClickHouse queries for influencers', {
influencersQuery: influencersQuery.replace(/\n\s+/g, ' ').trim(),
countQuery: countQuery.replace(/\n\s+/g, ' ').trim()
});
// Execute the influencers and count queries
const [influencersData, countResult] = await Promise.all([
this.executeClickhouseQuery(influencersQuery),
this.executeClickhouseQuery(countQuery)
]);
// If we have no influencers, return empty result
if (!Array.isArray(influencersData) || influencersData.length === 0) {
logger.info('No influencers found for the given criteria');
return {
kols: [],
total: 0
};
}
// Get the list of influencer IDs to fetch events for
const influencerIds = influencersData.map(inf => inf.influencer_id);
const influencerIdsList = influencerIds.map(id => `'${id}'`).join(', ');
// Now fetch the event metrics for these influencers
const eventsQuery = `
SELECT
influencer_id,
-- Current period metrics
SUM(CASE WHEN event_type = 'follow' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) -
SUM(CASE WHEN event_type = 'unfollow' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) AS followers_change,
SUM(CASE WHEN event_type = 'like' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) -
SUM(CASE WHEN event_type = 'unlike' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) AS likes_change,
SUM(CASE WHEN event_type = 'follow' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) AS follows_change
FROM
events
WHERE
influencer_id IN (${influencerIdsList})
GROUP BY
influencer_id
`;
logger.debug('Executing ClickHouse query for events', {
eventsQuery: eventsQuery.replace(/\n\s+/g, ' ').trim()
});
// Execute the events query
const eventsData = await this.executeClickhouseQuery(eventsQuery);
// Create a map of influencer_id to events data for fast lookup
const eventsMap: Record<string, any> = {};
if (Array.isArray(eventsData)) {
eventsData.forEach(event => {
if (event.influencer_id) {
eventsMap[event.influencer_id] = event;
}
});
}
// Combine the influencer data with events data
const kols = influencersData.map(influencer => {
const events = eventsMap[influencer.influencer_id] || {};
return {
influencer_id: String(influencer.influencer_id || ''),
name: String(influencer.name || ''),
platform: String(influencer.platform || ''),
profile_url: String(influencer.profile_url || ''),
followers_count: Number(influencer.followers_count || 0),
followers_change: Number(events.followers_change || 0),
followers_change_percentage: null, // We'll calculate this in a separate query if needed
likes_change: Number(events.likes_change || 0),
likes_change_percentage: null,
follows_change: Number(events.follows_change || 0),
follows_change_percentage: null
};
});
// Sort the results based on the requested sort field and order
kols.sort((a: any, b: any) => {
const aValue = a[sortBy] || 0;
const bValue = b[sortBy] || 0;
return sortOrder.toLowerCase() === 'asc'
? aValue - bValue
: bValue - aValue;
});
const total = this.parseCountResult(countResult);
const duration = Date.now() - startTime;
logger.info('KOL performance overview fetched successfully', {
duration,
kolCount: kols.length,
total
});
return {
kols,
total
};
} catch (error) {
const duration = Date.now() - startTime;
logger.error(`Error in getKolPerformanceOverview (${duration}ms)`, error);
throw error;
}
}
/**
* Check if tables contain data and log the results for diagnostics
*/
private async checkDataExistence(): Promise<void> {
try {
// Check if the influencers table has data
const influencerCountQuery = "SELECT COUNT(*) as count FROM influencers";
const eventsCountQuery = "SELECT COUNT(*) as count FROM events";
const [influencersResult, eventsResult] = await Promise.all([
this.executeClickhouseQuery(influencerCountQuery),
this.executeClickhouseQuery(eventsCountQuery)
]);
const influencersCount = Array.isArray(influencersResult) && influencersResult.length > 0
? influencersResult[0].count
: 0;
const eventsCount = Array.isArray(eventsResult) && eventsResult.length > 0
? eventsResult[0].count
: 0;
logger.info('Data existence check', {
influencers_count: influencersCount,
events_count: eventsCount
});
// Optional: Check for sample data
if (influencersCount > 0) {
const sampleQuery = "SELECT influencer_id, name, platform, profile_url, followers_count FROM influencers LIMIT 2";
const sampleResult = await this.executeClickhouseQuery(sampleQuery);
logger.info('Sample influencer data', { sample: sampleResult });
}
if (eventsCount > 0) {
const sampleQuery = `
SELECT
event_type,
COUNT(*) as count
FROM events
GROUP BY event_type
LIMIT 5
`;
const sampleResult = await this.executeClickhouseQuery(sampleQuery);
logger.info('Sample events data', { sample: sampleResult });
}
} catch (error) {
logger.warn('Error checking data existence', error);
// Don't throw, just continue with the query
}
}
/**
* Execute a ClickHouse query with proper error handling
* @param query SQL query to execute
* @returns Query result
*/
private async executeClickhouseQuery(query: string): Promise<any[]> {
try {
const result = await clickhouse.query({
query,
format: 'JSONEachRow'
});
// Handle different result formats
if ('json' in result) {
return await result.json();
} else if ('rows' in result) {
return result.rows;
}
return [];
} catch (error) {
logger.error('Error executing ClickHouse query', error);
// Re-throw to be handled by the caller
throw new Error(`ClickHouse query error: ${error instanceof Error ? error.message : String(error)}`);
}
}
/**
* Parse count result from ClickHouse
* @param data ClickHouse count query result
* @returns Total count
*/
private parseCountResult(data: any[]): number {
if (!Array.isArray(data) || data.length === 0) {
logger.warn('ClickHouse count result is invalid, returning 0');
return 0;
}
return Number(data[0]?.total || 0);
}
/**
* Format date for ClickHouse query
* @param date Date to format
* @returns Formatted date string (YYYY-MM-DD)
*/
private formatDateForClickhouse(date: Date): string {
return date.toISOString().split('T')[0];
}
/**
* Debug event data - only called if debug=true parameter is set
*/
async debugEventData(): Promise<void> {
try {
// Check events table
const eventCountQuery = "SELECT COUNT(*) as count FROM events";
const eventTypesQuery = "SELECT event_type, COUNT(*) as count FROM events GROUP BY event_type";
const sampleEventsQuery = "SELECT * FROM events LIMIT 5";
const sampleInfluencersEventsQuery = `
SELECT e.event_type, e.influencer_id, e.date, i.name
FROM events e
JOIN influencers i ON e.influencer_id = i.influencer_id
LIMIT 10
`;
const [eventCount, eventTypes, sampleEvents, joinedEvents] = await Promise.all([
this.executeClickhouseQuery(eventCountQuery),
this.executeClickhouseQuery(eventTypesQuery),
this.executeClickhouseQuery(sampleEventsQuery),
this.executeClickhouseQuery(sampleInfluencersEventsQuery).catch(() => [])
]);
logger.info('DEBUG: Event counts', {
total_events: eventCount[0]?.count || 0,
event_types: eventTypes,
});
logger.info('DEBUG: Sample events', {
sample_events: sampleEvents,
joined_events: joinedEvents
});
// Try executing our specific events query directly
const eventsQuery = `
SELECT
influencer_id,
SUM(CASE WHEN event_type = 'follow' THEN 1 ELSE 0 END) AS follows,
SUM(CASE WHEN event_type = 'unfollow' THEN 1 ELSE 0 END) AS unfollows,
SUM(CASE WHEN event_type = 'like' THEN 1 ELSE 0 END) AS likes,
SUM(CASE WHEN event_type = 'unlike' THEN 1 ELSE 0 END) AS unlikes
FROM events
GROUP BY influencer_id
LIMIT 10
`;
const eventsResult = await this.executeClickhouseQuery(eventsQuery).catch(err => {
logger.error('DEBUG: Error executing events query', err);
return [];
});
logger.info('DEBUG: Aggregated events', {
events_result: eventsResult
});
} catch (error) {
logger.warn('Error in debugEventData', error);
}
}
}
// Export singleton instance
export const analyticsService = new AnalyticsService();

View File

@@ -2088,6 +2088,157 @@ export const openAPISpec = {
} }
} }
}, },
'/api/analytics/kol-overview': {
get: {
summary: 'Get KOL performance overview',
description: 'Returns card-style layout showing key performance metrics for each KOL including followers growth, new likes, and new follows',
tags: ['Analytics'],
parameters: [
{
name: 'timeRange',
in: 'query',
description: 'Number of days to look back',
schema: {
type: 'string',
enum: ['7', '30', '90'],
default: '30'
}
},
{
name: 'projectId',
in: 'query',
description: 'Filter by project ID',
schema: {
type: 'string'
}
},
{
name: 'sortBy',
in: 'query',
description: 'Field to sort by',
schema: {
type: 'string',
enum: ['followers_change', 'likes_change', 'follows_change'],
default: 'followers_change'
}
},
{
name: 'sortOrder',
in: 'query',
description: 'Sort order',
schema: {
type: 'string',
enum: ['asc', 'desc'],
default: 'desc'
}
},
{
name: 'limit',
in: 'query',
description: 'Number of KOLs to return',
schema: {
type: 'integer',
default: 20
}
},
{
name: 'offset',
in: 'query',
description: 'Offset for pagination',
schema: {
type: 'integer',
default: 0
}
}
],
responses: {
'200': {
description: 'Successful response with KOL performance data',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
success: {
type: 'boolean',
example: true
},
data: {
type: 'array',
items: {
type: 'object',
properties: {
influencer_id: { type: 'string', example: 'inf-123-456' },
name: { type: 'string', example: 'John Influencer' },
platform: { type: 'string', example: 'instagram' },
profile_url: { type: 'string', example: 'https://instagram.com/johninfluencer' },
followers_count: { type: 'integer', example: 50000 },
followers_change: { type: 'integer', example: 1500 },
followers_change_percentage: {
type: 'number',
nullable: true,
example: 12.5
},
likes_change: { type: 'integer', example: 2800 },
likes_change_percentage: {
type: 'number',
nullable: true,
example: 15.3
},
follows_change: { type: 'integer', example: 1200 },
follows_change_percentage: {
type: 'number',
nullable: true,
example: 8.7
}
}
}
},
pagination: {
type: 'object',
properties: {
limit: { type: 'integer', example: 20 },
offset: { type: 'integer', example: 0 },
total: { type: 'integer', example: 42 }
}
}
}
}
}
}
},
'400': {
description: 'Bad request - invalid parameters',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
success: { type: 'boolean', example: false },
error: { type: 'string', example: 'Invalid sortBy. Must be one of: followers_change, likes_change, follows_change' }
}
}
}
}
},
'500': {
description: 'Server error',
content: {
'application/json': {
schema: {
type: 'object',
properties: {
success: { type: 'boolean', example: false },
error: { type: 'string', example: 'Failed to fetch KOL overview data' },
message: { type: 'string', example: 'ClickHouse query error: Connection refused' }
}
}
}
}
}
}
}
},
}, },
components: { components: {
schemas: { schemas: {

129
backend/src/utils/logger.ts Normal file
View File

@@ -0,0 +1,129 @@
import fs from 'fs';
import path from 'path';
import config from '../config';
/**
* Logger levels
*/
export enum LogLevel {
DEBUG = 'DEBUG',
INFO = 'INFO',
WARN = 'WARN',
ERROR = 'ERROR'
}
/**
* Utility class for logging messages to console and file
*/
class Logger {
private logDir: string;
private logFile: string;
private debugEnabled: boolean;
constructor() {
// Set up log directory
this.logDir = path.join(process.cwd(), 'logs');
// Create logs directory if it doesn't exist
if (!fs.existsSync(this.logDir)) {
fs.mkdirSync(this.logDir, { recursive: true });
}
// Set log file path with date in filename
const now = new Date();
const dateStr = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`;
this.logFile = path.join(this.logDir, `app-${dateStr}.log`);
// Enable debug logs based on environment (NODE_ENV)
this.debugEnabled = process.env.NODE_ENV !== 'production';
}
/**
* Format log message with timestamp and level
*/
private formatMessage(level: LogLevel, message: string, data?: any): string {
const timestamp = new Date().toISOString();
let formattedMessage = `[${timestamp}] [${level}] ${message}`;
if (data) {
if (typeof data === 'object') {
try {
const dataStr = JSON.stringify(data);
formattedMessage += ` - ${dataStr}`;
} catch (error) {
formattedMessage += ` - [Object cannot be stringified]`;
}
} else {
formattedMessage += ` - ${data}`;
}
}
return formattedMessage;
}
/**
* Write log to file
*/
private writeToFile(message: string): void {
fs.appendFileSync(this.logFile, message + '\n');
}
/**
* Debug level log
*/
debug(message: string, data?: any): void {
if (!this.debugEnabled) return;
const formattedMessage = this.formatMessage(LogLevel.DEBUG, message, data);
console.debug(formattedMessage);
this.writeToFile(formattedMessage);
}
/**
* Info level log
*/
info(message: string, data?: any): void {
const formattedMessage = this.formatMessage(LogLevel.INFO, message, data);
console.info(formattedMessage);
this.writeToFile(formattedMessage);
}
/**
* Warning level log
*/
warn(message: string, data?: any): void {
const formattedMessage = this.formatMessage(LogLevel.WARN, message, data);
console.warn(formattedMessage);
this.writeToFile(formattedMessage);
}
/**
* Error level log
*/
error(message: string, error?: any): void {
let errorData = error;
// Handle Error objects
if (error instanceof Error) {
errorData = {
errorMessage: error.message,
stack: error.stack,
name: error.name
};
}
const formattedMessage = this.formatMessage(LogLevel.ERROR, message, errorData);
console.error(formattedMessage);
this.writeToFile(formattedMessage);
}
/**
* Get log file path
*/
getLogFile(): string {
return this.logFile;
}
}
// Export singleton instance
export const logger = new Logger();