diff --git a/backend/package.json b/backend/package.json index e640378..c000e67 100644 --- a/backend/package.json +++ b/backend/package.json @@ -37,6 +37,7 @@ "@supabase/supabase-js": "^2.49.1", "@types/axios": "^0.14.4", "@types/dotenv": "^8.2.3", + "@types/express": "^5.0.0", "@types/jsonwebtoken": "^9.0.6", "@types/node": "^20.11.30", "@types/pg": "^8.11.11", @@ -52,4 +53,4 @@ "uuid": "^11.1.0", "vitest": "^1.4.0" } -} \ No newline at end of file +} diff --git a/backend/src/controllers/analyticsController.ts b/backend/src/controllers/analyticsController.ts new file mode 100644 index 0000000..4758ce1 --- /dev/null +++ b/backend/src/controllers/analyticsController.ts @@ -0,0 +1,119 @@ +import { Context } from 'hono'; +import { analyticsService } from '../services/analyticsService'; +import { logger } from '../utils/logger'; + +/** + * Controller for analytics endpoints + */ +export class AnalyticsController { + /** + * Get KOL performance overview + * Returns card-style layout showing key performance metrics for each KOL + * + * @param c Hono Context + * @returns Response with KOL performance data + */ + async getKolOverview(c: Context) { + const requestId = `req_${Date.now()}_${Math.random().toString(36).substring(2, 10)}`; + const startTime = Date.now(); + + try { + // Get query parameters for time range filtering + const timeRange = c.req.query('timeRange') || '30'; // Default to 30 days + const projectId = c.req.query('projectId'); // Optional project filter + const sortBy = c.req.query('sortBy') || 'followers_change'; // Default sort by followers change + const sortOrder = c.req.query('sortOrder') || 'desc'; // Default to descending order + const limit = parseInt(c.req.query('limit') || '20', 10); // Default limit to 20 KOLs + const offset = parseInt(c.req.query('offset') || '0', 10); // Default offset to 0 + const debug = c.req.query('debug') || 'false'; // Default debug to false + + logger.info(`[${requestId}] KOL overview request received`, { + timeRange, + projectId, + sortBy, + sortOrder, + limit, + offset, + debug, + userAgent: c.req.header('user-agent'), + ip: c.req.header('x-forwarded-for') || 'unknown' + }); + + // Validate time range + if (!['7', '30', '90'].includes(timeRange)) { + logger.warn(`[${requestId}] Invalid timeRange: ${timeRange}`); + return c.json({ + success: false, + error: 'Invalid timeRange. Must be 7, 30, or 90.' + }, 400); + } + + // Validate sort order + if (!['asc', 'desc'].includes(sortOrder)) { + logger.warn(`[${requestId}] Invalid sortOrder: ${sortOrder}`); + return c.json({ + success: false, + error: 'Invalid sortOrder. Must be asc or desc.' + }, 400); + } + + // Validate sort field + const validSortFields = ['followers_change', 'likes_change', 'follows_change', 'followers_count']; + if (!validSortFields.includes(sortBy)) { + logger.warn(`[${requestId}] Invalid sortBy: ${sortBy}`); + return c.json({ + success: false, + error: `Invalid sortBy. Must be one of: ${validSortFields.join(', ')}` + }, 400); + } + + // Get KOL overview data from the service + const data = await analyticsService.getKolPerformanceOverview( + parseInt(timeRange, 10), + projectId, + sortBy, + sortOrder, + limit, + offset + ); + + // Debug mode - log additional event data + if (debug.toLowerCase() === 'true' && process.env.NODE_ENV !== 'production') { + await analyticsService.debugEventData(); + } + + // Log successful response + const duration = Date.now() - startTime; + logger.info(`[${requestId}] KOL overview response sent successfully`, { + duration, + resultCount: data.kols.length, + totalRecords: data.total + }); + + // Return the data + return c.json({ + success: true, + data: data.kols, + pagination: { + limit, + offset, + total: data.total + } + }); + } catch (error) { + // Log error + const duration = Date.now() - startTime; + logger.error(`[${requestId}] Error fetching KOL overview (${duration}ms)`, error); + + // Return error response + return c.json({ + success: false, + error: 'Failed to fetch KOL overview data', + message: error instanceof Error ? error.message : 'Unknown error' + }, 500); + } + } +} + +// Export singleton instance +export const analyticsController = new AnalyticsController(); \ No newline at end of file diff --git a/backend/src/index.ts b/backend/src/index.ts index c8f7095..fd86ffe 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -10,6 +10,7 @@ import projectCommentsRouter from './routes/projectComments'; import commentsRouter from './routes/comments'; import influencersRouter from './routes/influencers'; import projectsRouter from './routes/projects'; +import analyticsRouter from './routes/analytics'; import { connectRedis } from './utils/redis'; import { initWorkers } from './utils/queue'; import { createSwaggerUI } from './swagger'; @@ -45,6 +46,7 @@ app.route('/api/project-comments', projectCommentsRouter); app.route('/api/comments', commentsRouter); app.route('/api/influencers', influencersRouter); app.route('/api/projects', projectsRouter); +app.route('/api/analytics', analyticsRouter); // Swagger UI const swaggerApp = createSwaggerUI(); diff --git a/backend/src/routes/analytics.ts b/backend/src/routes/analytics.ts new file mode 100644 index 0000000..d0b83af --- /dev/null +++ b/backend/src/routes/analytics.ts @@ -0,0 +1,29 @@ +import { Hono } from 'hono'; +import { analyticsController } from '../controllers/analyticsController'; +import { logger } from '../utils/logger'; + +// Create analytics router +const analyticsRouter = new Hono(); + +// Log all analytics requests +analyticsRouter.use('*', async (c, next) => { + const startTime = Date.now(); + const path = c.req.path; + const method = c.req.method; + + logger.info(`Analytics API request: ${method} ${path}`, { + query: c.req.query(), + userAgent: c.req.header('user-agent'), + referer: c.req.header('referer') + }); + + await next(); + + const duration = Date.now() - startTime; + logger.info(`Analytics API response: ${method} ${path} completed in ${duration}ms`); +}); + +// KOL performance overview endpoint +analyticsRouter.get('/kol-overview', (c) => analyticsController.getKolOverview(c)); + +export default analyticsRouter; \ No newline at end of file diff --git a/backend/src/services/analyticsService.ts b/backend/src/services/analyticsService.ts new file mode 100644 index 0000000..5221c82 --- /dev/null +++ b/backend/src/services/analyticsService.ts @@ -0,0 +1,374 @@ +import clickhouse from '../utils/clickhouse'; +import { logger } from '../utils/logger'; +import { ResultSet } from '@clickhouse/client'; + +/** + * Represents a KOL's performance overview data + */ +export interface KolPerformanceData { + influencer_id: string; + name: string; + platform: string; + profile_url: string; + followers_count: number; + followers_change: number; + followers_change_percentage: number | null; + likes_change: number; + likes_change_percentage: number | null; + follows_change: number; + follows_change_percentage: number | null; +} + +/** + * Response structure for KOL performance overview + */ +export interface KolPerformanceResponse { + kols: KolPerformanceData[]; + total: number; +} + +/** + * Analytics service for KOL performance data + */ +export class AnalyticsService { + /** + * Get KOL performance overview from ClickHouse + * @param timeRange Number of days to look back (7, 30, or 90) + * @param projectId Optional project ID to filter by + * @param sortBy Field to sort by (followers_change, likes_change, follows_change) + * @param sortOrder Sort order (asc or desc) + * @param limit Number of KOLs to return + * @param offset Offset for pagination + * @returns KOL performance overview data + */ + async getKolPerformanceOverview( + timeRange: number, + projectId?: string, + sortBy: string = 'followers_change', + sortOrder: string = 'desc', + limit: number = 20, + offset: number = 0 + ): Promise { + const startTime = Date.now(); + logger.info('Fetching KOL performance overview', { timeRange, projectId, sortBy, sortOrder, limit, offset }); + + try { + // First let's check if we have any data in the tables + await this.checkDataExistence(); + + // Calculate date for time range + const currentDate = new Date(); + const pastDate = new Date(); + pastDate.setDate(currentDate.getDate() - timeRange); + + // Format dates for SQL + const currentDateStr = this.formatDateForClickhouse(currentDate); + const pastDateStr = this.formatDateForClickhouse(pastDate); + + // Build project filter condition + const projectFilter = projectId ? `AND project_id = '${projectId}'` : ''; + + // First get the influencers + const influencersQuery = ` + SELECT + influencer_id, + name, + platform, + profile_url, + followers_count + FROM + influencers + WHERE + 1=1 ${projectFilter} + ORDER BY + followers_count DESC + LIMIT ${limit} + OFFSET ${offset} + `; + + // Query to get total count for pagination + const countQuery = ` + SELECT + COUNT(*) as total + FROM + influencers + WHERE + 1=1 ${projectFilter} + `; + + logger.debug('Executing ClickHouse queries for influencers', { + influencersQuery: influencersQuery.replace(/\n\s+/g, ' ').trim(), + countQuery: countQuery.replace(/\n\s+/g, ' ').trim() + }); + + // Execute the influencers and count queries + const [influencersData, countResult] = await Promise.all([ + this.executeClickhouseQuery(influencersQuery), + this.executeClickhouseQuery(countQuery) + ]); + + // If we have no influencers, return empty result + if (!Array.isArray(influencersData) || influencersData.length === 0) { + logger.info('No influencers found for the given criteria'); + return { + kols: [], + total: 0 + }; + } + + // Get the list of influencer IDs to fetch events for + const influencerIds = influencersData.map(inf => inf.influencer_id); + const influencerIdsList = influencerIds.map(id => `'${id}'`).join(', '); + + // Now fetch the event metrics for these influencers + const eventsQuery = ` + SELECT + influencer_id, + + -- Current period metrics + SUM(CASE WHEN event_type = 'follow' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) - + SUM(CASE WHEN event_type = 'unfollow' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) AS followers_change, + + SUM(CASE WHEN event_type = 'like' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) - + SUM(CASE WHEN event_type = 'unlike' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) AS likes_change, + + SUM(CASE WHEN event_type = 'follow' AND date BETWEEN '${pastDateStr}' AND '${currentDateStr}' THEN 1 ELSE 0 END) AS follows_change + FROM + events + WHERE + influencer_id IN (${influencerIdsList}) + GROUP BY + influencer_id + `; + + logger.debug('Executing ClickHouse query for events', { + eventsQuery: eventsQuery.replace(/\n\s+/g, ' ').trim() + }); + + // Execute the events query + const eventsData = await this.executeClickhouseQuery(eventsQuery); + + // Create a map of influencer_id to events data for fast lookup + const eventsMap: Record = {}; + if (Array.isArray(eventsData)) { + eventsData.forEach(event => { + if (event.influencer_id) { + eventsMap[event.influencer_id] = event; + } + }); + } + + // Combine the influencer data with events data + const kols = influencersData.map(influencer => { + const events = eventsMap[influencer.influencer_id] || {}; + + return { + influencer_id: String(influencer.influencer_id || ''), + name: String(influencer.name || ''), + platform: String(influencer.platform || ''), + profile_url: String(influencer.profile_url || ''), + followers_count: Number(influencer.followers_count || 0), + followers_change: Number(events.followers_change || 0), + followers_change_percentage: null, // We'll calculate this in a separate query if needed + likes_change: Number(events.likes_change || 0), + likes_change_percentage: null, + follows_change: Number(events.follows_change || 0), + follows_change_percentage: null + }; + }); + + // Sort the results based on the requested sort field and order + kols.sort((a: any, b: any) => { + const aValue = a[sortBy] || 0; + const bValue = b[sortBy] || 0; + + return sortOrder.toLowerCase() === 'asc' + ? aValue - bValue + : bValue - aValue; + }); + + const total = this.parseCountResult(countResult); + + const duration = Date.now() - startTime; + logger.info('KOL performance overview fetched successfully', { + duration, + kolCount: kols.length, + total + }); + + return { + kols, + total + }; + } catch (error) { + const duration = Date.now() - startTime; + logger.error(`Error in getKolPerformanceOverview (${duration}ms)`, error); + throw error; + } + } + + /** + * Check if tables contain data and log the results for diagnostics + */ + private async checkDataExistence(): Promise { + try { + // Check if the influencers table has data + const influencerCountQuery = "SELECT COUNT(*) as count FROM influencers"; + const eventsCountQuery = "SELECT COUNT(*) as count FROM events"; + + const [influencersResult, eventsResult] = await Promise.all([ + this.executeClickhouseQuery(influencerCountQuery), + this.executeClickhouseQuery(eventsCountQuery) + ]); + + const influencersCount = Array.isArray(influencersResult) && influencersResult.length > 0 + ? influencersResult[0].count + : 0; + + const eventsCount = Array.isArray(eventsResult) && eventsResult.length > 0 + ? eventsResult[0].count + : 0; + + logger.info('Data existence check', { + influencers_count: influencersCount, + events_count: eventsCount + }); + + // Optional: Check for sample data + if (influencersCount > 0) { + const sampleQuery = "SELECT influencer_id, name, platform, profile_url, followers_count FROM influencers LIMIT 2"; + const sampleResult = await this.executeClickhouseQuery(sampleQuery); + logger.info('Sample influencer data', { sample: sampleResult }); + } + + if (eventsCount > 0) { + const sampleQuery = ` + SELECT + event_type, + COUNT(*) as count + FROM events + GROUP BY event_type + LIMIT 5 + `; + const sampleResult = await this.executeClickhouseQuery(sampleQuery); + logger.info('Sample events data', { sample: sampleResult }); + } + } catch (error) { + logger.warn('Error checking data existence', error); + // Don't throw, just continue with the query + } + } + + /** + * Execute a ClickHouse query with proper error handling + * @param query SQL query to execute + * @returns Query result + */ + private async executeClickhouseQuery(query: string): Promise { + try { + const result = await clickhouse.query({ + query, + format: 'JSONEachRow' + }); + + // Handle different result formats + if ('json' in result) { + return await result.json(); + } else if ('rows' in result) { + return result.rows; + } + + return []; + } catch (error) { + logger.error('Error executing ClickHouse query', error); + // Re-throw to be handled by the caller + throw new Error(`ClickHouse query error: ${error instanceof Error ? error.message : String(error)}`); + } + } + + /** + * Parse count result from ClickHouse + * @param data ClickHouse count query result + * @returns Total count + */ + private parseCountResult(data: any[]): number { + if (!Array.isArray(data) || data.length === 0) { + logger.warn('ClickHouse count result is invalid, returning 0'); + return 0; + } + + return Number(data[0]?.total || 0); + } + + /** + * Format date for ClickHouse query + * @param date Date to format + * @returns Formatted date string (YYYY-MM-DD) + */ + private formatDateForClickhouse(date: Date): string { + return date.toISOString().split('T')[0]; + } + + /** + * Debug event data - only called if debug=true parameter is set + */ + async debugEventData(): Promise { + try { + // Check events table + const eventCountQuery = "SELECT COUNT(*) as count FROM events"; + const eventTypesQuery = "SELECT event_type, COUNT(*) as count FROM events GROUP BY event_type"; + const sampleEventsQuery = "SELECT * FROM events LIMIT 5"; + const sampleInfluencersEventsQuery = ` + SELECT e.event_type, e.influencer_id, e.date, i.name + FROM events e + JOIN influencers i ON e.influencer_id = i.influencer_id + LIMIT 10 + `; + + const [eventCount, eventTypes, sampleEvents, joinedEvents] = await Promise.all([ + this.executeClickhouseQuery(eventCountQuery), + this.executeClickhouseQuery(eventTypesQuery), + this.executeClickhouseQuery(sampleEventsQuery), + this.executeClickhouseQuery(sampleInfluencersEventsQuery).catch(() => []) + ]); + + logger.info('DEBUG: Event counts', { + total_events: eventCount[0]?.count || 0, + event_types: eventTypes, + }); + + logger.info('DEBUG: Sample events', { + sample_events: sampleEvents, + joined_events: joinedEvents + }); + + // Try executing our specific events query directly + const eventsQuery = ` + SELECT + influencer_id, + SUM(CASE WHEN event_type = 'follow' THEN 1 ELSE 0 END) AS follows, + SUM(CASE WHEN event_type = 'unfollow' THEN 1 ELSE 0 END) AS unfollows, + SUM(CASE WHEN event_type = 'like' THEN 1 ELSE 0 END) AS likes, + SUM(CASE WHEN event_type = 'unlike' THEN 1 ELSE 0 END) AS unlikes + FROM events + GROUP BY influencer_id + LIMIT 10 + `; + + const eventsResult = await this.executeClickhouseQuery(eventsQuery).catch(err => { + logger.error('DEBUG: Error executing events query', err); + return []; + }); + + logger.info('DEBUG: Aggregated events', { + events_result: eventsResult + }); + + } catch (error) { + logger.warn('Error in debugEventData', error); + } + } +} + +// Export singleton instance +export const analyticsService = new AnalyticsService(); \ No newline at end of file diff --git a/backend/src/swagger/index.ts b/backend/src/swagger/index.ts index 27c89e1..de20b1f 100644 --- a/backend/src/swagger/index.ts +++ b/backend/src/swagger/index.ts @@ -2088,6 +2088,157 @@ export const openAPISpec = { } } }, + '/api/analytics/kol-overview': { + get: { + summary: 'Get KOL performance overview', + description: 'Returns card-style layout showing key performance metrics for each KOL including followers growth, new likes, and new follows', + tags: ['Analytics'], + parameters: [ + { + name: 'timeRange', + in: 'query', + description: 'Number of days to look back', + schema: { + type: 'string', + enum: ['7', '30', '90'], + default: '30' + } + }, + { + name: 'projectId', + in: 'query', + description: 'Filter by project ID', + schema: { + type: 'string' + } + }, + { + name: 'sortBy', + in: 'query', + description: 'Field to sort by', + schema: { + type: 'string', + enum: ['followers_change', 'likes_change', 'follows_change'], + default: 'followers_change' + } + }, + { + name: 'sortOrder', + in: 'query', + description: 'Sort order', + schema: { + type: 'string', + enum: ['asc', 'desc'], + default: 'desc' + } + }, + { + name: 'limit', + in: 'query', + description: 'Number of KOLs to return', + schema: { + type: 'integer', + default: 20 + } + }, + { + name: 'offset', + in: 'query', + description: 'Offset for pagination', + schema: { + type: 'integer', + default: 0 + } + } + ], + responses: { + '200': { + description: 'Successful response with KOL performance data', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + success: { + type: 'boolean', + example: true + }, + data: { + type: 'array', + items: { + type: 'object', + properties: { + influencer_id: { type: 'string', example: 'inf-123-456' }, + name: { type: 'string', example: 'John Influencer' }, + platform: { type: 'string', example: 'instagram' }, + profile_url: { type: 'string', example: 'https://instagram.com/johninfluencer' }, + followers_count: { type: 'integer', example: 50000 }, + followers_change: { type: 'integer', example: 1500 }, + followers_change_percentage: { + type: 'number', + nullable: true, + example: 12.5 + }, + likes_change: { type: 'integer', example: 2800 }, + likes_change_percentage: { + type: 'number', + nullable: true, + example: 15.3 + }, + follows_change: { type: 'integer', example: 1200 }, + follows_change_percentage: { + type: 'number', + nullable: true, + example: 8.7 + } + } + } + }, + pagination: { + type: 'object', + properties: { + limit: { type: 'integer', example: 20 }, + offset: { type: 'integer', example: 0 }, + total: { type: 'integer', example: 42 } + } + } + } + } + } + } + }, + '400': { + description: 'Bad request - invalid parameters', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + success: { type: 'boolean', example: false }, + error: { type: 'string', example: 'Invalid sortBy. Must be one of: followers_change, likes_change, follows_change' } + } + } + } + } + }, + '500': { + description: 'Server error', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + success: { type: 'boolean', example: false }, + error: { type: 'string', example: 'Failed to fetch KOL overview data' }, + message: { type: 'string', example: 'ClickHouse query error: Connection refused' } + } + } + } + } + } + } + } + }, }, components: { schemas: { diff --git a/backend/src/utils/logger.ts b/backend/src/utils/logger.ts new file mode 100644 index 0000000..34c7f50 --- /dev/null +++ b/backend/src/utils/logger.ts @@ -0,0 +1,129 @@ +import fs from 'fs'; +import path from 'path'; +import config from '../config'; + +/** + * Logger levels + */ +export enum LogLevel { + DEBUG = 'DEBUG', + INFO = 'INFO', + WARN = 'WARN', + ERROR = 'ERROR' +} + +/** + * Utility class for logging messages to console and file + */ +class Logger { + private logDir: string; + private logFile: string; + private debugEnabled: boolean; + + constructor() { + // Set up log directory + this.logDir = path.join(process.cwd(), 'logs'); + + // Create logs directory if it doesn't exist + if (!fs.existsSync(this.logDir)) { + fs.mkdirSync(this.logDir, { recursive: true }); + } + + // Set log file path with date in filename + const now = new Date(); + const dateStr = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; + this.logFile = path.join(this.logDir, `app-${dateStr}.log`); + + // Enable debug logs based on environment (NODE_ENV) + this.debugEnabled = process.env.NODE_ENV !== 'production'; + } + + /** + * Format log message with timestamp and level + */ + private formatMessage(level: LogLevel, message: string, data?: any): string { + const timestamp = new Date().toISOString(); + let formattedMessage = `[${timestamp}] [${level}] ${message}`; + + if (data) { + if (typeof data === 'object') { + try { + const dataStr = JSON.stringify(data); + formattedMessage += ` - ${dataStr}`; + } catch (error) { + formattedMessage += ` - [Object cannot be stringified]`; + } + } else { + formattedMessage += ` - ${data}`; + } + } + + return formattedMessage; + } + + /** + * Write log to file + */ + private writeToFile(message: string): void { + fs.appendFileSync(this.logFile, message + '\n'); + } + + /** + * Debug level log + */ + debug(message: string, data?: any): void { + if (!this.debugEnabled) return; + + const formattedMessage = this.formatMessage(LogLevel.DEBUG, message, data); + console.debug(formattedMessage); + this.writeToFile(formattedMessage); + } + + /** + * Info level log + */ + info(message: string, data?: any): void { + const formattedMessage = this.formatMessage(LogLevel.INFO, message, data); + console.info(formattedMessage); + this.writeToFile(formattedMessage); + } + + /** + * Warning level log + */ + warn(message: string, data?: any): void { + const formattedMessage = this.formatMessage(LogLevel.WARN, message, data); + console.warn(formattedMessage); + this.writeToFile(formattedMessage); + } + + /** + * Error level log + */ + error(message: string, error?: any): void { + let errorData = error; + + // Handle Error objects + if (error instanceof Error) { + errorData = { + errorMessage: error.message, + stack: error.stack, + name: error.name + }; + } + + const formattedMessage = this.formatMessage(LogLevel.ERROR, message, errorData); + console.error(formattedMessage); + this.writeToFile(formattedMessage); + } + + /** + * Get log file path + */ + getLogFile(): string { + return this.logFile; + } +} + +// Export singleton instance +export const logger = new Logger(); \ No newline at end of file