auto refresh

windmill/sync_mongo_to_events.ts

@@ -39,13 +39,17 @@ interface SyncState {
   last_sync_id?: string;
 }
 
+// 同步状态键名
+const SYNC_STATE_KEY = "f/shorturl_analytics/mongo_sync_state";
+
 export async function main(
   batch_size = 1000,
   max_records = 9999999,
   timeout_minutes = 60,
   skip_clickhouse_check = false,
   force_insert = false,
-  database_override = "shorturl_analytics" // 添加数据库名称参数，默认为shorturl_analytics
+  database_override = "shorturl_analytics", // 添加数据库名称参数，默认为shorturl_analytics
+  reset_sync_state = false // 添加参数用于重置同步状态
 ) {
   const logWithTimestamp = (message: string) => {
     const now = new Date();
@@ -60,6 +64,9 @@ export async function main(
   if (force_insert) {
     logWithTimestamp("⚠️ 警告: 已启用强制插入模式，将尝试插入所有记录");
   }
+  if (reset_sync_state) {
+    logWithTimestamp("⚠️ 警告: 已启用重置同步状态，将从头开始同步数据");
+  }
 
   // 设置超时
   const startTime = Date.now();
@@ -127,6 +134,36 @@ export async function main(
     throw error;
   }
 
+  // 获取上次同步状态
+  let lastSyncState: SyncState | null = null;
+  if (!reset_sync_state) {
+    try {
+      const rawSyncState = await getVariable(SYNC_STATE_KEY);
+      if (rawSyncState) {
+        if (typeof rawSyncState === "string") {
+          try {
+            lastSyncState = JSON.parse(rawSyncState);
+          } catch (e) {
+            logWithTimestamp(`解析上次同步状态失败: ${e}, 将从头开始同步`);
+          }
+        } else {
+          lastSyncState = rawSyncState as SyncState;
+        }
+      }
+    } catch (error) {
+      logWithTimestamp(`获取上次同步状态失败: ${error}, 将从头开始同步`);
+    }
+  }
+
+  if (lastSyncState) {
+    logWithTimestamp(`找到上次同步状态: 最后同步时间 ${new Date(lastSyncState.last_sync_time).toISOString()}, 已同步记录数 ${lastSyncState.records_synced}`);
+    if (lastSyncState.last_sync_id) {
+      logWithTimestamp(`最后同步ID: ${lastSyncState.last_sync_id}`);
+    }
+  } else {
+    logWithTimestamp("没有找到上次同步状态，将从头开始同步");
+  }
+
   // 构建MongoDB连接URL
   let mongoUrl = "mongodb://";
   if (mongoConfig.username && mongoConfig.password) {
@@ -145,25 +182,32 @@ export async function main(
     const db = client.database(mongoConfig.db);
     const traceCollection = db.collection<TraceRecord>("trace");
     
-    // 构建查询条件，获取所有记录
+    // 构建查询条件，根据上次同步状态获取新记录
     const query: Record<string, unknown> = {
       type: 1  // 只同步type为1的记录
     };
 
+    // 如果有上次同步状态，则只获取更新的记录
+    if (lastSyncState && lastSyncState.last_sync_time) {
+      // 使用上次同步时间作为过滤条件
+      query.createTime = { $gt: lastSyncState.last_sync_time };
+      logWithTimestamp(`将只同步createTime > ${lastSyncState.last_sync_time} (${new Date(lastSyncState.last_sync_time).toISOString()}) 的记录`);
+    }
+
     // 计算总记录数
     const totalRecords = await traceCollection.countDocuments(query);
-    console.log(`找到 ${totalRecords} 条记录需要同步`);
+    console.log(`找到 ${totalRecords} 条新记录需要同步`);
 
     // 限制此次处理的记录数量
     const recordsToProcess = Math.min(totalRecords, max_records);
     console.log(`本次将处理 ${recordsToProcess} 条记录`);
 
     if (totalRecords === 0) {
-      console.log("没有记录需要同步，任务完成");
+      console.log("没有新记录需要同步，任务完成");
       return {
         success: true,
         records_synced: 0,
-        message: "没有记录需要同步"
+        message: "没有新记录需要同步"
       };
     }
 
@@ -464,6 +508,8 @@ export async function main(
     // 批量处理记录
     let processedRecords = 0;
     let totalBatchRecords = 0;
+    let lastSyncTime = 0;
+    let lastSyncId = "";
 
     for (let page = 0; processedRecords < recordsToProcess; page++) {
       // 检查超时
@@ -505,13 +551,40 @@ export async function main(
       processedRecords += records.length;
       totalBatchRecords += batchSize;
 
+      // 更新最后处理的记录时间和ID
+      if (records.length > 0) {
+        const lastRecord = records[records.length - 1];
+        lastSyncTime = Math.max(lastSyncTime, lastRecord.createTime);
+        lastSyncId = lastRecord._id.toString();
+      }
+
       logWithTimestamp(`第 ${page+1} 批次处理完成。已处理 ${processedRecords}/${recordsToProcess} 条记录，实际插入 ${totalBatchRecords} 条 (${Math.round(processedRecords/recordsToProcess*100)}%)`);
     }
 
+    // 更新同步状态
+    if (processedRecords > 0 && lastSyncTime > 0) {
+      // 创建新的同步状态
+      const newSyncState: SyncState = {
+        last_sync_time: lastSyncTime,
+        records_synced: (lastSyncState ? lastSyncState.records_synced : 0) + totalBatchRecords,
+        last_sync_id: lastSyncId
+      };
+      
+      try {
+        // 保存同步状态
+        await setVariable(SYNC_STATE_KEY, newSyncState);
+        logWithTimestamp(`同步状态已更新: 最后同步时间 ${new Date(newSyncState.last_sync_time).toISOString()}, 累计同步记录数 ${newSyncState.records_synced}`);
+      } catch (err) {
+        const error = err as Error;
+        logWithTimestamp(`更新同步状态失败: ${error.message}`);
+      }
+    }
+
     return {
       success: true,
       records_processed: processedRecords,
       records_synced: totalBatchRecords,
+      last_sync_time: lastSyncTime > 0 ? new Date(lastSyncTime).toISOString() : null,
       message: "数据同步完成"
     };
   } catch (err) {