update

HTEX.DataETL/Controllers/LessonRecordController.cs

@@ -4,8 +4,11 @@ using HTEX.Lib.ETL.Lesson;
 using MathNet.Numerics.Distributions;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Azure.Cosmos;
+using Microsoft.Extensions.Logging;
 using StackExchange.Redis;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Reflection;
 using System.Text.Json;
 using System.Text.RegularExpressions;
@@ -229,7 +232,7 @@ namespace HTEX.DataETL.Controllers
                 etime = 1731513599000;//2024-11-13 23:59:59
             }
             var resultSchools = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
-                 .GetList<LessonRecord>($"SELECT value c FROM   c  where  ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and c.startTime>={stime} and c.startTime<={etime} and   c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ");
+                 .GetList<LessonRecord>($"SELECT value c FROM   c  where  ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and c.startTime>{stime} and c.startTime<{etime} and   c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ");
             List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
             int index = 0;
             if (resultSchools.list.IsNotEmpty()) 
@@ -248,6 +251,8 @@ namespace HTEX.DataETL.Controllers
                 {
                     var item = await LessonETLService.GetLessonLocal(lessonRecord, localIds, _azureStorage, pathLessons);
                     lessonLocals.Add(item);
+                    string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
+                    await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
                     index++;
                 }
               
@@ -343,17 +348,26 @@ namespace HTEX.DataETL.Controllers
                     await System.IO.File.WriteAllTextAsync(overall_path, schoolOverallEducations.ToJsonString());
                 }
                 
-                List<(string id,string owner, List<StudentLessonData> studentLessonData)> studentLessonDatas= new List<(string id, string owner, List<StudentLessonData>)>();
-                List<(string id, string owner, List<StudentLessonItem> studentLessons,List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)> lessonItems = new List<(string id, string owner, List<StudentLessonItem> studentLessons, List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)>();
+                //List<(string id,string owner, List<StudentLessonData> studentLessonData)> studentLessonDatas= new List<(string id, string owner, List<StudentLessonData>)>();
+                ConcurrentQueue<(string id, string owner, List<StudentLessonItem> studentLessons,List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)> lessonItems = new ConcurrentQueue<(string id, string owner, List<StudentLessonItem> studentLessons, List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)>();
                 int n = 0;
-                Parallel.ForEach(lessonLocals,   (item, con) => 
+              
+                Parallel.ForEach(lessonLocals, (item, con) =>
                 {
-                    var studata = LessonETLService.DoStudentLessonDataV2(Constant.objectiveTypes, item, "China", studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
-                    if (studata.codeBools.FindAll(x => x.value).IsNotEmpty())
+                    try
                     {
-                        string owner = item.lessonRecord.scope.Equals("school") ? item.lessonRecord.school : item.lessonRecord.tmdid;
-                        studentLessonDatas.Add((item.lessonRecord.id, owner, studata.studentLessonDatas));
-                        lessonItems.Add((item.lessonRecord.id, owner, studata.lessonItems, studata.codeBools, studata.studentLessonDatas));
+                        var studata = LessonETLService.DoStudentLessonDataV2(Constant.objectiveTypes, item, "China", studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
+                        if (studata.codeBools.FindAll(x => x.value).IsNotEmpty())
+                        {
+                            string owner = item.lessonRecord.scope.Equals("school") ? item.lessonRecord.school : item.lessonRecord.tmdid;
+                            //studentLessonDatas.Add((item.lessonRecord.id, owner, studata.studentLessonDatas));
+                            lessonItems.Append((item.lessonRecord.id, owner, studata.lessonItems, studata.codeBools, studata.studentLessonDatas));
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        Console.WriteLine($"{ex.Message},{ex.StackTrace}");
+                        throw new Exception(ex.Message, ex);
                     }
                     n++;
                 });
@@ -377,7 +391,7 @@ namespace HTEX.DataETL.Controllers
                 var grpdata = studentSemesterRecords.GroupBy(x => x.code).Select(x => new { key = x.Key, list = x.ToList() });
                 foreach (var group in grpdata)
                 {
-                    var pages = group.list.Page(66);
+                    var pages = group.list.Page(200);
                    
                     foreach (var page in pages)
                     {
@@ -394,7 +408,7 @@ namespace HTEX.DataETL.Controllers
                 var gpover = overallEducations.GroupBy(x => x.code).Select(x => new { key = x.Key, list = x.ToList() });
                 foreach (var item in gpover)
                 {
-                    var pages = item.list.Page(66);
+                    var pages = item.list.Page(200);
                     foreach (var page in pages)
                     {
                         List<Task<ItemResponse<OverallEducation>>> list = new();
@@ -439,7 +453,7 @@ namespace HTEX.DataETL.Controllers
         [HttpPost("process-history")]
         public async Task<IActionResult> ProcessHistory(JsonElement json)
         {
-
+            Stopwatch stopwatch= new Stopwatch();
             List<string> localIds = new List<string>();
             string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
             string? pathLessons = $"{lessonBasePath}\\lessons";
@@ -454,114 +468,74 @@ namespace HTEX.DataETL.Controllers
                 }
             }
             bool loadLocal = true;
-            List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
             var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
             long stime = 1693497600000;//2023-09-01 00:00:00
-            foreach (var file in filesAnalysis)
-            {
-                //读取每月的数据
-                if (file.EndsWith("-m-analysis.json"))
-                {
-                    string jsons = await System.IO.File.ReadAllTextAsync(file);
-                    LessonDataAnalysisMonth lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisMonth>();
-                    lessonDataAnalysisMonths.Add(lessonDataAnalysis);
-                }
-            }
-            if (lessonDataAnalysisMonths.IsNotEmpty())
-            {
-                var maxUpdateTime = lessonDataAnalysisMonths.Max(x => x.updateTime);
-                if (maxUpdateTime>0)
-                {
-                    //更新周期是一周
-                    if (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()- maxUpdateTime>604800000)
-                    {
-                        stime=maxUpdateTime;
-                        loadLocal =true;
-                    }
-                    else
-                    {
-                        stime=maxUpdateTime;
-                        loadLocal=false;
-                    }
-                }
-            }
-            HashSet<string> yearMonth = new HashSet<string>();
-            long newest = 0;
             bool force = false;
             if ((json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
             {
                 force= _force.GetBoolean();
             }
-            // if (loadLocal ||force)
+            if (force) 
             {
-                List<LessonRecord> lessonRecords = new List<LessonRecord>();
-                var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
-                    .GetList<LessonRecord>($"SELECT value c FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime} and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ", null);
-                if (resultSchool.list.IsNotEmpty())
+                List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
+                foreach (var file in filesAnalysis)
                 {
-                    newest= resultSchool.list.Max(x => x.startTime);
-                    lessonRecords.AddRange(resultSchool.list);
-                }
-                else
-                {
-                    newest=stime;
-                }
-                var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
-                   .GetList<LessonRecord>($"SELECT value c FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime} and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord'   ", null);
-                if (resultTeacher.list.IsNotEmpty())
-                {
-                    long max = resultTeacher.list.Max(x => x.startTime);
-                    if (max<newest)
+                    //读取每月的数据
+                    if (file.EndsWith("-m-analysis.json"))
                     {
-                        newest=max;
+                        string jsons = await System.IO.File.ReadAllTextAsync(file);
+                        LessonDataAnalysisMonth lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisMonth>();
+                        lessonDataAnalysisMonths.Add(lessonDataAnalysis);
                     }
-                    lessonRecords.AddRange(resultTeacher.list);
                 }
-                List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
-                if (lessonRecords.IsNotEmpty())
+                if (lessonDataAnalysisMonths.IsNotEmpty())
                 {
-                    await Parallel.ForEachAsync(lessonRecords, async(record, _)=>{
-                        LessonLocal  item = await LessonETLService.GetLessonLocal(record, localIds, _azureStorage, pathLessons);
-                        string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
-                        item.lessonRecord.learningCategory= item.lessonBase?.summary?.learningCategory;
-                        if (item.lessonRecord.learningCategory == null)
+                    var maxUpdateTime = lessonDataAnalysisMonths.Max(x => x.updateTime);
+                    if (maxUpdateTime>0)
+                    {
+                        //更新周期是一周
+                        if (DateTimeOffset.Now.ToUnixTimeMilliseconds()- maxUpdateTime>604800000)
                         {
-
-                            item.lessonRecord.learningCategory = new LearningCategory();
+                            stime=maxUpdateTime;
+                            loadLocal =true;
                         }
-                        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
-                    });
-
-
-                   //await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
-                   // {
-                   //     string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
-                   //     item.lessonRecord.learningCategory= item.lessonBase?.summary?.learningCategory;
-                   //     if (item.lessonRecord.learningCategory == null)
-                   //     {
-
-                   //         item. lessonRecord.learningCategory = new LearningCategory();
-                   //     }
-                   //     await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
-                   // }
+                        else
+                        {
+                            stime=maxUpdateTime;
+                            loadLocal=false;
+                        }
+                    }
                 }
+            }
+           
+            // if (loadLocal ||force)
+            {
+                
+                List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
+                await LessonETLService.FixLocalData(localIds, _azureCosmos, _azureStorage, pathLessons, stime, DateTimeOffset.Now.ToUnixTimeMilliseconds());
                 List<TechCount> techCounts = new List<TechCount>();
                 filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
-                //await foreach (var item in LessonETLService.GetTeachCount(_azureCosmos,lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force))
-                //{
-                //    techCounts.Add(item);
-                //}
 
+                stopwatch.Start();
+                List<LessonLocal> lessonLocals = new List<LessonLocal>();
+                Parallel.ForEach(filesLessons, file =>
+                {
+                    string jsonp = System.IO.File.ReadAllText(file);
+                    var lessonLocal = jsonp.ToObject<LessonLocal>();
+                    lessonLocals.Add(lessonLocal);
+                });
+                stopwatch.Stop();
+                _logger.LogInformation($"Loaded {lessonLocals.Count} lessons in {stopwatch.Elapsed.TotalSeconds} seconds");
                 await Parallel.ForEachAsync(filesLessons, async (item, _) => {
                     TechCount techCount=  await  LessonETLService.GetTeachCount(_azureCosmos, item, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force);
                     if (techCount != null) {
                         techCounts.Add(techCount);
                     }
                 });
-
+                long newest = lessonLocals.Max(x=>x.lessonRecord.startTime);
                 await LessonETLService.GenAnalysisData(pathAnalysis, newest, techCounts,_azureStorage);
             }
-            return Ok(new { yearMonth });
+            return Ok(new {   });
         }
 
 

TEAMModelOS.Extension/HTEX.Lib/ETL/Lesson/LessonETLService.cs

@@ -10,6 +10,7 @@ using Microsoft.Azure.Cosmos;
 using Microsoft.Azure.Cosmos.Linq;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Microsoft.OData.UriParser;
 using OfficeOpenXml;
 using OpenXmlPowerTools;
 using System;
@@ -35,6 +36,65 @@ namespace HTEX.Lib.ETL.Lesson
 {
     public class LessonETLService
     {
+
+        public static async Task<List<LessonLocal>> FixLocalData(List<string> localIds, AzureCosmosFactory _azureCosmos, AzureStorageFactory _azureStorage, string pathLessons,long  stime,long etime)
+        {
+            if (etime<=1693497600000) 
+            {
+                etime= DateTimeOffset.Now.ToUnixTimeMilliseconds();
+            }
+            
+            List<string> dbids= new List<string>();
+            var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
+                   .GetList<string>($"SELECT value c.id FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime} and c.startTime<{etime} and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ", null);
+            if (resultSchool.list.IsNotEmpty())
+            {
+                dbids.AddRange(resultSchool.list);
+            }
+            var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
+                   .GetList<string>($"SELECT value c.id FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime} and c.startTime<{etime}  and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ", "LessonRecord");
+            if (resultTeacher.list.IsNotEmpty())
+            {
+                dbids.AddRange(resultTeacher.list);
+            }
+            var newIds=  dbids.Except(localIds);
+            List<LessonRecord> list=new List<LessonRecord>();
+            if (newIds!=null  && newIds.Count()>0) 
+            {
+                var resultSchoolLessons = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
+                  .GetList<LessonRecord>($"SELECT value c.id FROM   c  where  c.id in ({string.Join(",",newIds.Select(x=>$"'{x}'"))}) and    ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime} and c.startTime<{etime}  and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ", null);
+                if (resultSchoolLessons.list.IsNotEmpty())
+                {
+                    list.AddRange(resultSchoolLessons.list);
+                    newIds= newIds.Except(resultSchoolLessons.list.Select(x => x.id));
+                }
+                if (newIds!=null && newIds.Count()>0) 
+                {
+                    var resultTeacherLessons = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
+                  .GetList<LessonRecord>($"SELECT value c.id FROM   c  where  c.id in ({string.Join(",", newIds.Select(x => $"'{x}'"))}) and    ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime}  and c.startTime<{etime}  and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ", "LessonRecord");
+                    if (resultTeacherLessons.list.IsNotEmpty())
+                    {
+                        list.AddRange(resultTeacherLessons.list);
+                    }
+                }
+            }
+            List<LessonLocal> lessonLocals = new List<LessonLocal>();
+            foreach (var record in list) 
+            {
+                var item = await LessonETLService.GetLessonLocal(record, localIds, _azureStorage, pathLessons);
+                string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
+                item.lessonRecord.learningCategory= item.lessonBase?.summary?.learningCategory;
+                if (item.lessonRecord.learningCategory == null)
+                {
+
+                    item.lessonRecord.learningCategory = new LearningCategory();
+                }
+                await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
+                lessonLocals.Add(item);
+            }
+            return lessonLocals;
+        }
+
         public static int GetRandomValueByWeight(List<WeightedItem> items)
         {
             Random random = new Random();
@@ -147,15 +207,12 @@ namespace HTEX.Lib.ETL.Lesson
             string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
             if (codeBools.FindAll(x => x.value).IsNotEmpty())
             {
- 
                 lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis, codeBools);
- 
                 if (lessonLocal.lessonRecord.scope.Equals("school")&& !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.school)  && location.Equals("China", StringComparison.OrdinalIgnoreCase))
                 {
                     School school = schools.Find(x => x.id.Equals(lessonLocal.lessonRecord.school));
                     if (school!=null)
                     {
- 
                         string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : school.period.FirstOrDefault()?.id;
                         var period = school.period.Find(x => x.id.Equals(periodId));
                         if (period!=null)
@@ -287,342 +344,43 @@ namespace HTEX.Lib.ETL.Lesson
                                         };
                                         overallEducations.Add(overallEducation);
                                     }
-                                    var hasrecord = overallEducation.lessonScore.Find(x => x.lessonId.Equals(lessonLocal.lessonRecord.id));
-                                    if (hasrecord==null)
+                                    if (les.attend==1)
                                     {
-                                        hasrecord=   new StudentLessonRecord();
-                                        overallEducation.lessonScore.Add(hasrecord);
-                                    }
-                                    hasrecord.gscore = studentLessonData.gscore;
-                                    hasrecord.pscore = studentLessonData.pscore;
-                                    hasrecord.tscore = studentLessonData.tscore;
-                                    hasrecord.tmdid = lessonLocal.lessonRecord.tmdid;
-                                    hasrecord.school = school.id;
-                                    hasrecord.scope = lessonLocal.lessonRecord.scope;
-                                    hasrecord.lessonId = lessonLocal.lessonRecord.id;
-                                    hasrecord.courseId = lessonLocal.lessonRecord.courseId;
-                                    periodId = period?.id;
-                                    hasrecord.subjectId = lessonLocal.lessonRecord.subjectId;
-                                    hasrecord.time= lessonLocal.lessonRecord.startTime;
-                                    hasrecord.attend=les.attend;
-                                    if (lesson!=null)
-                                    {
-                                        hasrecord.hrate=lesson.hrate;
-                                        hasrecord.crate=lesson.crate;
-                                        hasrecord.trate=lesson.trate;
-                                        hasrecord.xrate=lesson.xrate;
-                                        hasrecord.prate=lesson.prate;
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            return (studentLessonDatas, lessonItems, codeBools);
-        }
-
-        /// <summary>
-        /// 生成学生student-analysis.json
-        /// </summary>
-        /// <param name="objectiveTypes"></param>
-        /// <param name="azureStorage"></param>
-        /// <param name="lessonLocal"></param>
-        /// <returns></returns>
-        public static async Task<(List<StudentLessonData> studentLessonDatas, List<StudentLessonItem> lessonItems, List<CodeBool> codeBools)>
-            DoStudentLessonData(List<string> objectiveTypes, AzureStorageFactory azureStorage, LessonLocal lessonLocal,DingDing _dingDing,
-            CosmosClient client,string location ,AzureRedisFactory azureRedis, List<StudentSemesterRecord> studentSemesterRecords, List<OverallEducation> overallEducations,
-            LessonDataAnalysisModel lessonDataAnalysis, List<Student> studentsBase,List<School> schools )
-        {
-            List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
-            studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
-            studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal?.lessonRecord?.id);
-            studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-            studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
-            studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-            studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-            var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal. timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
-            studentLessonDatas= pickupData.studentLessonDatas;
-            var codeBools=  GetCodeBools(studentLessonDatas);
-            List<StudentLessonItem> lessonItems = new List<StudentLessonItem>();
-            string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
-            if (codeBools.FindAll(x => x.value).IsNotEmpty()) 
-            {
-#if !DEBUG
-               try
-                {
-
-                    bool exists = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis-model.json").ExistsAsync();
-                    if (exists)
-                    {
-                        if (lessonDataAnalysis==null)
-                        {
-                            BlobDownloadResult blobDownload = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis-model.json").DownloadContentAsync();
-                            lessonDataAnalysis = blobDownload.Content.ToObjectFromJson<LessonDataAnalysisModel>();
-                        }
-
-                        
-                        XmlDocument xmlDocument = new XmlDocument();
-                        var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
-                        xmlDocument.Load($"{runtimePath}\\summary.xml");
-                        // 获取类的属性
-                        PropertyInfo[] properties = typeof(StudentLessonItem).GetProperties();
-                        List<string> summaryes= new List<string>();
-                        for (int i = 0; i < properties.Length; i++)
-                        {
-                            string summary = Regex.Replace(LessonETLService.GetPropertySummary(properties[i], xmlDocument), @"\s+", "");
-                            summaryes.Add(summary);
-                        }
-                        await LessonETLService.ExportToExcelAzureBlob(lessonItems, azureStorage, owner, $"{lessonLocal.lessonRecord.id}/student-analysis.xlsx", xmlDocument,summaryes,properties);
-                    }
-                }
-                catch (Exception ex)
-                {
-                    if (!ex.Message.Contains("The specified blob does not exist"))
-                    {
-                        await _dingDing.SendBotMsg($"{Environment.GetEnvironmentVariable("Option:Location")},lesson/analysis/analysis-model.json转换异常，{ex.Message}{ex.StackTrace}", GroupNames.成都开发測試群組);
-                    }
-                }
-                await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
-#else
-                lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis, codeBools);
-#endif
-                if (lessonLocal.lessonRecord.scope.Equals("school")&& !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.school)  && location.Equals("China", StringComparison.OrdinalIgnoreCase))
-                {
-                    School school = schools.Find(x => x.id.Equals(lessonLocal.lessonRecord.school));
-                    if (school==null)
-                    {
-                        ResponseMessage response = await client.GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemStreamAsync(lessonLocal.lessonRecord.school, new PartitionKey("Base"));
-                        if (response.IsSuccessStatusCode)
-                        {
-                            school = JsonDocument.Parse(response.Content).RootElement.ToObject<School>();
-                            schools.Add(school);
-                        }
-                    }
-                    if (school!=null)
-                    {
-#if !DEBUG
-                        var sdtus = studentLessonDatas.Where(y => !studentsBase.Where(z => z.schoolId.Equals(school.id)).Select(x => x.id).Contains(y.id));
-                        if (sdtus!=null && sdtus.Count()>0)
-                        {
-                            var result = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<Student>($"select value c from c where c.id in ({string.Join(",", sdtus.Select(d => $"'{d.id}'"))})", $"Base-{school.id}");
-                            if (result.list.Any())
-                            {
-                                studentsBase.AddRange(result.list);
-                            }
-                        }
-#endif
-                        string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : school.period.FirstOrDefault()?.id;
-                        var period = school.period.Find(x => x.id.Equals(periodId));
-                        if (period!=null)
-                        {
-                            var semester = SchoolService.GetSemester(period, lessonLocal.lessonRecord.startTime);
-                            string pre_id = $"{semester.studyYear}-{semester.currSemester.id}";
-                            string code = $"StudentSemesterRecord-{school.id}";
-                            foreach (var studentLessonData in studentLessonDatas)
-                            {
-                                StudentSemesterRecord studentSemester = studentSemesterRecords.Find(x => x.id.Equals($"{pre_id}-{studentLessonData.id}") && x.code.Equals(code));
-                                if (studentSemester==null)
-                                {
-#if !DEBUG
-                                    ResponseMessage responseMessage = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).ReadItemStreamAsync($"{pre_id}-{studentLessonData.id}", new PartitionKey(code));
-                                    if (responseMessage.IsSuccessStatusCode)
-                                    {
-                                        studentSemester= JsonDocument.Parse(responseMessage.Content).ToObject<StudentSemesterRecord>();
-                                    }
-                                    else
-                                    {
-                                        studentSemester= new StudentSemesterRecord
+                                        var hasrecord = overallEducation.lessonScore.Find(x => x.lessonId.Equals(lessonLocal.lessonRecord.id));
+                                        if (hasrecord==null)
                                         {
-                                            id= $"{pre_id}-{studentLessonData.id}",
-                                            code=code,
-                                            stuid= studentLessonData.id,
-                                            userType=Constant.ScopeStudent,
-                                            school=school.id,
-                                            studyYear=semester.studyYear,
-                                            semesterId=semester.currSemester.id,
-                                            period= period?.id,
-                                            pk="StudentSemesterRecord"
-                                        };
-                                    }
-#else
-                                    studentSemester= new StudentSemesterRecord
-                                    {
-                                        id= $"{pre_id}-{studentLessonData.id}",
-                                        code=code,
-                                        stuid= studentLessonData.id,
-                                        userType=Constant.ScopeStudent,
-                                        school=school.id,
-                                        studyYear=semester.studyYear,
-                                        semesterId=semester.currSemester.id,
-                                        period= period?.id,
-                                        pk="StudentSemesterRecord"
-                                    };
-#endif
-                                    studentSemesterRecords.Add(studentSemester);
-                                }
-
-                                var les = studentSemester.les.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
-                                if (les==null)
-                                {
-                                    les = new StuLesson()
-                                    {
-                                        id= lessonLocal.lessonRecord.id,
-                                        time= lessonLocal.lessonRecord.startTime,
-                                        attend=0
-                                    };
-                                    studentSemester.les.Add(les);
-                                }
-                                else
-                                {
-                                    les.time= lessonLocal.lessonRecord.startTime;
-                                }
-                                var lesson = studentSemester.lessons.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
-                                if (lesson!=null)
-                                {
-                                    if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0)
-                                    {
-                                        les.attend=1;
-                                        lesson.tmdid = lessonLocal.lessonRecord.tmdid;
-                                        lesson.sid = lessonLocal.lessonRecord.subjectId;
-                                        lesson.cid = lessonLocal.lessonRecord.courseId;
-                                        lesson.hrate = studentLessonData.cooperation;
-                                        lesson.crate = studentLessonData.achieve;
-                                        lesson.trate = studentLessonData.attitude;
-                                        lesson.xrate = studentLessonData.cowork;
-                                        lesson.prate = studentLessonData.appraise;
-                                    }
-                                    else
-                                    {
-                                        if (studentLessonData.attend==1)
-                                        {
-                                            les.attend=1;
-                                        }
-                                        else
-                                        {
-                                            les.attend=0;
-                                        }
-                                    }
-                                }
-                                else
-                                {
-                                    if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0)
-                                    {
-                                        les.attend=1;
-                                        lesson = new StuLessonLite
-                                        {
-                                            id=lessonLocal.lessonRecord.id,
-                                            tmdid=lessonLocal.lessonRecord.tmdid,
-                                            sid= lessonLocal.lessonRecord.subjectId,
-                                            cid= lessonLocal.lessonRecord.courseId,
-                                            hrate=studentLessonData.cooperation,
-                                            crate=studentLessonData.achieve,
-                                            trate=studentLessonData.attitude,
-                                            xrate=studentLessonData.cowork,
-                                            prate=studentLessonData.appraise
-                                        };
-                                        studentSemester.lessons.Add(lesson);
-                                    }
-                                    else
-                                    {
-                                        if (studentLessonData.attend==1)
-                                        {
-                                            les.attend=1;
+                                            hasrecord=   new StudentLessonRecord();
+                                            overallEducation.lessonScore.Add(hasrecord);
                                         }
-                                        else
+                                        hasrecord.gscore = studentLessonData.gscore;
+                                        hasrecord.pscore = studentLessonData.pscore;
+                                        hasrecord.tscore = studentLessonData.tscore;
+                                        hasrecord.tmdid = lessonLocal.lessonRecord.tmdid;
+                                        hasrecord.school = school.id;
+                                        hasrecord.scope = lessonLocal.lessonRecord.scope;
+                                        hasrecord.lessonId = lessonLocal.lessonRecord.id;
+                                        hasrecord.courseId = lessonLocal.lessonRecord.courseId;
+                                        periodId = period?.id;
+                                        hasrecord.subjectId = lessonLocal.lessonRecord.subjectId;
+                                        hasrecord.time= lessonLocal.lessonRecord.startTime;
+                                        hasrecord.attend=les.attend;
+                                        if (lesson!=null)
                                         {
-                                            les.attend=0;
+                                            hasrecord.hrate=lesson.hrate;
+                                            hasrecord.crate=lesson.crate;
+                                            hasrecord.trate=lesson.trate;
+                                            hasrecord.xrate=lesson.xrate;
+                                            hasrecord.prate=lesson.prate;
                                         }
                                     }
-                                }
-                                //await client.GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
-                                string oid = $"{semester.studyYear}-{semester.currSemester.id}-{studentSemester.stuid}";
-                                string ocode = $"OverallEducation-{school.id}";
-
-                                var student = studentsBase.Find(x => x.id.Equals(studentSemester.stuid));
-                                if (student!=null)
-                                {
-                                    OverallEducation overallEducation = overallEducations.Find(x => x.id.Equals(oid) && x.code.Equals(ocode));
-                                    if (overallEducation== null)
-                                    {
-#if !DEBUG
-                                        ResponseMessage oresponse = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).ReadItemStreamAsync(oid, new PartitionKey(ocode));
-                                        if (!oresponse.IsSuccessStatusCode)
-                                        {
-                                            overallEducation = new OverallEducation
-                                            {
-                                                id =oid,
-                                                code = $"OverallEducation-{school.id}",
-                                                pk = "OverallEducation",
-                                                ttl = -1,
-                                                name = student.name,
-                                                classId = student?.classId,
-                                                schoolCode = $"{school.id}",
-                                                semesterId = semester.currSemester.id,
-                                                year = semester.studyYear,
-                                                periodId = $"{period.id}",
-                                                stuYear = student.year,
-                                                studentId = student.id,
-                                                lessonScore= new List<StudentLessonRecord>()
-                                            };
-                                        }
-                                        else
+                                    else {
+                                        var lesData=  overallEducation.les.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
+                                        if (lesData== null)
                                         {
-                                            overallEducation = JsonDocument.Parse(oresponse.Content).RootElement.ToObject<OverallEducation>();
+                                            overallEducation.les.Add(new StuLesson { id=lessonLocal.lessonRecord.id, attend=0, time = lessonLocal.lessonRecord.startTime });
                                         }
-#else
-                                        overallEducation = new OverallEducation
-                                        {
-                                            id =oid,
-                                            code = $"OverallEducation-{school.id}",
-                                            pk = "OverallEducation",
-                                            ttl = -1,
-                                            name = student.name,
-                                            classId = student?.classId,
-                                            schoolCode = $"{school.id}",
-                                            semesterId = semester.currSemester.id,
-                                            year = semester.studyYear,
-                                            periodId = $"{period.id}",
-                                            stuYear = student.year,
-                                            studentId = student.id,
-                                            lessonScore= new List<StudentLessonRecord>()
-                                        };
-#endif
-                                        overallEducations.Add(overallEducation);
-                                    }
-                                    var hasrecord = overallEducation.lessonScore.Find(x => x.lessonId.Equals(lessonLocal.lessonRecord.id));
-                                    if (hasrecord==null)
-                                    {
-                                        hasrecord=   new StudentLessonRecord();
-                                        overallEducation.lessonScore.Add(hasrecord);
                                     }
-                                    hasrecord.gscore = studentLessonData.gscore;
-                                    hasrecord.pscore = studentLessonData.pscore;
-                                    hasrecord.tscore = studentLessonData.tscore;
-                                    hasrecord.tmdid = lessonLocal.lessonRecord.tmdid;
-                                    hasrecord.school = school.id;
-                                    hasrecord.scope = lessonLocal.lessonRecord.scope;
-                                    hasrecord.lessonId = lessonLocal.lessonRecord.id;
-                                    hasrecord.courseId = lessonLocal.lessonRecord.courseId;
-                                    periodId = period?.id;
-                                    hasrecord.subjectId = lessonLocal.lessonRecord.subjectId;
-                                    hasrecord.time= lessonLocal.lessonRecord.startTime;
-                                    hasrecord.attend=les.attend;
-                                    if (lesson!=null)
-                                    {
-                                        hasrecord.hrate=lesson.hrate;
-                                        hasrecord.crate=lesson.crate;
-                                        hasrecord.trate=lesson.trate;
-                                        hasrecord.xrate=lesson.xrate;
-                                        hasrecord.prate=lesson.prate;
-                                    }
-                                    //if (overallEducation!=null) 
-                                    //{
-                                    //    await client.GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code));
-                                    //    string key = $"OverallEducation:{school.id}:{period.id}:{semester.studyYear}:{semester.currSemester.id}:{student?.classId}";
-                                    //    await azureRedis.GetRedisClient(8).HashSetAsync(key, studentLessonData.id, overallEducation.ToJsonString());
-                                    //    await azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
-                                    //}
+                                   
                                 }
                             }
                         }
@@ -631,7 +389,8 @@ namespace HTEX.Lib.ETL.Lesson
             }
             return (studentLessonDatas, lessonItems, codeBools);
         }
-        public static async Task GenAnalysisData(string pathAnalysis, long newest, List<TechCount> techCounts,AzureStorageFactory azureStorage)
+
+      public static async Task GenAnalysisData(string pathAnalysis, long newest, List<TechCount> techCounts,AzureStorageFactory azureStorage)
         {
             var yearMonthDatas = techCounts.GroupBy(x => x.yearMonth).Select(x => new { key = x.Key, list = x.ToList() });
             // lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
@@ -776,6 +535,137 @@ namespace HTEX.Lib.ETL.Lesson
             await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(analysisModel.ToJsonString(), "lesson", $"analysis/analysis-model.json");
             System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis-model.json"), analysisModel.ToJsonString());
         }
+        public static async Task<TechCount>  GenTeachCount(LessonLocal lessonLocal, AzureStorageFactory azureStorage, List<string> objectiveTypes, List<string> ignore,string pathLessons,AzureCosmosFactory azureCosmos)
+        {
+            var data = GenStudentLessonData(lessonLocal,objectiveTypes);
+            string yearMonth = DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
+            if (data.codeBools.FindAll(x => x.value).IsNotEmpty())
+            {
+                TechCount count = new TechCount();
+
+                count.lessonId=lessonLocal.lessonRecord.id;
+                count.examCount= lessonLocal.examDatas.Count;
+                count.taskCount= lessonLocal.taskDatas.Count;
+                count.irsCount= lessonLocal.irsDatas.Count;
+                count.interactNormalCount=count.irsCount;
+                count.coworkCount= lessonLocal.coworkDatas.Count;
+                count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
+                count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
+
+
+                if (lessonLocal.lessonRecord!=null)
+                {
+
+                    count.yearMonth=yearMonth;
+
+                    if (lessonLocal?.lessonBase?.summary!=null)
+                    {
+                        count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
+                        count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
+                        count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
+                        count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
+                        count.examCountBase=lessonLocal.lessonBase.summary.examCount;
+                        count.interactNormalCountBase= count.interactNormalCount;
+                    }
+                    if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
+                    {
+                        count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
+                        count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
+                        count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
+                    }
+                    {
+
+                        count.pickup= data.pickup;
+                        //  var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId)  &&  !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) &&  x.lessonId.Equals(lessonLocal.lessonRecord.id));
+                        int sumUpload = 0;
+                        int taskCount = 0;
+                        int maxUpload = 0;
+                        HashSet<string> pickUp = new HashSet<string>();
+                        foreach (var stu in data.studentLessonDatas)
+                        {
+                            var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
+                            if (countS.Count()>0)
+                            {
+                                int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
+                                if (stuUploadmax> maxUpload)
+                                {
+                                    maxUpload=stuUploadmax;
+                                }
+                            }
+                            int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);
+
+                            sumUpload+=stuUpload;
+                            if (stu.taskRecord.itemRecords.Count()> taskCount)
+                            {
+                                taskCount=stu.taskRecord.itemRecords.Count();
+                            }
+                            var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
+                            if (stu_scores!=null  && stu_scores.Count()>0)
+                            {
+                                count.stuCowork.AddRange(stu_scores);
+                            }
+                            var grp_scores = stu.group_coworkScore.Where(x => x>0);
+                            if (grp_scores!=null  && grp_scores.Count()>0)
+                            {
+                                count.groupCowork.AddRange(grp_scores);
+                            }
+                            //if (stu.pickups.IsNotEmpty())
+                            //{
+                            //    foreach (var pickup in stu.pickups)
+                            //    {
+                            //        pickUp.Add(pickup);
+                            //    }
+                            //}
+                        }
+                        if (data.studentLessonDatas.Count>0&& taskCount>0  && maxUpload>0)
+                        {
+                            var avgUpload = sumUpload*1.0/(data.studentLessonDatas.Count *taskCount);
+                            count.upload.Add(new List<double>() { avgUpload, maxUpload });
+                        }
+                        string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
+                        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
+                    }
+                }
+            }
+            else
+            {
+                lessonLocal.lessonRecord.analysis=-1;
+                if (lessonLocal.lessonRecord.scope.Equals("school"))
+                {
+                    await azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).UpsertItemAsync(lessonLocal.lessonRecord, new PartitionKey(lessonLocal.lessonRecord.code));
+                }
+                else
+                {
+                    await azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).UpsertItemAsync(lessonLocal.lessonRecord, new PartitionKey(lessonLocal.lessonRecord.code));
+                }
+                try
+                {
+                    System.IO.File.Delete($"{pathLessons}\\MM{yearMonth}\\{lessonLocal.lessonRecord!.id}-local.json");
+                }
+                catch (Exception ex) { }
+                try
+                {
+                    System.IO.File.Delete($"{pathLessons}\\MM{yearMonth}\\{lessonLocal.lessonRecord!.id}-count.json");
+                }
+                catch (Exception ex) { }
+            }
+            return null; 
+        }
+        public static (List<StudentLessonData> studentLessonDatas, List<CodeBool> codeBools,List<string> pickup) GenStudentLessonData(LessonLocal lessonLocal, List<string> objectiveTypes)
+        {
+            List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
+            studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
+            studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal.lessonRecord.id);
+            studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+            studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
+            studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+            studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+            var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
+            studentLessonDatas= pickupData.studentLessonDatas;
+            var codeBools = LessonETLService.GetCodeBools(studentLessonDatas);
+            return (studentLessonDatas,codeBools, pickupData.pickup);
+        }
+
         /// <summary>
         /// 
         /// </summary>
@@ -822,136 +712,7 @@ namespace HTEX.Lib.ETL.Lesson
                     var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
                     studentLessonDatas= pickupData.studentLessonDatas;
                     var codeBools = LessonETLService.GetCodeBools(studentLessonDatas);
-                    string yearMonth = DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
-                    if (codeBools.FindAll(x => x.value).IsNotEmpty())
-                    {
-                        if (count==null) { count= new TechCount(); }
-                        count.lessonId=item.Split("\\").Last().Replace("-local.json", "");
-                        count.examCount= lessonLocal.examDatas.Count;
-                        count.taskCount= lessonLocal.taskDatas.Count;
-                        count.irsCount= lessonLocal.irsDatas.Count;
-                        count.interactNormalCount=count.irsCount;
-                        count.coworkCount= lessonLocal.coworkDatas.Count;
-                        count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
-                        count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
-
-
-                        if (lessonLocal.lessonRecord!=null)
-                        {
-
-                            count.yearMonth=yearMonth;
-
-                            if (lessonLocal?.lessonBase?.summary!=null)
-                            {
-                                count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
-                                count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
-                                count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
-                                count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
-                                count.examCountBase=lessonLocal.lessonBase.summary.examCount;
-                                count.interactNormalCountBase= count.interactNormalCount;
-                            }
-                            if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
-                            {
-                                count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
-                                count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
-                                count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
-                            }
-                            {
-
-                                count.pickup= pickupData.pickup;
-                                //  var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId)  &&  !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) &&  x.lessonId.Equals(lessonLocal.lessonRecord.id));
-                                int sumUpload = 0;
-                                int taskCount = 0;
-                                int maxUpload = 0;
-                                HashSet<string> pickUp = new HashSet<string>();
-                                foreach (var stu in studentLessonDatas)
-                                {
-                                    var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
-                                    if (countS.Count()>0)
-                                    {
-                                        int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
-                                        if (stuUploadmax> maxUpload)
-                                        {
-                                            maxUpload=stuUploadmax;
-                                        }
-                                    }
-                                    int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);
-
-                                    sumUpload+=stuUpload;
-                                    if (stu.taskRecord.itemRecords.Count()> taskCount)
-                                    {
-                                        taskCount=stu.taskRecord.itemRecords.Count();
-                                    }
-                                    var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
-                                    if (stu_scores!=null  && stu_scores.Count()>0)
-                                    {
-                                        count.stuCowork.AddRange(stu_scores);
-                                    }
-                                    var grp_scores = stu.group_coworkScore.Where(x => x>0);
-                                    if (grp_scores!=null  && grp_scores.Count()>0)
-                                    {
-                                        count.groupCowork.AddRange(grp_scores);
-                                    }
-                                    //if (stu.pickups.IsNotEmpty())
-                                    //{
-                                    //    foreach (var pickup in stu.pickups)
-                                    //    {
-                                    //        pickUp.Add(pickup);
-                                    //    }
-                                    //}
-                                }
-                                if (studentLessonDatas.Count>0&& taskCount>0  && maxUpload>0)
-                                {
-                                    var avgUpload = sumUpload*1.0/(studentLessonDatas.Count *taskCount);
-                                    count.upload.Add(new List<double>() { avgUpload, maxUpload });
-                                }
-                                //if (pickUp.Count>0)
-                                //{
-                                //    count.pickup.AddRange(pickUp.ToList());
-                                //}
-                                string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
-                                await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
-                                //if (force)
-                                //{
-
-                                //}
-                                //else
-                                //{
-                                //    if (lessonRecords.FindAll(x => x.id.Equals(lessonLocal.lessonRecord.id)).IsNotEmpty())
-                                //    {
-                                //        // await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json", studentLessonDatas.ToJsonString());
-                                //        //只有不是强制更新的时候再去刷新线上的json文件。
-
-                                //        await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
-                                //        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
-                                //    }
-                                //}
-                            }
-                        }
-                    }
-                    else
-                    {
-                        lessonLocal.lessonRecord.analysis=-1;
-                        if (lessonLocal.lessonRecord.scope.Equals("school"))
-                        {
-                            await azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).UpsertItemAsync(lessonLocal.lessonRecord, new PartitionKey(lessonLocal.lessonRecord.code));
-                        }
-                        else
-                        {
-                            await azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).UpsertItemAsync(lessonLocal.lessonRecord, new PartitionKey(lessonLocal.lessonRecord.code));
-                        }
-                        try
-                        {
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonth}\\{lessonLocal.lessonRecord!.id}-local.json");
-                        }
-                        catch (Exception ex) { }
-                        try
-                        {
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonth}\\{lessonLocal.lessonRecord!.id}-count.json");
-                        }
-                        catch (Exception ex) { }
-                    }
-                }
+                 }
 
                 if (count!=null)
                 {
@@ -1163,20 +924,11 @@ namespace HTEX.Lib.ETL.Lesson
         /// <returns></returns>
         public static async Task<LessonLocal> GetLessonLocal(LessonRecord lessonRecord, List<string> localIds, AzureStorageFactory _azureStorage, string pathLessons)
         {
-            string scope = lessonRecord.scope;
             string owner = lessonRecord.scope.Equals("school") ? lessonRecord.school : lessonRecord.tmdid;
 
             string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(lessonRecord.startTime).ToString("yyyyMM");
-            LessonLocal lessonLocal = null;
-            if (System.IO.File.Exists($"{pathLessons}\\MM{yearMonthPath}\\{lessonRecord.id}-local.json"))
-            {
-                string jsonp = await System.IO.File.ReadAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{lessonRecord.id}-local.json");
-                lessonLocal = jsonp.ToObject<LessonLocal>();
-            }
-            else
-            {
-                lessonLocal= new LessonLocal { lessonRecord=lessonRecord };
-                List<string> files = new List<string>()
+
+            List<string> files = new List<string>()
                     {
                         $"/records/{lessonRecord.id}/IES/TimeLine.json",
                         $"/records/{lessonRecord.id}/IES/base.json",
@@ -1186,66 +938,67 @@ namespace HTEX.Lib.ETL.Lesson
                         $"/records/{lessonRecord.id}/IES/Cowork.json",
                         $"/records/{lessonRecord.id}/Sokrates/SokratesRecords.json",
                     };
-                lessonLocal  = new LessonLocal { lessonRecord=lessonRecord };
-                lessonLocal =  await GetLessonFiles(lessonLocal, files, owner, _azureStorage);
-                if (lessonLocal.lessonBase!=null  &&  lessonLocal.lessonBase.student!=null)
-                {
-                    var baseData = GetBaseData(lessonLocal.lessonBase!);
-                    lessonLocal.studentLessonDatas= baseData.studentLessonDatas;
-                    List<ExamData> examDatas = await GetExamInfo(lessonRecord, lessonLocal.timeLineData, _azureStorage, owner);
-                    lessonLocal.examDatas = examDatas;
-                    lessonLocal.sokratesDatas= lessonLocal.sokratesDatas.IsNotEmpty() ? lessonLocal.sokratesDatas : lessonLocal.timeLineData!=null ? lessonLocal.timeLineData.events : new List<TimeLineEvent>();
-                }
+            LessonLocal lessonLocal  = new LessonLocal { lessonRecord=lessonRecord };
+            lessonLocal =  await GetLessonFiles(lessonLocal, files, owner, _azureStorage);
+            if (lessonLocal.lessonBase!=null  &&  lessonLocal.lessonBase.student!=null)
+            {
+                var baseData = GetBaseData(lessonLocal.lessonBase!);
+                lessonLocal.studentLessonDatas= baseData.studentLessonDatas;
+                List<ExamData> examDatas = await GetExamInfo(lessonRecord, lessonLocal.timeLineData, _azureStorage, owner);
+                lessonLocal.examDatas = examDatas;
+                lessonLocal.sokratesDatas= lessonLocal.sokratesDatas.IsNotEmpty() ? lessonLocal.sokratesDatas : lessonLocal.timeLineData!=null ? lessonLocal.timeLineData.events : new List<TimeLineEvent>();
             }
-
-            
             return lessonLocal;
         }
-        public static async IAsyncEnumerable<LessonLocal> GetLessonLocal(List<LessonRecord> lessonRecords, List<string> localIds, AzureStorageFactory _azureStorage, string pathLessons)
+       
+        private static async Task<LessonLocal> GetLessonFiles(LessonLocal lessonLocal, List<string> files, string owner, AzureStorageFactory _azureStorage)
         {
-            foreach (var lessonRecord in lessonRecords)
+#if DEBUG
+            foreach (var file in files) 
             {
-                string scope = lessonRecord.scope;
-                string owner = lessonRecord.scope.Equals("school") ? lessonRecord.school : lessonRecord.tmdid;
-             
-                string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(lessonRecord.startTime).ToString("yyyyMM");
-                LessonLocal lessonLocal = new LessonLocal { lessonRecord=lessonRecord };
-                if (System.IO.File.Exists($"{pathLessons}\\MM{yearMonthPath}\\{lessonRecord.id}-local.json"))
-                {
-                    string jsonp = await System.IO.File.ReadAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{lessonRecord.id}-local.json");
-                    lessonLocal = jsonp.ToObject<LessonLocal>();
-                }
-                else
+                try
                 {
-                    List<string> files = new List<string>()
-                    {
-                        $"/records/{lessonRecord.id}/IES/TimeLine.json",
-                        $"/records/{lessonRecord.id}/IES/base.json",
-                        $"/records/{lessonRecord.id}/IES/Task.json",
-                        $"/records/{lessonRecord.id}/IES/SmartRating.json",
-                        $"/records/{lessonRecord.id}/IES/IRS.json",
-                        $"/records/{lessonRecord.id}/IES/Cowork.json",
-                        $"/records/{lessonRecord.id}/Sokrates/SokratesRecords.json",
-                    };
-                    lessonLocal  = new LessonLocal { lessonRecord=lessonRecord };
-                    lessonLocal =  await GetLessonFiles(lessonLocal, files, owner, _azureStorage);
-                    if (lessonLocal.lessonBase!=null  &&  lessonLocal.lessonBase.student!=null)
+                    var exists = _azureStorage.GetBlobContainerClient(owner).GetBlobClient(file).Exists();
+                    if (exists)
                     {
-                        var baseData = GetBaseData(lessonLocal.lessonBase!);
-                        lessonLocal.studentLessonDatas= baseData.studentLessonDatas;
-                        List<ExamData> examDatas = await GetExamInfo(lessonRecord, lessonLocal.timeLineData, _azureStorage, owner);
-                        lessonLocal.examDatas = examDatas;
-                        lessonLocal.sokratesDatas= lessonLocal.sokratesDatas.IsNotEmpty() ? lessonLocal.sokratesDatas : lessonLocal.timeLineData!=null ? lessonLocal.timeLineData.events : new List<TimeLineEvent>();
+                        BlobDownloadResult blobDownloadResult = await _azureStorage.GetBlobContainerClient(owner).GetBlobClient(file).DownloadContentAsync();
+                        switch (true)
+                        {
+                            case bool when file.Contains("IES/TimeLine.json"):
+                                lessonLocal.timeLineData= blobDownloadResult.Content.ToObjectFromJson<TimeLineData>();
+                                break;
+                            case bool when file.Contains("IES/base.json"):
+                                lessonLocal.lessonBase= blobDownloadResult.Content.ToObjectFromJson<LessonBase>();
+                                break;
+                            case bool when file.Contains("IES/Task.json"):
+                                lessonLocal.taskDatas= blobDownloadResult.Content.ToObjectFromJson<List<TaskData>>();
+                                break;
+                            case bool when file.Contains("IES/SmartRating.json"):
+                                lessonLocal.smartRatingDatas= blobDownloadResult.Content.ToObjectFromJson<List<SmartRatingData>>();
+                                break;
+                            case bool when file.Contains("IES/IRS.json"):
+                                lessonLocal.irsDatas= blobDownloadResult.Content.ToObjectFromJson<List<IRSData>>();
+                                break;
+                            case bool when file.Contains("IES/Cowork.json"):
+                                lessonLocal.coworkDatas= blobDownloadResult.Content.ToObjectFromJson<List<CoworkData>>();
+                                break;
+                            case bool when file.Contains("Sokrates/SokratesRecords.json"):
+                                lessonLocal.sokratesDatas= blobDownloadResult.Content.ToObjectFromJson<List<TimeLineEvent>>();
+                                break;
+                        }
                     }
                 }
+                catch (RequestFailedException ex)
+                {
 
-                
-                yield return lessonLocal;
+                }
+                catch (Exception ex)
+                {
+
+                }
             }
-        }
-        private static async Task<LessonLocal> GetLessonFiles(LessonLocal lessonLocal, List<string> files, string owner, AzureStorageFactory _azureStorage)
-        {
 
+#else
             await Parallel.ForEachAsync(files, async (file, _) =>
             {
                 try
@@ -1282,13 +1035,14 @@ namespace HTEX.Lib.ETL.Lesson
                 }
                 catch (RequestFailedException ex)
                 {
-                    Console.WriteLine($"{file},{ex.Message},{ex.StackTrace}");
+                   
                 }
                 catch (Exception ex)
                 {
-                    Console.WriteLine($"{file},{ex.Message},{ex.StackTrace}");
+                   
                 }
             });
+#endif
             return lessonLocal;
         }
 
@@ -3011,21 +2765,25 @@ namespace HTEX.Lib.ETL.Lesson
 
                         var w = studentLessonData.coworkRecord.itemRecords.Where(x => x.resultWeight>0);
                         double ss = w.Sum(x => x.itemScore)*1.0;
-                        double sw = w.Sum(x => x.resultWeight)*1.0;
+                        //double sw = w.Sum(x => x.resultWeight)*1.0;
                         double wc = w.Count()*1.0;
-                        double x = 0.0;
-                        if (wc>0)
-                        {
-                            x=sw/(j *wc);
-                        }
-                        double max_xzcg = 40;
-                        double k = (wc*wc/n+x)/n+ wc*(ss/max_xzcg)* (wc/n);
-                        double f6 = Math.Round(190*1.0/(1+Math.Exp(-(k)))-95, 4);
+                        //double x = 0.0;
+                        //if (wc>0)
+                        //{
+                        //    //x=sw/(j *wc);
+                        //}
+                       // double max_xzcg = 75;
+                        //double k = (wc*wc/n+x)/n+ wc*(ss/max_xzcg)* (wc/n);
+                        //double k = wc/n +wc*(ss/max_xzcg)* (wc/n);
+                        var avg = lessonDataAnalysis.stuCowork.Select(x => x.Value* x.Key).Sum()*1.0/lessonDataAnalysis.stuCowork.Sum(x => x.Value);
+                        double k=(GetPersent(lessonDataAnalysis.stuCowork, ss).persent/100+ ss/ avg)* (wc/n  );
+                       // double f6 = Math.Round(190*1.0/(1+Math.Exp(-(k)))-95, 4);
                         double f7 = Math.Round(200*1.0/(1+Math.Exp(-(k)))-100, 4);
+
                         lessonItem.xz_fqc =n;
                         lessonItem.xz_cyc =wc;
                         lessonItem.xz_cgf =ss;
-                        lessonItem.xz_cx =f6;
+                        lessonItem.xz_cx =f7;
                         lessonItem.xz_cy =f7;
                     }
                     //_logger.LogInformation($"{studentLessonData.id}=>协作指数：{f6}\t专注指数：{f7}\t协作次数：{n}\t参与次数：{wc}\t协作成果分数：{ss}\t{k}");

TEAMModelOS.Extension/HTEX.Test/Controllers/MockDataController.cs

@@ -37,7 +37,7 @@ namespace HTEX.Test.Controllers
             //评价次数
             int pcount = Random.Shared.Next(3, 4);
             //协作次数
-            int xcount = Random.Shared.Next(1, 3);
+            int xcount = Random.Shared.Next(2, 3);
             //挑人次数
             int kcount = Random.Shared.Next(2, 4);
             string type = "hd";
@@ -348,7 +348,7 @@ namespace HTEX.Test.Controllers
                             case bool when w==1:
                                 item.resultWeight= InteractWeight.TP;
                                 item.resultType= InteractReultType.TP;
-                                int[] q = new int[] { 10, 5, 5, 20, 30, 10, 25, 10, 15, 15, 30, 5, 10, 20, 15, 25, 10, 5, 15, 20, 20, 25, 5, 5 };
+                                int[] q = new int[] { 10, 5, 5, 20, 30, 10, 25, 70, 75, 15, 30, 75, 10, 20, 15, 25, 70, 5, 75, 20, 20, 25, 5, 5 };
                                 item.itemScore=q[Random.Shared.Next(0, q.Count())];
                                 break;
                         }
@@ -369,7 +369,7 @@ namespace HTEX.Test.Controllers
                         //
                         case bool when w==1:
 
-                            int[] q = new int[] { 10, 5, 5, 20, 30, 10, 25, 10, 15, 15, 30, 5, 10, 20, 15, 25, 10, 5, 15, 20, 20, 25, 5, 5 };
+                            int[] q = new int[] { 10, 5, 5, 20, 30, 70, 25, 10, 15,75, 30, 75, 10, 20, 75, 25, 10, 5, 75, 20, 20, 25, 5, 5 };
                             grpcoworkScore= q[Random.Shared.Next(0, q.Count())]* Random.Shared.Next(2, 4);
                             student.group_coworkScore.Add(grpcoworkScore);
                             break;

TEAMModelOS.Function/IESServiceBusTrigger.cs

@@ -1766,27 +1766,62 @@ namespace TEAMModelOS.Function
                                         try {
                                             if (lessonRecord.duration>0)
                                             {
-
                                                 var location = Environment.GetEnvironmentVariable("Option:Location");
-                                                List<School> schools= new List<School>();
-                                                List<StudentSemesterRecord> studentSemesterRecords = new List<StudentSemesterRecord>();
-                                                List<OverallEducation> overallEducations = new List<OverallEducation>();
-                                                List<Student> studentsBase = new List<Student>();
-                                                List<StudentSemesterRecord> students = new List<StudentSemesterRecord>();
-                                                LessonDataAnalysisModel lessonDataAnalysis = null;
-                                                await LessonETLService.DoStudentLessonData(Constant.objectiveTypes, _azureStorage, lessonLocal, _dingDing, client, location,_azureRedis, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
-                                                await Parallel.ForEachAsync(studentSemesterRecords, async (studentSemester, cancellationToken) =>
+                                                if (location.Equals("China", StringComparison.OrdinalIgnoreCase)) 
                                                 {
-                                                    await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
+                                                    List<School> schools = new List<School>();
+                                                    List<StudentSemesterRecord> studentSemesterRecords = new List<StudentSemesterRecord>();
+                                                    List<OverallEducation> overallEducations = new List<OverallEducation>();
+                                                    List<Student> studentsBase = new List<Student>();
+                                                    List<StudentSemesterRecord> students = new List<StudentSemesterRecord>();
+                                                    LessonDataAnalysisModel lessonDataAnalysis = null;
+                                                    bool exists = await _azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis-model.json").ExistsAsync();
+                                                    if (exists)
+                                                    {
+                                                        BlobDownloadResult blobDownload = await _azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis-model.json").DownloadContentAsync();
+                                                        lessonDataAnalysis = blobDownload.Content.ToObjectFromJson<LessonDataAnalysisModel>();
+                                                        string studentSql = $"select value c from c where c.id in ({string.Join(",", lessonLocal.studentLessonDatas.Select(x => $"'{x.id}'"))})";
+                                                        var studentResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<Student>(studentSql, $"Base-{school}");
+                                                        if (studentResults.list.IsNotEmpty())
+                                                        {
+                                                            studentsBase.AddRange(studentResults.list);
+                                                        }
+                                                        School schoolBase = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemAsync<School>(school, new PartitionKey("Base"));
+                                                        schools.Add(schoolBase);
+                                                        string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : schoolBase.period.FirstOrDefault()?.id;
+                                                        var period = schoolBase.period.Find(x => x.id.Equals(periodId));
+                                                        var semester = SchoolService.GetSemester(period, lessonLocal.lessonRecord.startTime);
+                                                        string code = $"StudentSemesterRecord-{schoolBase.id}";
+                                                        string studentSemesterRecordSql = $"select value c from c where c.stuid in ({string.Join(",", lessonLocal.studentLessonDatas.Select(x => $"'{x.id}'"))}) and c.semesterId='{semester.currSemester.id}' and c.studyYear={semester.studyYear}";
+                                                        var studentSemesterRecordResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<StudentSemesterRecord>(studentSemesterRecordSql, code);
+                                                        if (studentSemesterRecordResults.list.IsNotEmpty())
+                                                        {
+                                                            studentSemesterRecords.AddRange(studentSemesterRecordResults.list);
+                                                        }
+                                                        string overallEducationSql = $"select value c from c where c.studentId in ({string.Join(",", lessonLocal.studentLessonDatas.Select(x => $"'{x.id}'"))}) and c.semesterId='{semester.currSemester.id}' and c.year={semester.studyYear}";
+                                                        var overallEducationResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<OverallEducation>(overallEducationSql, $"OverallEducation-{schoolBase.id}");
+                                                        if (overallEducationResults.list.IsNotEmpty())
+                                                        {
+                                                            overallEducations.AddRange(overallEducationResults.list);
+                                                        }
+                                                        LessonETLService.DoStudentLessonDataV2(Constant.objectiveTypes, lessonLocal, location, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
+                                                        await Parallel.ForEachAsync(studentSemesterRecords, async (studentSemester, cancellationToken) =>
+                                                        {
+                                                            await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
+
+                                                        });
+                                                        await Parallel.ForEachAsync(overallEducations, async (overallEducation, cancellationToken) =>
+                                                        {
+                                                            await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code));
+                                                            string key = $"OverallEducation:{overallEducation.schoolCode}:{overallEducation.periodId}:{overallEducation.year}:{overallEducation.semesterId}:{overallEducation?.classId}";
+                                                            await _azureRedis.GetRedisClient(8).HashSetAsync(key, overallEducation.studentId, overallEducation.ToJsonString());
+                                                            await _azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
+                                                        });
+                                                    }
+
+                                                }
+
 
-                                                });
-                                                await Parallel.ForEachAsync(overallEducations, async (overallEducation, cancellationToken) =>
-                                                {
-                                                    await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code));
-                                                    string key = $"OverallEducation:{overallEducation.schoolCode}:{overallEducation.periodId}:{overallEducation.year}:{overallEducation.semesterId}:{overallEducation?.classId}";
-                                                    await _azureRedis.GetRedisClient(8).HashSetAsync(key, overallEducation.studentId, overallEducation.ToJsonString());
-                                                    await _azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
-                                                });
                                                 // 使用当前文化设置的日历
                                                 CultureInfo cultureInfo = CultureInfo.CurrentCulture;
                                                 Calendar calendar = cultureInfo.Calendar;
@@ -1813,7 +1848,7 @@ namespace TEAMModelOS.Function
                                         } catch (Exception ex) {
                                             await _dingDing.SendBotMsg($"{Environment.GetEnvironmentVariable("Option:Location")}处理学生课中数据发生异常,{_lessonId}\n{ex.Message}\n{ex.StackTrace}\n\n{lessonRecord.ToJsonString()}", GroupNames.醍摩豆服務運維群組);
                                         }
-                                        DoLessonStudentRecord(_dingDing, _snowflakeId, lessonRecord, scope, client, school, tmdid, teacher, _serviceBus, _azureStorage, _configuration, lessonBase, _azureRedis, PickupMemberIds, taskDatas, irsDatas);
+                                       // DoLessonStudentRecord(_dingDing, _snowflakeId, lessonRecord, scope, client, school, tmdid, teacher, _serviceBus, _azureStorage, _configuration, lessonBase, _azureRedis, PickupMemberIds, taskDatas, irsDatas);
 
                                     }
                                     catch (Exception ex)

TEAMModelOS.SDK/Models/Cosmos/Common/StudentScoreRecord.cs

@@ -6,9 +6,7 @@ using System.Threading.Tasks;
 
 namespace TEAMModelOS.SDK.Models.Cosmos.Common
 {
-    public class TeacherSemesterRecord : CosmosEntity { 
-    
-    }
+     
 
     /// <summary>
     /// 

TEAMModelOS.SDK/Models/Cosmos/Student/OverallEducation.cs

@@ -76,6 +76,7 @@ namespace TEAMModelOS.SDK.Models
         /// 学生课堂积分
         /// </summary>
         public List<StudentLessonRecord> lessonScore { get; set; } = new List<StudentLessonRecord>();
+        public List<StuLesson> les { get; set; } = new List<StuLesson>();
 
     }
     public class EducationScore {