update

HTEX.DataETL/Controllers/LessonRecordController.cs

@@ -143,8 +143,8 @@ namespace HTEX.DataETL.Controllers
                     studentLessonDatas= pickupData.studentLessonDatas;
                     var codeBools= LessonETLService.GetCodeBools(studentLessonDatas);
                     await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString());
-                    string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis.json");
-                    LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
+                    string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis-model.json");
+                    LessonDataAnalysisModel lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisModel>();
                     
 
                     var studentLessons = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis,codeBools);
@@ -162,24 +162,24 @@ namespace HTEX.DataETL.Controllers
         /// </summary>
         /// <param name="json"></param>
         /// <returns></returns>
-        //[HttpPost("process-fix-history-students")]
-        //public async Task<IActionResult> ProcessFixHistoryStudents(JsonElement json)
-        //{
-        //    string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
-        //    string studentsPath = $"{lessonBasePath}\\students";
-        //    List<string> studentsPs = FileHelper.ListAllFiles(studentsPath);
-        //    string? pathLessons = $"{lessonBasePath}\\lessons";
-        //    List<string> filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
-        //    int index = 0;
-        //    foreach (var stu in studentsPs)
-        //    {
-        //        string stujson = await System.IO.File.ReadAllTextAsync(stu);
-        //        var studentSemester = stujson.ToObject<StudentSemesterRecord>();
-        //        await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
-        //        index++;
-        //    }
-        //    return Ok();
-        //}
+        [HttpPost("process-fix-history-students")]
+        public async Task<IActionResult> ProcessFixHistoryStudents(JsonElement json)
+        {
+            string? pathAnalysis = $"F:\\lesson-local\\analysis";
+            try
+            {
+                string jsons = await System.IO.File.ReadAllTextAsync($"F:\\lesson-local\\analysis\\analysis-model.json");
+                var s = JsonSerializer.Deserialize<LessonDataAnalysisModel>(jsons);
+                LessonDataAnalysisModel lessonDataAnalysis = JsonDocument.Parse(jsons).RootElement.ToObject<LessonDataAnalysisModel>();
+                var per = LessonETLService.GetPersent(lessonDataAnalysis.irs, 2);
+
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.ToString());
+            }
+            return Ok();
+        }
         /// <summary>
         /// 
         /// </summary>
@@ -193,8 +193,8 @@ namespace HTEX.DataETL.Controllers
             string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
             string? pathLessons = $"{lessonBasePath}\\lessons";
             string? pathAnalysis = $"{lessonBasePath}\\analysis";
-            string jsons = await System.IO.File.ReadAllTextAsync($"{pathAnalysis}\\analysis.json");
-            LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
+            string jsons = await System.IO.File.ReadAllTextAsync($"{pathAnalysis}\\analysis-model.json");
+            LessonDataAnalysisModel lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisModel>();
             List<string> filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
             var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
             XmlDocument xmlDocument = new XmlDocument();
@@ -211,9 +211,9 @@ namespace HTEX.DataETL.Controllers
                     localIds.Add(lessonId);
                 }
             }
-            long stime = 1690819200000;//2023-08-01 00:00:00
+            long stime = 1693497600000;//2023-08-01 00:00:00
             var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
-                 .GetList<LessonRecord>($"SELECT value c FROM   c  where c.startTime>{stime} and   c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ", $"LessonRecord-ydzt");
+                 .GetList<LessonRecord>($"SELECT value c FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and c.startTime>{stime} and   c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ");
             List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
             int index = 0;
             if (resultSchool.list.IsNotEmpty())
@@ -221,66 +221,12 @@ namespace HTEX.DataETL.Controllers
                 List<StudentSemesterRecord>  studentSemesterRecords= new List<StudentSemesterRecord>();
                 List<OverallEducation> overallEducations= new List<OverallEducation>();
                 List<Student> studentsBase = new List<Student>();
-             
-                await foreach (var item in LessonETLService.GetLessonLocal(resultSchool.list, localIds, _azureStorage, pathLessons))
+                await Parallel.ForEachAsync(resultSchool.list, async (lessonRecord, cancellationToken) =>
                 {
-                    string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
-                    TechCount techCount = null;
-                    if (item.lessonBase!=null && item.lessonBase.student!=null)
-                    {
-                         techCount = new TechCount
-                        {
-                            lessonId=item.lessonRecord?.id,
-                            examCount = item.examDatas.Count,
-                            taskCount = item.taskDatas.Count,
-                            irsCount = item.irsDatas.Count,
-                            coworkCount = item.coworkDatas.Count,
-                            smartRatingCount =item.smartRatingDatas.Count,
-                            timeCount=item.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList()
-                        };
-                      
-                    }
-                    else
-                    {
-                        try
-                        {
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
-                        }
-                        catch (Exception ex) { }
-                        try
-                        {
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
-                        }
-                        catch (Exception ex)
-                        {
-
-                        }
-                    }
+                    var  item = await LessonETLService.GetLessonLocal(lessonRecord,localIds,_azureStorage,pathLessons);
                     var studata = await LessonETLService.DoStudentLessonData(Constant.objectiveTypes, _azureStorage, item, _dingDing, _azureCosmos.GetCosmosClient(), "China", _azureRedis, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
-                    if (techCount!= null) 
-                    {
-                        if (studata.codeBools.FindAll(x => x.value).IsNotEmpty())
-                        {
-                            await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord.id}-count.json", techCount.ToJsonString());
-                            await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
-                        }
-                        else {
-                            try {
-                                System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
-                            }
-                            catch (Exception ex) { }
-                            try
-                            {
-                                System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
-                            }
-                            catch (Exception ex) 
-                            {
-                            
-                            }
-                        }
-                    }
                     index++;
-                }
+                });
                
                 await Parallel.ForEachAsync(studentSemesterRecords, async (studentSemester, cancellationToken) => 
                 {
@@ -389,61 +335,46 @@ namespace HTEX.DataETL.Controllers
                 List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
                 if (lessonRecords.IsNotEmpty())
                 {
-                    await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
-                    {
+                    await Parallel.ForEachAsync(lessonRecords, async(record, _)=>{
+                        LessonLocal  item = await LessonETLService.GetLessonLocal(record, localIds, _azureStorage, pathLessons);
                         string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
                         item.lessonRecord.learningCategory= item.lessonBase?.summary?.learningCategory;
                         if (item.lessonRecord.learningCategory == null)
                         {
 
-                            item. lessonRecord.learningCategory = new LearningCategory();
+                            item.lessonRecord.learningCategory = new LearningCategory();
                         }
                         await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
-                        //if (item.lessonBase!=null && item.lessonBase.student!=null)
-                        //{
-                        //    TechCount techCount = new TechCount
-                        //    {
-                        //        yearMonth=yearMonthPath,
-                        //        lessonId=item.lessonRecord?.id,
-                        //        examCount = item.examDatas.Count,
-                        //        taskCount = item.taskDatas.Count,
-                        //        irsCount = item.irsDatas.Count,
-                        //        coworkCount = item.coworkDatas.Count,
-                        //        smartRatingCount =item.smartRatingDatas.Count,
-                        //        timeCount=item.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList()
-                        //    };
-                        //    if (techCount.examCount>0 || techCount.taskCount>0 ||techCount.irsCount>0 || techCount.coworkCount>0 || techCount.smartRatingCount>0
-                                
-                        //        )
-                        //    {
-                        //        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord.id}-count.json", techCount.ToJsonString());
-                               
-                        //    }
-                        //    else {
-                        //        try
-                        //        {
-                        //            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
-                        //        }
-                        //        catch (Exception ex) { }
-                              
-                        //    }
-                           
-                        //}
-                        //else
-                        //{
+                    });
 
-                        //    System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
-                        //    System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
-                        //}
-                    }
+
+                   //await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
+                   // {
+                   //     string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
+                   //     item.lessonRecord.learningCategory= item.lessonBase?.summary?.learningCategory;
+                   //     if (item.lessonRecord.learningCategory == null)
+                   //     {
+
+                   //         item. lessonRecord.learningCategory = new LearningCategory();
+                   //     }
+                   //     await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
+                   // }
                 }
                 List<TechCount> techCounts = new List<TechCount>();
                 filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
-                await foreach (var item in LessonETLService.GetTeachCount(_azureCosmos,lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force))
-                {
-                    techCounts.Add(item);
-                }
-                await   LessonETLService.GenAnalysisData(pathAnalysis, newest, techCounts,_azureStorage);
+                //await foreach (var item in LessonETLService.GetTeachCount(_azureCosmos,lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force))
+                //{
+                //    techCounts.Add(item);
+                //}
+
+                await Parallel.ForEachAsync(filesLessons, async (item, _) => {
+                    TechCount techCount=  await  LessonETLService.GetTeachCount(_azureCosmos, item, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force);
+                    if (techCount != null) {
+                        techCounts.Add(techCount);
+                    }
+                });
+
+                await LessonETLService.GenAnalysisData(pathAnalysis, newest, techCounts,_azureStorage);
             }
             return Ok(new { yearMonth });
         }

TEAMModelBI/TEAMModelBI.csproj

@@ -65,9 +65,9 @@
 		<SpaRoot>ClientApp\</SpaRoot>
 		<DefaultItemExcludes>$(DefaultItemExcludes);$(SpaRoot)node_modules\**</DefaultItemExcludes>
 		<UserSecretsId>078b5d89-7d90-4f6a-88fc-7d96025990a8</UserSecretsId>
-		<Version>5.2411.6</Version>
-		<AssemblyVersion>5.2411.6.1</AssemblyVersion>
-		<FileVersion>5.2411.6.1</FileVersion>
+		<Version>5.2411.13</Version>
+		<AssemblyVersion>5.2411.13.1</AssemblyVersion>
+		<FileVersion>5.2411.13.1</FileVersion>
 		<Description>TEAMModelBI(BI)</Description>
 		<PackageReleaseNotes>BI版本说明版本切换标记2022000908</PackageReleaseNotes>
 		<PackageId>TEAMModelBI</PackageId>

TEAMModelOS.Extension/HTEX.Lib/ETL/Lesson/LessonETLService.cs

@@ -115,7 +115,7 @@ namespace HTEX.Lib.ETL.Lesson
         public static async Task<(List<StudentLessonData> studentLessonDatas, List<StudentLessonItem> lessonItems, List<CodeBool> codeBools)>
             DoStudentLessonData(List<string> objectiveTypes, AzureStorageFactory azureStorage, LessonLocal lessonLocal,DingDing _dingDing,
             CosmosClient client,string location ,AzureRedisFactory azureRedis, List<StudentSemesterRecord> studentSemesterRecords, List<OverallEducation> overallEducations,
-            LessonDataAnalysisCluster lessonDataAnalysis, List<Student> studentsBase,List<School> schools )
+            LessonDataAnalysisModel lessonDataAnalysis, List<Student> studentsBase,List<School> schools )
         {
             List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
             studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
@@ -135,13 +135,13 @@ namespace HTEX.Lib.ETL.Lesson
                 try
                 {
 
-                    bool exists = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis.json").ExistsAsync();
+                    bool exists = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis-model.json").ExistsAsync();
                     if (exists)
                     {
                         if (lessonDataAnalysis==null)
                         {
-                            BlobDownloadResult blobDownload = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis.json").DownloadContentAsync();
-                            lessonDataAnalysis = blobDownload.Content.ToObjectFromJson<LessonDataAnalysisCluster>();
+                            BlobDownloadResult blobDownload = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis-model.json").DownloadContentAsync();
+                            lessonDataAnalysis = blobDownload.Content.ToObjectFromJson<LessonDataAnalysisModel>();
                         }
 
                         lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis, codeBools);
@@ -156,7 +156,7 @@ namespace HTEX.Lib.ETL.Lesson
                 {
                     if (!ex.Message.Contains("The specified blob does not exist"))
                     {
-                        await _dingDing.SendBotMsg($"{Environment.GetEnvironmentVariable("Option:Location")},lesson/analysis/analysis.json转换异常，{ex.Message}{ex.StackTrace}", GroupNames.成都开发測試群組);
+                        await _dingDing.SendBotMsg($"{Environment.GetEnvironmentVariable("Option:Location")},lesson/analysis/analysis-model.json转换异常，{ex.Message}{ex.StackTrace}", GroupNames.成都开发測試群組);
                     }
                 }
                 await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
@@ -479,8 +479,8 @@ namespace HTEX.Lib.ETL.Lesson
             List<KeyValuePair<double, List<double>>> levelInteract = new List<KeyValuePair<double, List<double>>>()
             {
                 new KeyValuePair<double, List<double>>(low, new List<double> { all.Where(x => x<pass).Min(), all.Where(x => x<pass).Max() }),
-                new KeyValuePair<double, List<double>>(low, new List<double> { all.Where(x => x>=pass  &&x<good).Min(), all.Where(x => x>=pass  &&x<good).Max() }),
-                new KeyValuePair<double, List<double>>(low, new List<double> { all.Where(x => x>=good).Min(), all.Where(x => x>=good).Max() })
+                new KeyValuePair<double, List<double>>(medium, new List<double> { all.Where(x => x>=pass  &&x<good).Min(), all.Where(x => x>=pass  &&x<good).Max() }),
+                new KeyValuePair<double, List<double>>(high, new List<double> { all.Where(x => x>=good).Min(), all.Where(x => x>=good).Max() })
             };
             lessonDataAnalysisCluster.levelInteract= levelInteract;
             ////  设置0 为自动规划聚类，11 则为自动规划后得到的数字。
@@ -494,10 +494,210 @@ namespace HTEX.Lib.ETL.Lesson
             //    lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
             //    //Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
             //}
-            await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(lessonDataAnalysisCluster.ToJsonString(), "lesson", $"analysis/analysis.json");
-            System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
+            //await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(lessonDataAnalysisCluster.ToJsonString(), "lesson", $"analysis/analysis.json");
+            //System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
+            LessonDataAnalysisModel analysisModel = new LessonDataAnalysisModel();
+            analysisModel.task= lessonDataAnalysisCluster.task.GroupBy(x =>Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.irs= lessonDataAnalysisCluster.irs.GroupBy(x => Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.interactNormal= lessonDataAnalysisCluster.interactNormal.GroupBy(x => Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.pscore= lessonDataAnalysisCluster.pscore.GroupBy(x => Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.gscore= lessonDataAnalysisCluster.gscore.GroupBy(x => Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.tscore= lessonDataAnalysisCluster.tscore.GroupBy(x => Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.stuCowork= lessonDataAnalysisCluster.stuCowork.GroupBy(x => Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.groupCowork= lessonDataAnalysisCluster.groupCowork.GroupBy(x => Math.Ceiling(x)).Select(x => new KeyValuePair<double, int>(x.Key, x.ToList().Count)).ToList();
+            analysisModel.interactPass= lessonDataAnalysisCluster.interactPass;
+            analysisModel.interactGood= lessonDataAnalysisCluster.interactGood;
+            analysisModel.interactLow= lessonDataAnalysisCluster.interactLow;
+            analysisModel.interactMedium= lessonDataAnalysisCluster.interactMedium;
+            analysisModel.interactHigh= lessonDataAnalysisCluster.interactHigh;
+            analysisModel.levelInteract= lessonDataAnalysisCluster.levelInteract;
+            await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(analysisModel.ToJsonString(), "lesson", $"analysis/analysis-model.json");
+            System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis-model.json"), analysisModel.ToJsonString());
         }
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="azureCosmos"></param>
+        /// <param name="item"></param>
+        /// <param name="pathLessons"></param>
+        /// <param name="ignore"></param>
+        /// <param name="objectiveTypes"></param>
+        /// <param name="azureStorage"></param>
+        /// <param name="force"></param>
+        /// <returns></returns>
+        public static async Task<TechCount> GetTeachCount(AzureCosmosFactory azureCosmos, string item, string pathLessons, List<string> ignore, List<string> objectiveTypes, AzureStorageFactory azureStorage, bool force)
+        {
+            List<TechCount> techCounts = new List<TechCount>();
+            if (item.EndsWith("-local.json"))
+            {
+                string lessonId = item.Split("\\").Last().Replace("-local.json", "");
+                string countFile = item.Replace("-local.json", "-count.json");
+                TechCount count = null;
+                if (!force)
+                {
+                    if (System.IO.File.Exists(countFile))
+                    {
+                        string contjson = await System.IO.File.ReadAllTextAsync(countFile);
+                        count = contjson.ToObject<TechCount>();
+                    }
+                    else
+                    {
+                        force=true;
+
+                    }
+                }
+                if (force)
+                {
+                    string localjson = await System.IO.File.ReadAllTextAsync(item);
+                    var lessonLocal = localjson.ToObject<LessonLocal>();
+                    List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
+                    studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
+                    studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal.lessonRecord.id);
+                    studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+                    studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
+                    studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+                    studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+                    var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
+                    studentLessonDatas= pickupData.studentLessonDatas;
+                    var codeBools = LessonETLService.GetCodeBools(studentLessonDatas);
+                    string yearMonth = DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
+                    if (codeBools.FindAll(x => x.value).IsNotEmpty())
+                    {
+                        if (count==null) { count= new TechCount(); }
+                        count.lessonId=item.Split("\\").Last().Replace("-local.json", "");
+                        count.examCount= lessonLocal.examDatas.Count;
+                        count.taskCount= lessonLocal.taskDatas.Count;
+                        count.irsCount= lessonLocal.irsDatas.Count;
+                        count.interactNormalCount=count.irsCount;
+                        count.coworkCount= lessonLocal.coworkDatas.Count;
+                        count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
+                        count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
+
 
+                        if (lessonLocal.lessonRecord!=null)
+                        {
+
+                            count.yearMonth=yearMonth;
+
+                            if (lessonLocal?.lessonBase?.summary!=null)
+                            {
+                                count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
+                                count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
+                                count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
+                                count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
+                                count.examCountBase=lessonLocal.lessonBase.summary.examCount;
+                                count.interactNormalCountBase= count.interactNormalCount;
+                            }
+                            if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
+                            {
+                                count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
+                                count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
+                                count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
+                            }
+                            {
+
+                                count.pickup= pickupData.pickup;
+                                //  var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId)  &&  !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) &&  x.lessonId.Equals(lessonLocal.lessonRecord.id));
+                                int sumUpload = 0;
+                                int taskCount = 0;
+                                int maxUpload = 0;
+                                HashSet<string> pickUp = new HashSet<string>();
+                                foreach (var stu in studentLessonDatas)
+                                {
+                                    var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
+                                    if (countS.Count()>0)
+                                    {
+                                        int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
+                                        if (stuUploadmax> maxUpload)
+                                        {
+                                            maxUpload=stuUploadmax;
+                                        }
+                                    }
+                                    int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);
+
+                                    sumUpload+=stuUpload;
+                                    if (stu.taskRecord.itemRecords.Count()> taskCount)
+                                    {
+                                        taskCount=stu.taskRecord.itemRecords.Count();
+                                    }
+                                    var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
+                                    if (stu_scores!=null  && stu_scores.Count()>0)
+                                    {
+                                        count.stuCowork.AddRange(stu_scores);
+                                    }
+                                    var grp_scores = stu.group_coworkScore.Where(x => x>0);
+                                    if (grp_scores!=null  && grp_scores.Count()>0)
+                                    {
+                                        count.groupCowork.AddRange(grp_scores);
+                                    }
+                                    //if (stu.pickups.IsNotEmpty())
+                                    //{
+                                    //    foreach (var pickup in stu.pickups)
+                                    //    {
+                                    //        pickUp.Add(pickup);
+                                    //    }
+                                    //}
+                                }
+                                if (studentLessonDatas.Count>0&& taskCount>0  && maxUpload>0)
+                                {
+                                    var avgUpload = sumUpload*1.0/(studentLessonDatas.Count *taskCount);
+                                    count.upload.Add(new List<double>() { avgUpload, maxUpload });
+                                }
+                                //if (pickUp.Count>0)
+                                //{
+                                //    count.pickup.AddRange(pickUp.ToList());
+                                //}
+                                string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
+                                await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
+                                //if (force)
+                                //{
+
+                                //}
+                                //else
+                                //{
+                                //    if (lessonRecords.FindAll(x => x.id.Equals(lessonLocal.lessonRecord.id)).IsNotEmpty())
+                                //    {
+                                //        // await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json", studentLessonDatas.ToJsonString());
+                                //        //只有不是强制更新的时候再去刷新线上的json文件。
+
+                                //        await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
+                                //        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
+                                //    }
+                                //}
+                            }
+                        }
+                    }
+                    else
+                    {
+                        lessonLocal.lessonRecord.analysis=-1;
+                        if (lessonLocal.lessonRecord.scope.Equals("school"))
+                        {
+                            await azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).UpsertItemAsync(lessonLocal.lessonRecord, new PartitionKey(lessonLocal.lessonRecord.code));
+                        }
+                        else
+                        {
+                            await azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).UpsertItemAsync(lessonLocal.lessonRecord, new PartitionKey(lessonLocal.lessonRecord.code));
+                        }
+                        try
+                        {
+                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonth}\\{lessonLocal.lessonRecord!.id}-local.json");
+                        }
+                        catch (Exception ex) { }
+                        try
+                        {
+                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonth}\\{lessonLocal.lessonRecord!.id}-count.json");
+                        }
+                        catch (Exception ex) { }
+                    }
+                }
+
+                if (count!=null)
+                {
+                    return count;
+                }
+            }
+            return null; 
+        }
         public static async IAsyncEnumerable<TechCount> GetTeachCount(AzureCosmosFactory azureCosmos,List<LessonRecord> lessonRecords, List<string> filesLessons, string pathLessons, List<string> ignore, List<string> objectiveTypes, AzureStorageFactory azureStorage,bool force)
         {
             List<TechCount> techCounts = new List<TechCount>();
@@ -691,7 +891,52 @@ namespace HTEX.Lib.ETL.Lesson
             }
             return studentLessonDatas;
         }
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="lessonRecord"></param>
+        /// <param name="localIds"></param>
+        /// <param name="_azureStorage"></param>
+        /// <param name="pathLessons"></param>
+        /// <returns></returns>
+        public static async Task<LessonLocal> GetLessonLocal(LessonRecord lessonRecord, List<string> localIds, AzureStorageFactory _azureStorage, string pathLessons)
+        {
+            string scope = lessonRecord.scope;
+            string owner = lessonRecord.scope.Equals("school") ? lessonRecord.school : lessonRecord.tmdid;
+
+            string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(lessonRecord.startTime).ToString("yyyyMM");
+            LessonLocal lessonLocal = new LessonLocal { lessonRecord=lessonRecord };
+            if (System.IO.File.Exists($"{pathLessons}\\MM{yearMonthPath}\\{lessonRecord.id}-local.json"))
+            {
+                string jsonp = await System.IO.File.ReadAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{lessonRecord.id}-local.json");
+                lessonLocal = jsonp.ToObject<LessonLocal>();
+            }
+            else
+            {
+                List<string> files = new List<string>()
+                    {
+                        $"/records/{lessonRecord.id}/IES/TimeLine.json",
+                        $"/records/{lessonRecord.id}/IES/base.json",
+                        $"/records/{lessonRecord.id}/IES/Task.json",
+                        $"/records/{lessonRecord.id}/IES/SmartRating.json",
+                        $"/records/{lessonRecord.id}/IES/IRS.json",
+                        $"/records/{lessonRecord.id}/IES/Cowork.json",
+                        $"/records/{lessonRecord.id}/Sokrates/SokratesRecords.json",
+                    };
+                lessonLocal  = new LessonLocal { lessonRecord=lessonRecord };
+                lessonLocal =  await GetLessonFiles(lessonLocal, files, owner, _azureStorage);
+            }
 
+            if (lessonLocal.lessonBase!=null  &&  lessonLocal.lessonBase.student!=null)
+            {
+                var baseData = GetBaseData(lessonLocal.lessonBase!);
+                lessonLocal.studentLessonDatas= baseData.studentLessonDatas;
+                List<ExamData> examDatas = await GetExamInfo(lessonRecord, lessonLocal.timeLineData, _azureStorage, owner);
+                lessonLocal.examDatas = examDatas;
+                lessonLocal.sokratesDatas= lessonLocal.sokratesDatas.IsNotEmpty() ? lessonLocal.sokratesDatas : lessonLocal.timeLineData!=null ? lessonLocal.timeLineData.events : new List<TimeLineEvent>();
+            }
+            return lessonLocal;
+        }
         public static async IAsyncEnumerable<LessonLocal> GetLessonLocal(List<LessonRecord> lessonRecords, List<string> localIds, AzureStorageFactory _azureStorage, string pathLessons)
         {
             foreach (var lessonRecord in lessonRecords)
@@ -2097,7 +2342,7 @@ namespace HTEX.Lib.ETL.Lesson
         /// <param name="studentLessonDatas"></param>
         /// <param name="lessonDataAnalysis"></param>
         /// <returns></returns>
-        public static List<StudentLessonItem>   ProcessStudentDataV2(List<StudentLessonData> studentLessonDatas, LessonDataAnalysisCluster lessonDataAnalysis, List<CodeBool> codeBools)
+        public static List<StudentLessonItem>   ProcessStudentDataV2(List<StudentLessonData> studentLessonDatas, LessonDataAnalysisModel lessonDataAnalysis, List<CodeBool> codeBools)
         {
 
             //历史记录的个人计分集合，通过“2倍标准差规则”移除异常值后得到的集合
@@ -2421,7 +2666,7 @@ namespace HTEX.Lib.ETL.Lesson
                     double n = studentLessonData.taskRecord.itemRecords.Count()*1.0;
                     if (n>0)
                     {
-                        double max_m = lessonDataAnalysis.task.Max();
+                        double max_m = lessonDataAnalysis.task.Select(x=>x.Key).Max();
                         double w = studentLessonData.taskRecord.itemRecords.Where(x => x.resultWeight>0).Count()*1.0;
                         double y = (10 *w/n+(j/t) *w)/max_m;
                         double l = max_m*(w*w/n+(j/t) * w)/n;
@@ -2741,6 +2986,29 @@ namespace HTEX.Lib.ETL.Lesson
             }
             return property.Name;
         }
+        /// <summary>
+        /// 当前数超越集合的百分比
+        /// </summary>
+        /// <param name="nums"></param>
+        /// <param name="curr"></param>
+        /// <returns></returns>
+        public static (double persent, int count) GetPersent(List<KeyValuePair<double, int>>  nums, double curr)
+        {
+            int count = 0;
+            foreach (var op in nums.Select(x=>x.Key).OrderBy(x => x))
+            {
+                if (op <= curr)
+                {
+                    count+=nums.Find(x=>x.Key==op).Value;
+                }
+                else
+                {
+                    break;
+                }
+            }
+            return (count *1.0/ nums.Select(x=>x.Value).Sum() * 100, count);
+        }
+
         /// <summary>
         /// 当前数超越集合的百分比
         /// </summary>
@@ -2931,6 +3199,66 @@ namespace HTEX.Lib.ETL.Lesson
         /// </summary>
         public double xx_cy { get; set; } = 0;
     }
+    /// <summary>
+    /// 
+    /// </summary>
+    public class LessonDataAnalysisModel {
+        /// <summary>
+        /// 任务模型
+        /// </summary>
+        public List<KeyValuePair<double, int>> task { get; set;}= new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// irs模型
+        /// </summary>
+        public List<KeyValuePair<double, int>>  irs { get; set; } = new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// 互动模型
+        /// </summary>
+        public List<KeyValuePair<double, int>> interactNormal { get; set; } = new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// 个人计分模型
+        /// </summary>
+        public List<KeyValuePair<double, int>> pscore { get; set; } = new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// 小组计分模型
+        /// </summary>
+        public List<KeyValuePair<double, int>> gscore { get; set; } = new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// 互动计分模型
+        /// </summary>
+        public List<KeyValuePair<double, int>> tscore { get; set; } = new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// 学生个人协作模型
+        /// </summary>
+        public List<KeyValuePair<double, int>> stuCowork { get; set; } = new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// 学生小组协作模型
+        /// </summary>
+        public List<KeyValuePair<double, int>> groupCowork { get; set; } = new List<KeyValuePair<double, int>>();
+        /// <summary>
+        /// 互动次数及格线
+        /// </summary>
+        public double interactPass { get; set; }
+        /// <summary>
+        /// 互动次数良好线
+        /// </summary>
+
+        public double interactGood { get; set; }
+        /// <summary>
+        /// 互动次数差线
+        /// </summary>
+        public double interactLow { get; set; }
+        /// <summary>
+        /// 互动次数中线
+        /// </summary>
+        public double interactMedium { get; set; }
+        /// <summary>
+        /// 互动次数高线
+        /// </summary>
+        public double interactHigh { get; set; }
+        public List<KeyValuePair<double, List<double>>> levelInteract { get; set; } = new List<KeyValuePair<double, List<double>>>();
+    }
+
     /// <summary>
     /// 历史课例的关键数据模型
     /// </summary>

TEAMModelOS.Extension/HTEX.Test/Controllers/MockDataController.cs

@@ -457,9 +457,9 @@ namespace HTEX.Test.Controllers
             }
            
             #endregion 数据模拟
-            string jsons = await System.IO.File.ReadAllTextAsync("F:\\lesson-local\\analysis\\analysis.json");
+            string jsons = await System.IO.File.ReadAllTextAsync("F:\\lesson-local\\analysis\\analysis-model.json");
             long time = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
-            LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
+            LessonDataAnalysisModel lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisModel>();
             var codeBools = LessonETLService.GetCodeBools(students);
             var studentLessons = LessonETLService. ProcessStudentDataV2(students, lessonDataAnalysis, codeBools);
             var excleFile = $"F:\\mock-data\\{time}.xlsx";

TEAMModelOS.Extension/HTEX.Test/Program.cs

@@ -11,6 +11,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using TEAMModelOS.SDK;
 using TEAMModelOS.SDK.DI;
@@ -26,10 +27,10 @@ namespace HTEX.Test
     {
         public static async Task Main(string[] args)
         {
-            //Test2();
+            Test2();
 
            
-           await MockDataController.MockData();
+           //await MockDataController.MockData();
             //var builder = WebApplication.CreateBuilder(args);
 
             //// Add services to the container.
@@ -55,218 +56,33 @@ namespace HTEX.Test
 
             //app.Run();
         }
-        static double CalculateScore(int n, double k = 1)
-        {
-            if (n == 0)
-            {
-                return 0;
-            }
-            else
-            {
-                double score = 60 + (40 / (1 + Math.Exp(-k * (n - 1))));
-                return Math.Max(80, Math.Min(score, 100));
-            }
-        }
+        
 
-        public static string Test2() 
+        public static async Task<string> Test2() 
         {
             string? pathAnalysis = $"F:\\lesson-local\\analysis";
-            var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
-            List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
-            LessonDataAnalysisCluster lessonDataAnalysisCluster = new LessonDataAnalysisCluster();
-            foreach (var file in filesAnalysis)
-            {
-                //读取每月的数据
-                if (file.EndsWith("-m-analysis.json"))
-                {
-                    string jsons =   System.IO.File.ReadAllText(file);
-                    LessonDataAnalysisMonth lessonDataAnalysisMonth = jsons.ToObject<LessonDataAnalysisMonth>();
-
-                    lessonDataAnalysisMonths.Add(lessonDataAnalysisMonth);
-                    if (lessonDataAnalysisMonth.task.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.task.AddRange(lessonDataAnalysisMonth.task);
-                    }
-                    if (lessonDataAnalysisMonth.irs.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.irs.AddRange(lessonDataAnalysisMonth.irs);
-                    }
-                    if (lessonDataAnalysisMonth.interactNormal.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.interactNormal.AddRange(lessonDataAnalysisMonth.interactNormal);
-                    }
-                    if (lessonDataAnalysisMonth.pscore.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.pscore.AddRange(lessonDataAnalysisMonth.pscore);
-                    }
-                    if (lessonDataAnalysisMonth.gscore.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.gscore.AddRange(lessonDataAnalysisMonth.gscore);
-                    }
-                    if (lessonDataAnalysisMonth.tscore.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.tscore.AddRange(lessonDataAnalysisMonth.tscore);
-                    }
-                    if (lessonDataAnalysisMonth.stuCowork.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.stuCowork.AddRange(lessonDataAnalysisMonth.stuCowork);
-                    }
-                    if (lessonDataAnalysisMonth.groupCowork.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.groupCowork.AddRange(lessonDataAnalysisMonth.groupCowork);
-                    }
-                }
-            }
-          
-            //标准差偏差N倍，视为异常数据
-            double thresholdMultiplier =2;
-            lessonDataAnalysisCluster.task= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.task.OrderBy(x => x), thresholdMultiplier);
-            lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
-            lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
-            lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
-            lessonDataAnalysisCluster.irs = LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.irs.OrderBy(x => x), thresholdMultiplier);
-            lessonDataAnalysisCluster.interactNormal=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.interactNormal.OrderBy(x => x), thresholdMultiplier);
-            lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x => x), thresholdMultiplier);
-            lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
-             //超过60 80的
-            var all = lessonDataAnalysisCluster.interactNormal.Select(x =>  x).OrderBy(x => x).ToArray();
-            double n = all.Max()+1;
-            var clusterInteract = HTEX.Lib.ETL. KMeansService.KMeansOptimized(all,3);
-            //foreach (var item in clusterInteract.OrderBy(x => x.Key))
-            //{
-            //    lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
-            //    Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
-            //}
+            try {
+                string jsons = await System.IO.File.ReadAllTextAsync($"F:\\lesson-local\\analysis\\analysis-model.json");
+                var s = JsonSerializer.Deserialize<LessonDataAnalysisModel>(jsons);
+                LessonDataAnalysisModel lessonDataAnalysis =JsonDocument.Parse(jsons).RootElement.ToObject<LessonDataAnalysisModel>();
+                var per = LessonETLService.GetPersent(lessonDataAnalysis.irs, 2);
 
-           //IEnumerable<double> all = lessonDataAnalysisCluster.clustersInteract.SelectMany(x => x.Value);
-            int pass = 0;
-            for (var i = 1; i<n; i++) 
-            {
-                var p = LessonETLService.GetPersent(all, i);
-                if (p.persent>=60) 
-                {
-                    pass = i;
-                    break;
-                }
             }
-            var low = all.Where(x => x<pass).Average();
-            int good = 0;
-            for (var i = 1; i<n; i++)
-            {
-                var p = LessonETLService.GetPersent(all, i);
-                if (p.persent>=80)
-                {
-                    good = i;
-                    break;
-                }
+            catch (Exception ex) {
+                Console.WriteLine(ex.ToString());
             }
-            var medium = all.Where(x => x>=pass  &&x<good ).Average();
-            var high = all.Where(x => x>=good).Average();
-            lessonDataAnalysisCluster.interactGood=good;
-            lessonDataAnalysisCluster.interactPass=pass;
-            lessonDataAnalysisCluster.interactHigh=high;
-            lessonDataAnalysisCluster.interactMedium=medium;
-            lessonDataAnalysisCluster.interactLow=low;
-            List<KeyValuePair<double, List<double>>> levelInteract = new List<KeyValuePair<double, List<double>>>()
-            {
-                new KeyValuePair<double, List<double>>(low, all.Where(x => x<pass).ToList()),
-                new KeyValuePair<double, List<double>>(medium, all.Where(x => x>=pass  &&x<good).ToList() ),
-                new KeyValuePair<double, List<double>>(high, all.Where(x => x>=good).ToList())
-            };
-            lessonDataAnalysisCluster.levelInteract= levelInteract;
-            double ss = 0;
-            for (var i = 1; i<n; i++) 
-            {
-               // KeyValuePair<double, List<int>> curr = new KeyValuePair<double, List<int>>();
-                var s = lessonDataAnalysisCluster.levelInteract.FindAll(x => x.Value.Min()<=i  && x.Value.Max()>=i).MinBy(x=>x.Key);
-                var p = LessonETLService.GetPersent(all, i);
-                var l= i<pass?pass:i<good?pass:good;
-                var e = (i*1.0/l) *(p.persent)  *  (s.Value.Count*1.0/all.Count());
-                ss+= e;
-                Console.WriteLine($"n: {i},l: {l}, persent: {p.persent},count: {p.count},s-min: {s.Value.Min()},s-max: {s.Value.Max()},value: {e}");
-            }
-            //foreach (var s in clusterInteract.OrderBy(x => x.Key))
-            //{
-            //    clustersDataInteract.Add(s);
-            //}
-            //lessonDataAnalysisCluster.clustersInteract= clustersDataInteract;
-            System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
             return "";
         }
 
-        public static string Test()
-        {
-            
-            // 创建一个随机数生成器
-            Random random = new Random();
-
-            // 示例数据集
-            var data = Enumerable.Range(1, 25000).Select(i => Vector<double>.Build.Dense(i % 10, i % 10)).ToArray();
-          
-            return "Hello World!";
-        }
+        
          
-    static bool[] MarkAnomalies(List<int> array)
-        {
-            if (array.Count == 0) return new bool[0];
-
-            double average = array.Average();
-            double variance = array.Average(x => Math.Pow(x - average, 2));
-            double standardDeviation = Math.Sqrt(variance);
-
-            // 定义异常值的阈值，这里使用2倍标准差
-            double threshold =3* standardDeviation;
-
-            bool[] anomalies = new bool[array.Count];
-            for (int i = 0; i < array.Count; i++)
-            {
-                double deviation = Math.Abs(array[i] - average);
-                anomalies[i] = deviation > threshold;
-            }
-
-            return anomalies;
-        }
+     
 
        
        
         
-        /// <summary>
-        /// 计算当前元素在集合中超过了多少百分比的值
-        /// </summary>
-        /// <param name="nums"></param>
-        /// <param name="curr"></param>
-        /// <returns></returns>
-        public static double GetPersent(IEnumerable<double> nums, int curr)
-        {
-            int count = 0;
-            foreach (var op in nums.OrderBy(x => x))
-            {
-                if (op < curr)
-                {
-                    count++;
-                }
-                else if (op == curr)
-                {
-                    count++;
-                }
-                else
-                {
-                    break;
-                }
-            }
-            return count *1.0/ nums.Count() * 100;
-        }
 
 
-        public class KeyCount
-        {
-            public int count { get; set;}
-            public int key { get; set;}
-        }
-        class WeightedItem
-        {
-            public int Value { get; set; }
-            public double Weight { get; set; }
-        }
+         
     }
 } 

TEAMModelOS.Extension/HTEX.Test/x.xml

@@ -4,14 +4,6 @@
         <name>HTEX.Test</name>
     </assembly>
     <members>
-        <member name="M:HTEX.Test.Program.GetPersent(System.Collections.Generic.IEnumerable{System.Double},System.Int32)">
-            <summary>
-            计算当前元素在集合中超过了多少百分比的值
-            </summary>
-            <param name="nums"></param>
-            <param name="curr"></param>
-            <returns></returns>
-        </member>
         <member name="M:HTEX.Test.Service.MLService.KMeans(System.Single[],System.Int32)">
             <summary>
             

TEAMModelOS.Function/IESServiceBusTrigger.cs

@@ -1773,9 +1773,20 @@ namespace TEAMModelOS.Function
                                                 List<OverallEducation> overallEducations = new List<OverallEducation>();
                                                 List<Student> studentsBase = new List<Student>();
                                                 List<StudentSemesterRecord> students = new List<StudentSemesterRecord>();
-                                                LessonDataAnalysisCluster lessonDataAnalysis = null;
+                                                LessonDataAnalysisModel lessonDataAnalysis = null;
                                                 await LessonETLService.DoStudentLessonData(Constant.objectiveTypes, _azureStorage, lessonLocal, _dingDing, client, location,_azureRedis, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
-                                                
+                                                await Parallel.ForEachAsync(studentSemesterRecords, async (studentSemester, cancellationToken) =>
+                                                {
+                                                    await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
+
+                                                });
+                                                await Parallel.ForEachAsync(overallEducations, async (overallEducation, cancellationToken) =>
+                                                {
+                                                    await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code));
+                                                    string key = $"OverallEducation:{overallEducation.schoolCode}:{overallEducation.periodId}:{overallEducation.year}:{overallEducation.semesterId}:{overallEducation?.classId}";
+                                                    await _azureRedis.GetRedisClient(8).HashSetAsync(key, overallEducation.studentId, overallEducation.ToJsonString());
+                                                    await _azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
+                                                });
                                                 // 使用当前文化设置的日历
                                                 CultureInfo cultureInfo = CultureInfo.CurrentCulture;
                                                 Calendar calendar = cultureInfo.Calendar;

TEAMModelOS.Function/TEAMModelOS.Function.csproj

@@ -5,9 +5,9 @@
     <OutputType>Exe</OutputType>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-	<Version>5.2411.6</Version>
-	<AssemblyVersion>5.2411.6.1</AssemblyVersion>
-	<FileVersion>5.2411.6.1</FileVersion>
+	<Version>5.2411.13</Version>
+	<AssemblyVersion>5.2411.13.1</AssemblyVersion>
+	<FileVersion>5.2411.13.1</FileVersion>
 	<PackageId>TEAMModelOS.FunctionV4</PackageId>
 	<Authors>teammodel</Authors>
 	<Company>醍摩豆（成都）信息技术有限公司</Company>

TEAMModelOS.SDK/TEAMModelOS.SDK.csproj

@@ -1,9 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
 		<TargetFramework>net8.0</TargetFramework>
-		<Version>5.2411.6</Version>
-		<AssemblyVersion>5.2411.6.1</AssemblyVersion>
-		<FileVersion>5.2411.6.1</FileVersion>
+		<Version>5.2411.13</Version>
+		<AssemblyVersion>5.2411.13.1</AssemblyVersion>
+		<FileVersion>5.2411.13.1</FileVersion>
 		<PackageReleaseNotes>发版</PackageReleaseNotes>
 	</PropertyGroup>
 

TEAMModelOS/TEAMModelOS.csproj

@@ -79,11 +79,11 @@
 		<SpaRoot>ClientApp\</SpaRoot>
 		<DefaultItemExcludes>$(DefaultItemExcludes);$(SpaRoot)node_modules\**</DefaultItemExcludes>
 		<UserSecretsId>078b5d89-7d90-4f6a-88fc-7d96025990a8</UserSecretsId>
-		<Version>5.2411.6</Version>
-		<AssemblyVersion>5.2411.6.1</AssemblyVersion>
-		<FileVersion>5.2411.6.1</FileVersion>
+		<Version>5.2411.13</Version>
+		<AssemblyVersion>5.2411.13.1</AssemblyVersion>
+		<FileVersion>5.2411.13.1</FileVersion>
 		<Description>TEAMModelOS(IES5)</Description>
-		<PackageReleaseNotes>IES版本说明版本切换标记5.2411.6.1</PackageReleaseNotes>
+		<PackageReleaseNotes>IES版本说明版本切换标记5.2411.13.1</PackageReleaseNotes>
 		<PackageId>TEAMModelOS</PackageId>
 		<Authors>teammodel</Authors>
 		<Company>醍摩豆（成都）信息技术有限公司</Company>

TEAMModelOS/appsettings.Development.json

@@ -18,7 +18,7 @@
     "IdTokenSalt": "8263692E2213497BB55E74792B7900B4",
     "HttpTrigger": "https://teammodelosfunction-test.chinacloudsites.cn/api/",
     //"HttpTrigger": "http://localhost:7071/api/"
-    "Version": "5.2411.6.1"
+    "Version": "5.2411.13.1"
   },
   "Azure": {
     // 测试站数据库

TEAMModelOS/appsettings.json

@@ -18,7 +18,7 @@
     "Exp": 86400,
     "IdTokenSalt": "8263692E2213497BB55E74792B7900B4",
     "HttpTrigger": "https://teammodelosfunction.chinacloudsites.cn/api/",
-    "Version": "5.2411.6.1"
+    "Version": "5.2411.13.1"
   },
   "Azure": {
     "Storage": {