update

HTEX.DataETL/Controllers/LessonRecordController.cs

@@ -135,6 +135,9 @@ namespace HTEX.DataETL.Controllers
                     studentLessonDatas = LessonETLService.GetExamData(lessonBase, timeLineData, examDatas, studentLessonDatas, Constant.objectiveTypes, id);
                     studentLessonDatas = LessonETLService.GetSmartRatingData(lessonBase, timeLineData, smartRatingDatas, studentLessonDatas, id);
                     studentLessonDatas = LessonETLService.GetTaskData(lessonBase, timeLineData, taskDatas, studentLessonDatas,id);
+                    var pickupData  = LessonETLService.GetPickupData(lessonBase, timeLineData, studentLessonDatas, id);
+                    studentLessonDatas= pickupData.studentLessonDatas;
+
                     await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString());
                     string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis.json");
                     LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
@@ -246,7 +249,12 @@ namespace HTEX.DataETL.Controllers
             }
             HashSet<string> yearMonth = new HashSet<string>();
             long newest = 0;
-            if (loadLocal ||(json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
+            bool force = false;
+            if ((json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
+            {
+                force= _force.GetBoolean();
+            }
+             // if (loadLocal ||force)
             {
                 List<LessonRecord> lessonRecords = new List<LessonRecord>();
                 var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
@@ -256,7 +264,8 @@ namespace HTEX.DataETL.Controllers
                     newest= resultSchool.list.Max(x => x.startTime);
                     lessonRecords.AddRange(resultSchool.list);
                 }
-                else {
+                else
+                {
                     newest=stime;
                 }
                 var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
@@ -273,7 +282,7 @@ namespace HTEX.DataETL.Controllers
                 List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
                 if (lessonRecords.IsNotEmpty())
                 {
-                  
+
                     await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
                     {
                         string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
@@ -302,231 +311,15 @@ namespace HTEX.DataETL.Controllers
                 }
                 List<TechCount> techCounts = new List<TechCount>();
                 filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
-                await foreach (var item in  GetTeachCount(lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage))
+                await foreach (var item in LessonETLService.GetTeachCount(lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force))
                 {
                     techCounts.Add(item);
                 }
-                var yearMonthDatas = techCounts.GroupBy(x => x.yearMonth).Select(x => new { key = x.Key, list = x.ToList() });
-               // lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
-                LessonDataAnalysisCluster lessonDataAnalysisCluster = new LessonDataAnalysisCluster();
-                foreach (var yearMonthData in yearMonthDatas) 
-                {
-                    if (string.IsNullOrWhiteSpace(yearMonthData.key)) 
-                    {
-                        Console.WriteLine(yearMonthData.list.Select(x => x.lessonId).ToJsonString());
-                    }
-                    LessonDataAnalysisMonth lessonDataAnalysisMonth= new LessonDataAnalysisMonth() { updateTime= newest, yearMonth= yearMonthData.key };
-                    lessonDataAnalysisMonth.pscore  = yearMonthData.list.SelectMany(x => x.pscore).ToList();
-                    lessonDataAnalysisMonth.tscore  = yearMonthData.list.SelectMany(x => x.tscore).ToList();
-                    lessonDataAnalysisMonth.gscore  = yearMonthData.list.SelectMany(x => x.gscore).ToList();
-                    lessonDataAnalysisMonth.irs= yearMonthData.list.Where(x => x.irsCount>0).Select(x =>(double)x.irsCount).ToList();
-                    lessonDataAnalysisMonth.interactNormal= yearMonthData.list.Where(x => x.interactNormalCount > 0).Select(x => (double)x.interactNormalCount).ToList();
-                    lessonDataAnalysisMonth.task = yearMonthData.list.Where(x => x.taskCount > 0).Select(x => (double)x.taskCount).ToList();
-                    lessonDataAnalysisMonth.stuCowork=yearMonthData.list.SelectMany(x => x.stuCowork).ToList();
-                    lessonDataAnalysisMonth.groupCowork=yearMonthData.list.SelectMany(x => x.groupCowork).ToList();
-                    System.IO.File.WriteAllText(Path.Combine(pathAnalysis, $"{yearMonthData.key}-m-analysis.json"), lessonDataAnalysisMonth.ToJsonString());
-                    // lessonDataAnalysisMonths.Add( lessonDataAnalysisMonth);
-                    if (lessonDataAnalysisMonth.task.IsNotEmpty()) 
-                    {
-                        lessonDataAnalysisCluster.task.AddRange(lessonDataAnalysisMonth.task);
-                    }
-                    if (lessonDataAnalysisMonth.irs.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.irs.AddRange(lessonDataAnalysisMonth.irs);
-                    }
-                    if (lessonDataAnalysisMonth.interactNormal.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.interactNormal.AddRange(lessonDataAnalysisMonth.interactNormal);
-                    }
-                    if (lessonDataAnalysisMonth.pscore.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.pscore.AddRange(lessonDataAnalysisMonth.pscore);
-                    }
-                    if (lessonDataAnalysisMonth.gscore.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.gscore.AddRange(lessonDataAnalysisMonth.gscore);
-                    }
-                    if (lessonDataAnalysisMonth.tscore.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.tscore.AddRange(lessonDataAnalysisMonth.tscore);
-                    }
-                    if (lessonDataAnalysisMonth.stuCowork.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.stuCowork.AddRange(lessonDataAnalysisMonth.stuCowork);
-                    }
-                    if (lessonDataAnalysisMonth.groupCowork.IsNotEmpty())
-                    {
-                        lessonDataAnalysisCluster.groupCowork.AddRange(lessonDataAnalysisMonth.groupCowork);
-                    }
-                }
-
-                //标准差偏差N倍，视为异常数据
-                int thresholdMultiplier = 2;
-                lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
-                lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
-                lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
-                lessonDataAnalysisCluster.irs = LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.irs.OrderBy(x => x), thresholdMultiplier);
-                lessonDataAnalysisCluster.interactNormal=LessonETLService. CleanDataBySDThreshold(lessonDataAnalysisCluster.interactNormal.OrderBy(x => x), thresholdMultiplier);
-                lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x=>x), thresholdMultiplier);
-                lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
-                var d = lessonDataAnalysisCluster.interactNormal.Select(x => x).OrderBy(x => x).ToArray();
-                //  设置0 为自动规划聚类，11 则为自动规划后得到的数字。
-                var clusterInteract = KMeansService.KMeansOptimized(d,11,10);
-                //foreach (var item in clusterInteract)
-                //{
-                //    Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()}");
-                //}
-                foreach (var item in clusterInteract.OrderBy(x => x.Key))
-                {
-                    lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
-                    //Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
-                }
-                System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
+                LessonETLService. GenAnalysisData(pathAnalysis, newest, techCounts);
             }
             return Ok(new { yearMonth });
         }
 
-        private static async IAsyncEnumerable<TechCount> GetTeachCount(List<LessonRecord> lessonRecords,List<string> filesLessons, string pathLessons,  List<string> ignore, List<string> objectiveTypes,AzureStorageFactory azureStorage)
-        {
-            foreach (var item in filesLessons) 
-            {
-                if (item.EndsWith("-local.json"))
-                {
-                    TechCount count = null;
-
-                    string lessonId = item.Split("\\").Last().Replace("-local.json", "");
-                    string countFile = item.Replace("-local.json", "-count.json");
-                    if (System.IO.File.Exists(countFile))
-                    {
-                        if (System.IO.File.Exists(countFile))
-                        {
-                            string contjson = await System.IO.File.ReadAllTextAsync(countFile);
-                            count = contjson.ToObject<TechCount>();
-                        }
-                    }
-                    if (string.IsNullOrWhiteSpace(count?.yearMonth))
-                    {
-                        string localjson = await System.IO.File.ReadAllTextAsync(item);
-                        var lessonLocal = localjson.ToObject<LessonLocal>();
-                        count = new TechCount();
-                        count.lessonId=item.Split("\\").Last().Replace("-local.json", "");
-                        count.examCount= lessonLocal.examDatas.Count;
-                        count.taskCount= lessonLocal.taskDatas.Count;
-                        count.irsCount= lessonLocal.irsDatas.Count;
-                        count.interactNormalCount=count.irsCount;
-                        count.coworkCount= lessonLocal.coworkDatas.Count;
-                        count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
-                        count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
-                        
-                        
-                        if (lessonLocal.lessonRecord!=null)
-                        {
-
-                            count.yearMonth=DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
-                           
-                            if (lessonLocal?.lessonBase?.summary!=null)
-                            {
-                                count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
-                                count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
-                                count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
-                                count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
-                                count.examCountBase=lessonLocal.lessonBase.summary.examCount;
-                                count.interactNormalCountBase= count.interactNormalCount;
-                            }
-                            if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
-                            {
-                                count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
-                                count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
-                                count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
-                            }
-                            ///处理学生数据
-                            {
-                                List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
-                                studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
-                                studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal.lessonRecord.id);
-                                studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-                                studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
-                                studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-                                studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-                                //  var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId)  &&  !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) &&  x.lessonId.Equals(lessonLocal.lessonRecord.id));
-                                int sumUpload = 0;
-                                int taskCount = 0;
-                                int maxUpload = 0;
-                                //HashSet<string> pickUp = new HashSet<string>();
-                                foreach (var stu in studentLessonDatas)
-                                {
-                                    var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
-                                    if (countS.Count()>0)
-                                    {
-                                        int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
-                                        if (stuUploadmax> maxUpload)
-                                        {
-                                            maxUpload=stuUploadmax;
-                                        }
-                                    }
-                                    int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);
-
-                                    sumUpload+=stuUpload;
-                                    if (stu.taskRecord.itemRecords.Count()> taskCount)
-                                    {
-                                        taskCount=stu.taskRecord.itemRecords.Count();
-                                    }
-                                    var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
-                                    if (stu_scores!=null  && stu_scores.Count()>0)
-                                    {
-                                        count.stuCowork.AddRange(stu_scores);
-                                    }
-                                    var grp_scores = stu.group_coworkScore.Where(x => x>0);
-                                    if (grp_scores!=null  && grp_scores.Count()>0)
-                                    {
-                                        count.groupCowork.AddRange(grp_scores);
-                                    }
-                                    //if (stu.pickups.IsNotEmpty())
-                                    //{
-                                    //    foreach (var pickup in stu.pickups)
-                                    //    {
-                                    //        pickUp.Add(pickup);
-                                    //    }
-                                    //}
-                                }
-                                if (studentLessonDatas.Count>0&& taskCount>0  && maxUpload>0)
-                                {
-                                    var avgUpload = sumUpload*1.0/(studentLessonDatas.Count *taskCount);
-                                    count.upload.Add(new List<double>() { avgUpload, maxUpload });
-                                }
-                                //if (pickUp.Count>0)
-                                //{
-                                //    count.pickup.Add(pickUp.ToList());
-                                //}
-                                string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
-                                // if (!azureStorage.GetBlobContainerClient(owner).GetBlobClient($"records/{lessonLocal.lessonRecord.id}/student-analysis.json").Exists())
-                                {
-
-
-                                }
-                                // if (!System.IO.File.Exists($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json"))
-                                {
-
-                                }
-                                if (lessonRecords.FindAll(x => x.id.Equals(lessonLocal.lessonRecord.id)).IsNotEmpty())
-                                {
-                                    await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json", studentLessonDatas.ToJsonString());
-                                    await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
-                                    await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
-                                }
-                            }
-                        }
-                    }
-
-                    
-                
-                    if (count!=null) 
-                    {
-                        yield return count;
-                    }
-                }
-            }
-           
-        }
+       
     }
 }

TEAMModelOS.Extension/HTEX.Lib/summary.xml

@@ -8205,6 +8205,19 @@
             <param name="lessonBase"></param>
             <returns></returns>
         </member>
+        <member name="M:HTEX.Lib.ETL.Lesson.LessonETLService.GetPickupData(TEAMModelOS.SDK.Models.LessonBase,TEAMModelOS.SDK.Models.TimeLineData,System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData},System.String)">
+             <summary>
+            单独处理挑人的逻辑
+            是否从小组里面挑人。
+            不需要去重页面，直接获取挑人大类 PickupResult
+            小类处理：PickupRight  , PickupOption  , PickupNthGrp  ,PickupEachGrp  ,PickupDiff   ， PickupResult   挑人算不算互动？？  读取PickupMemberId "[\r\n  35\r\n]"
+             </summary>
+             <param name="lessonBase"></param>
+             <param name="timeLineData"></param>
+             <param name="studentLessonDatas"></param>
+             <param name="lessonId"></param>
+             <returns></returns>
+        </member>
         <member name="M:HTEX.Lib.ETL.Lesson.LessonETLService.GetIRSData(TEAMModelOS.SDK.Models.LessonBase,TEAMModelOS.SDK.Models.TimeLineData,System.Collections.Generic.List{TEAMModelOS.SDK.Models.IRSData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.ExamData},System.String)">
              <summary>
             读取互动信息
@@ -8282,14 +8295,6 @@
             </summary>
             <returns></returns>
         </member>
-        <member name="M:HTEX.Lib.ETL.Lesson.LessonETLService.ProcessStudentDataV1(System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData},HTEX.Lib.ETL.Lesson.LessonDataAnalysisCluster)">
-            <summary>
-            计算学生的学习成效，学习态度，合作能力，协作能力，评价能力
-            </summary>
-            <param name="studentLessonDatas"></param>
-            <param name="lessonDataAnalysis"></param>
-            <returns></returns>
-        </member>
         <member name="M:HTEX.Lib.ETL.Lesson.LessonETLService.CleanDataBySDThreshold(System.Collections.Generic.IEnumerable{System.Double},System.Double)">
             <summary>
             使用标准差定义异常值。如果一个数字与平均值的偏差超过某个标准差倍数（例如2倍或3倍），则可以认为它是异常的。
@@ -8350,7 +8355,7 @@
         </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.hd_cy">
             <summary>
-            互动参与指数
+            互动专注指数
             </summary>
         </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.pc_df">
@@ -8373,6 +8378,11 @@
             任务参与次数
             </summary>
         </member>
+        <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.rw_zpc">
+            <summary>
+            任务作品数
+            </summary>
+        </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.rw_cx">
             <summary>
             任务成效指数
@@ -8380,7 +8390,7 @@
         </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.rw_cy">
             <summary>
-            任务参与指数
+            任务专注指数
             </summary>
         </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.pj_cs">
@@ -8460,7 +8470,32 @@
         </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.xz_cy">
             <summary>
-            协作参与指数
+            协作专注指数
+            </summary>
+        </member>
+        <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.xz_tr">
+            <summary>
+            小组挑人
+            </summary>
+        </member>
+        <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.xz_jf">
+            <summary>
+            小组计分
+            </summary>
+        </member>
+        <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.xz_xz">
+            <summary>
+            小组协作成果分
+            </summary>
+        </member>
+        <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.xz_sc">
+            <summary>
+            组任务上传数
+            </summary>
+        </member>
+        <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.hz_nl">
+            <summary>
+            合作能力
             </summary>
         </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.xx_cx">
@@ -8470,7 +8505,7 @@
         </member>
         <member name="P:HTEX.Lib.ETL.Lesson.StudentLessonItem.xx_cy">
             <summary>
-            学习参与
+            学习专注度
             </summary>
         </member>
         <member name="T:HTEX.Lib.ETL.Lesson.LessonDataAnalysisCluster">
@@ -8478,7 +8513,7 @@
             历史课例的关键数据模型
             </summary>
         </member>
-        <member name="P:HTEX.Lib.ETL.Lesson.LessonDataAnalysisCluster.clustersInteract">
+        <member name="P:HTEX.Lib.ETL.Lesson.LessonDataAnalysisCluster.levelInteract">
             <summary>
             
             </summary>
@@ -8533,6 +8568,11 @@
             小组协作成果数
             </summary>
         </member>
+        <member name="P:HTEX.Lib.ETL.Lesson.LessonDataAnalysisBase.pickup">
+            <summary>
+            挑人集合
+            </summary>
+        </member>
         <member name="T:HTEX.Lib.ETL.Lesson.LessonDataAnalysisMonth">
             <summary>
             每月的课例模型数据

TEAMModelOS.Extension/HTEX.Test/Controllers/MockDataController.cs

@@ -10,6 +10,8 @@ using System.Xml;
 using System.Text.RegularExpressions;
 using HTEX.Lib.ETL.Lesson;
 using OfficeOpenXml.Style;
+using DocumentFormat.OpenXml.Spreadsheet;
+using OpenXmlPowerTools;
 
 namespace HTEX.Test.Controllers
 {
@@ -223,7 +225,9 @@ namespace HTEX.Test.Controllers
                         case bool when w==1:
                             item.resultWeight= InteractWeight.TT;
                             item.resultType= InteractReultType.TT;
-                            item.itemScore=10;
+                            item.optCount=  Random.Shared.Next(0, 4);
+                            item.itemScore=item.optCount*10;
+                            item.isGroup= Random.Shared.Next(0, 6)%2==3;
                             break;
                     }
                     student.taskRecord.taskRate= student.taskRecord.itemRecords.Where(x => x.resultWeight>0).Count()*1.0/tcount;
@@ -289,30 +293,53 @@ namespace HTEX.Test.Controllers
                 }
 
 
-                //协作
-                for (var x = 0; x<xcount; x++)
+                //个人协作
                 {
-                    int[] r = new int[] { 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1 };
+                    for (var x = 0; x<xcount; x++)
+                    {
+                        int[] r = new int[] { 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1 };
+                        var w = r[Random.Shared.Next(0, r.Length)];
+                        var item = new ItemRecord { };
+                        switch (true)
+                        {
+                            //未参与
+                            case bool when w==0:
+                                item.resultWeight= InteractWeight.T0;
+                                item.resultType= InteractReultType.T0;
+                                item.itemScore=0;
+                                break;
+                            //
+                            case bool when w==1:
+                                item.resultWeight= InteractWeight.TP;
+                                item.resultType= InteractReultType.TP;
+                                int[] q = new int[] { 10, 5, 5, 20, 30, 10, 25, 10, 15, 15, 30, 5, 10, 20, 15, 25, 10, 5, 15, 20, 20, 25, 5, 5 };
+                                item.itemScore=q[Random.Shared.Next(0, q.Count())];
+                                break;
+                        }
+                        student.coworkRecord.itemRecords.Add(item);
+                    }
+                }
+                //小组协作
+                {
+                    int[] r = new int[] { 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1 };
                     var w = r[Random.Shared.Next(0, r.Length)];
                     var item = new ItemRecord { };
+                    double grpcoworkScore = 0;
                     switch (true)
                     {
                         //未参与
                         case bool when w==0:
-                            item.resultWeight= InteractWeight.T0;
-                            item.resultType= InteractReultType.T0;
-                            item.itemScore=0;
                             break;
                         //
                         case bool when w==1:
-                            item.resultWeight= InteractWeight.TP;
-                            item.resultType= InteractReultType.TP;
+                            
                             int[] q = new int[] { 10, 5, 5, 20, 30, 10, 25, 10, 15, 15, 30, 5, 10, 20, 15, 25, 10, 5, 15, 20, 20, 25, 5, 5 };
-                            item.itemScore=q[Random.Shared.Next(0, q.Count())];
+                            grpcoworkScore= q[Random.Shared.Next(0, q.Count())]* Random.Shared.Next(0,3 );
+                            student.group_coworkScore.Add(grpcoworkScore);
                             break;
                     }
-                    student.coworkRecord.itemRecords.Add(item);
                 }
+              
                 //评测
                 for (var e = 0; e<ecount; e++)
                 {
@@ -385,6 +412,7 @@ namespace HTEX.Test.Controllers
                     }
                 }
             }
+            //个人协作
             for (var p = 0; p<xcount; p++)
             {
                 var order = students.OrderByDescending(x => x.coworkRecord.itemRecords[p].itemScore);
@@ -407,10 +435,12 @@ namespace HTEX.Test.Controllers
                     }
                 }
             }
+           
             #endregion 数据模拟
             string jsons = await System.IO.File.ReadAllTextAsync("F:\\lesson-local\\analysis\\analysis.json");
             long time = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
             LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
+
             var lessonItems= LessonETLService. ProcessStudentDataV2(students, lessonDataAnalysis);
             var excleFile = $"F:\\mock-data\\{time}.xlsx";
             var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);

TEAMModelOS.Extension/HTEX.Test/Program.cs

@@ -26,9 +26,10 @@ namespace HTEX.Test
     {
         public static async Task Main(string[] args)
         {
-              //Test2();
+            //Test2();
 
-            await MockDataController.MockData();
+           
+           await MockDataController.MockData();
             //var builder = WebApplication.CreateBuilder(args);
 
             //// Add services to the container.
@@ -54,6 +55,18 @@ namespace HTEX.Test
 
             //app.Run();
         }
+        static double CalculateScore(int n, double k = 1)
+        {
+            if (n == 0)
+            {
+                return 0;
+            }
+            else
+            {
+                double score = 60 + (40 / (1 + Math.Exp(-k * (n - 1))));
+                return Math.Max(80, Math.Min(score, 100));
+            }
+        }
 
         public static string Test2() 
         {
@@ -107,6 +120,7 @@ namespace HTEX.Test
           
             //标准差偏差N倍，视为异常数据
             double thresholdMultiplier =2;
+            lessonDataAnalysisCluster.task= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.task.OrderBy(x => x), thresholdMultiplier);
             lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
             lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
             lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
@@ -115,18 +129,16 @@ namespace HTEX.Test
             lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x => x), thresholdMultiplier);
             lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
              //超过60 80的
+            var all = lessonDataAnalysisCluster.interactNormal.Select(x =>  x).OrderBy(x => x).ToArray();
+            double n = all.Max()+1;
+            var clusterInteract = HTEX.Lib.ETL. KMeansService.KMeansOptimized(all,3);
+            //foreach (var item in clusterInteract.OrderBy(x => x.Key))
+            //{
+            //    lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
+            //    Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
+            //}
 
-
-            var d = lessonDataAnalysisCluster.interactNormal.Select(x =>  x).OrderBy(x => x).ToArray();
-            double n = d.Max()+1;
-            var clusterInteract = HTEX.Lib.ETL. KMeansService.KMeansOptimized(d,3);
-            foreach (var item in clusterInteract.OrderBy(x => x.Key))
-            {
-                lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
-                Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
-            }
-
-            IEnumerable<double> all = lessonDataAnalysisCluster.clustersInteract.SelectMany(x => x.Value);
+           //IEnumerable<double> all = lessonDataAnalysisCluster.clustersInteract.SelectMany(x => x.Value);
             int pass = 0;
             for (var i = 1; i<n; i++) 
             {
@@ -155,18 +167,24 @@ namespace HTEX.Test
             lessonDataAnalysisCluster.interactHigh=high;
             lessonDataAnalysisCluster.interactMedium=medium;
             lessonDataAnalysisCluster.interactLow=low;
+            List<KeyValuePair<double, List<double>>> levelInteract = new List<KeyValuePair<double, List<double>>>()
+            {
+                new KeyValuePair<double, List<double>>(low, all.Where(x => x<pass).ToList()),
+                new KeyValuePair<double, List<double>>(medium, all.Where(x => x>=pass  &&x<good).ToList() ),
+                new KeyValuePair<double, List<double>>(high, all.Where(x => x>=good).ToList())
+            };
+            lessonDataAnalysisCluster.levelInteract= levelInteract;
             double ss = 0;
             for (var i = 1; i<n; i++) 
             {
                // KeyValuePair<double, List<int>> curr = new KeyValuePair<double, List<int>>();
-                var s = lessonDataAnalysisCluster.clustersInteract.FindAll(x => x.Value.Min()<=i  && x.Value.Max()>=i).MinBy(x=>x.Key);
+                var s = lessonDataAnalysisCluster.levelInteract.FindAll(x => x.Value.Min()<=i  && x.Value.Max()>=i).MinBy(x=>x.Key);
                 var p = LessonETLService.GetPersent(all, i);
                 var l= i<pass?pass:i<good?pass:good;
-                var e = (i*1.0/l) *(p.persent)  *  (s.Value.Count*1.0/d.Count());
+                var e = (i*1.0/l) *(p.persent)  *  (s.Value.Count*1.0/all.Count());
                 ss+= e;
                 Console.WriteLine($"n: {i},l: {l}, persent: {p.persent},count: {p.count},s-min: {s.Value.Min()},s-max: {s.Value.Max()},value: {e}");
             }
-
             //foreach (var s in clusterInteract.OrderBy(x => x.Key))
             //{
             //    clustersDataInteract.Add(s);

TEAMModelOS.Extension/HTEX.Test/x.xml

@@ -4,207 +4,6 @@
         <name>HTEX.Test</name>
     </assembly>
     <members>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.pscore">
-            <summary>
-            个人计分
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.gscore">
-            <summary>
-            小组计分
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.tscore">
-            <summary>
-            互动计分
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.upload">
-            <summary>
-            作品上传数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.stuCowork">
-            <summary>
-            学生协作成果数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.groupCowork">
-            <summary>
-            小组协作成果数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.pickup">
-            <summary>
-            挑人集合
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.LessonDataAnalysis.pickup_group">
-            <summary>
-            挑人集合-小组
-            </summary>
-        </member>
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.GetPersent(System.Collections.Generic.IEnumerable{System.Double},System.Int32)">
-            <summary>
-            计算当前元素在集合中超过了多少百分比的值
-            </summary>
-            <param name="nums"></param>
-            <param name="curr"></param>
-            <returns></returns>
-        </member>
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.CleanDataBySDThreshold(System.Collections.Generic.IEnumerable{System.Double},System.Double)">
-            <summary>
-            使用标准差定义异常值。如果一个数字与平均值的偏差超过某个标准差倍数（例如2倍或3倍），则可以认为它是异常的。
-            </summary>
-            <param name="array"></param>
-            <returns></returns>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.examCount">
-            <summary>
-            评测数量
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.taskCount">
-            <summary>
-            任务数量
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.irsCount">
-            <summary>
-            IRS次数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.interactNormalCount">
-            <summary>
-            互动次数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.coworkCount">
-            <summary>
-            协作次数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.smartRatingCount">
-            <summary>
-            智能评分次数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.examCountBase">
-            <summary>
-            评测数量
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.taskCountBase">
-            <summary>
-            任务数量
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.irsCountBase">
-            <summary>
-            IRS次数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.interactNormalCountBase">
-            <summary>
-            互动次数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.coworkCountBase">
-            <summary>
-            协作次数
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LessonRecordController.TechCount.smartRatingCountBase">
-            <summary>
-            智能评分次数
-            </summary>
-        </member>
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.GetBaseData(TEAMModelOS.SDK.Models.LessonBase)">
-            <summary>
-            处理base.json的数据
-            </summary>
-            <param name="lessonRecord"></param>
-            <param name="lessonBase"></param>
-            <returns></returns>
-        </member>
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.GetIRSData(TEAMModelOS.SDK.Models.LessonBase,TEAMModelOS.SDK.Models.TimeLineData,System.Collections.Generic.List{TEAMModelOS.SDK.Models.IRSData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.ExamData},System.String)">
-             <summary>
-            读取互动信息
-            Event 过滤类型  'PopQuesLoad', 'ReAtmpAnsStrt', 'BuzrAns','BuzrLoad'
-             在IRS.json处理 'PopQuesLoad'互动问答 , 'ReAtmpAnsStrt' 二次作答 , 'BuzrAns'  抢权（新）, 'BuzrLoad'抢权（旧）
-            TimeLine.json 中找到对应类型，根据Pgid 去 IRS.json 中找到对应数据，从clientAnswers 的下标对应 base.json 中的 student 找到对应学生信息 clientAnswers.length > 1 则表示有二次作答
-            读取IRS.json
-             </summary>
-             <param name="lessonRecord"></param>
-             <param name="lessonBase"></param>
-             <param name="irsDatas"></param>
-             <returns></returns>
-        </member>
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.GetExamData(TEAMModelOS.SDK.Models.LessonBase,TEAMModelOS.SDK.Models.TimeLineData,System.Collections.Generic.List{TEAMModelOS.SDK.Models.ExamData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData})">
-            <summary>
-            获取课中评测数据
-            </summary>
-            <param name="lessonRecord"></param>
-            <param name="lessonBase"></param>
-            <param name="timeLineData"></param>
-            <param name="coworkDatas"></param>
-            <param name="studentLessonDatas"></param>
-            <returns></returns>
-        </member>
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.GetCoworkData(TEAMModelOS.SDK.Models.LessonBase,TEAMModelOS.SDK.Models.TimeLineData,System.Collections.Generic.List{TEAMModelOS.SDK.Models.CoworkData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData})">
-            <summary>
-            协作参与率 态度计算
-            </summary>
-            <param name="lessonRecord"></param>
-            <param name="lessonBase"></param>
-            <param name="timeLineData"></param>
-            <param name="coworkDatas"></param>
-            <param name="studentLessonDatas"></param>
-            <returns></returns>
-        </member>
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.GetTaskData(TEAMModelOS.SDK.Models.LessonBase,TEAMModelOS.SDK.Models.TimeLineData,System.Collections.Generic.List{TEAMModelOS.SDK.Models.TaskData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData})">
-            <summary>
-            处理学生回推数据，并将回推纳入学习态度计算。
-            </summary>
-            <param name="lessonRecord"></param>
-            <param name="lessonBase"></param>
-            <param name="timeLineData"></param>
-            <param name="taskDatas"></param>
-            <param name="studentLessonDatas"></param>
-            <returns></returns>
-        </member>
-        <!-- Badly formed XML comment ignored for member "M:HTEX.Test.Controllers.LessonRecordController.GetSmartRatingData(TEAMModelOS.SDK.Models.LessonBase,TEAMModelOS.SDK.Models.TimeLineData,System.Collections.Generic.List{TEAMModelOS.SDK.Models.SmartRatingData},System.Collections.Generic.List{TEAMModelOS.SDK.Models.StudentLessonData},System.String)" -->
-        <member name="M:HTEX.Test.Controllers.LessonRecordController.MinMaxNormalization(System.Double,System.Double,System.Double)">
-            <summary>
-            最小-最大归一化（Min-Max Normalization）算法。这种算法通常用于将数据的特征值缩放到一个指定的范围内，通常是0到1之间，或者任何其他指定的范围。
-            </summary>
-            <returns></returns>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LocalStudent.attend">
-            <summary>
-            出席状态 1出席，6公假，5事假，4病假，2缺席，0未签到
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LocalStudent.id">
-            <summary>
-            学生的学号
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LocalStudent.index">
-            <summary>
-            学生所在下标
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LocalStudent.seatID">
-            <summary>
-            学生座位号
-            </summary>
-        </member>
-        <member name="P:HTEX.Test.Controllers.LocalStudent.groupId">
-            <summary>
-            小组编号
-            </summary>
-        </member>
         <member name="M:HTEX.Test.Controllers.MockDataController.GetPersent(System.Collections.Generic.IEnumerable{System.Double},System.Double)">
             <summary>
             计算当前元素在集合中超过了多少百分比的值