update

HTEX.DataETL/Controllers/LessonRecordController.cs

@@ -346,19 +346,17 @@ namespace HTEX.DataETL.Controllers
                 List<(string id,string owner, List<StudentLessonData> studentLessonData)> studentLessonDatas= new List<(string id, string owner, List<StudentLessonData>)>();
                 List<(string id, string owner, List<StudentLessonItem> studentLessons,List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)> lessonItems = new List<(string id, string owner, List<StudentLessonItem> studentLessons, List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)>();
                 int n = 0;
-                foreach (var  item in lessonLocals) 
+                Parallel.ForEach(lessonLocals,   (item, con) => 
                 {
-                    var studata = await LessonETLService.DoStudentLessonData(Constant.objectiveTypes, _azureStorage, item, _dingDing, _azureCosmos.GetCosmosClient(), "China", _azureRedis, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
-                    if (studata.codeBools.FindAll(x=>x.value).IsNotEmpty()) 
+                    var studata = LessonETLService.DoStudentLessonDataV2(Constant.objectiveTypes, item, "China", studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
+                    if (studata.codeBools.FindAll(x => x.value).IsNotEmpty())
                     {
                         string owner = item.lessonRecord.scope.Equals("school") ? item.lessonRecord.school : item.lessonRecord.tmdid;
                         studentLessonDatas.Add((item.lessonRecord.id, owner, studata.studentLessonDatas));
-                        lessonItems.Add((item.lessonRecord.id, owner, studata.lessonItems,studata.codeBools,studata.studentLessonDatas));
+                        lessonItems.Add((item.lessonRecord.id, owner, studata.lessonItems, studata.codeBools, studata.studentLessonDatas));
                     }
                     n++;
-                }
-
-
+                });
                 foreach (var group in schoolGroup)
                 {
                     string students_path = $"{lessonBasePath}\\students\\{group.key}\\students.json";

TEAMModelOS.Extension/HTEX.Lib/ETL/Lesson/LessonETLService.cs

@@ -104,8 +104,206 @@ namespace HTEX.Lib.ETL.Lesson
             return codeBools;
         }
 
-      
-       
+        /// <summary>
+        /// 生成学生student-analysis.json
+        /// </summary>
+        /// <param name="objectiveTypes"></param>
+        /// <param name="azureStorage"></param>
+        /// <param name="lessonLocal"></param>
+        /// <returns></returns>
+        public static   (List<StudentLessonData> studentLessonDatas, List<StudentLessonItem> lessonItems, List<CodeBool> codeBools)
+            DoStudentLessonDataV2(List<string> objectiveTypes,  LessonLocal lessonLocal,   string location,  List<StudentSemesterRecord> studentSemesterRecords, List<OverallEducation> overallEducations,
+            LessonDataAnalysisModel lessonDataAnalysis, List<Student> studentsBase, List<School> schools)
+        {
+            List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
+            studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
+            studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal?.lessonRecord?.id);
+            studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+            studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
+            studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+            studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+            var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
+            studentLessonDatas= pickupData.studentLessonDatas;
+            var codeBools = GetCodeBools(studentLessonDatas);
+            List<StudentLessonItem> lessonItems = new List<StudentLessonItem>();
+            string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
+            if (codeBools.FindAll(x => x.value).IsNotEmpty())
+            {
+ 
+                lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis, codeBools);
+ 
+                if (lessonLocal.lessonRecord.scope.Equals("school")&& !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.school)  && location.Equals("China", StringComparison.OrdinalIgnoreCase))
+                {
+                    School school = schools.Find(x => x.id.Equals(lessonLocal.lessonRecord.school));
+                    if (school!=null)
+                    {
+ 
+                        string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : school.period.FirstOrDefault()?.id;
+                        var period = school.period.Find(x => x.id.Equals(periodId));
+                        if (period!=null)
+                        {
+                            var semester = SchoolService.GetSemester(period, lessonLocal.lessonRecord.startTime);
+                            string pre_id = $"{semester.studyYear}-{semester.currSemester.id}";
+                            string code = $"StudentSemesterRecord-{school.id}";
+                            foreach (var studentLessonData in studentLessonDatas)
+                            {
+                                StudentSemesterRecord studentSemester = studentSemesterRecords.Find(x => x.id.Equals($"{pre_id}-{studentLessonData.id}") && x.code.Equals(code));
+                                if (studentSemester==null)
+                                {
+ 
+                                    studentSemester= new StudentSemesterRecord
+                                    {
+                                        id= $"{pre_id}-{studentLessonData.id}",
+                                        code=code,
+                                        stuid= studentLessonData.id,
+                                        userType=Constant.ScopeStudent,
+                                        school=school.id,
+                                        studyYear=semester.studyYear,
+                                        semesterId=semester.currSemester.id,
+                                        period= period?.id,
+                                        pk="StudentSemesterRecord"
+                                    };
+ 
+                                    studentSemesterRecords.Add(studentSemester);
+                                }
+
+                                var les = studentSemester.les.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
+                                if (les==null)
+                                {
+                                    les = new StuLesson()
+                                    {
+                                        id= lessonLocal.lessonRecord.id,
+                                        time= lessonLocal.lessonRecord.startTime,
+                                        attend=0
+                                    };
+                                    studentSemester.les.Add(les);
+                                }
+                                else
+                                {
+                                    les.time= lessonLocal.lessonRecord.startTime;
+                                }
+                                var lesson = studentSemester.lessons.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
+                                if (lesson!=null)
+                                {
+                                    if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0)
+                                    {
+                                        les.attend=1;
+                                        lesson.tmdid = lessonLocal.lessonRecord.tmdid;
+                                        lesson.sid = lessonLocal.lessonRecord.subjectId;
+                                        lesson.cid = lessonLocal.lessonRecord.courseId;
+                                        lesson.hrate = studentLessonData.cooperation;
+                                        lesson.crate = studentLessonData.achieve;
+                                        lesson.trate = studentLessonData.attitude;
+                                        lesson.xrate = studentLessonData.cowork;
+                                        lesson.prate = studentLessonData.appraise;
+                                    }
+                                    else
+                                    {
+                                        if (studentLessonData.attend==1)
+                                        {
+                                            les.attend=1;
+                                        }
+                                        else
+                                        {
+                                            les.attend=0;
+                                        }
+                                    }
+                                }
+                                else
+                                {
+                                    if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0)
+                                    {
+                                        les.attend=1;
+                                        lesson = new StuLessonLite
+                                        {
+                                            id=lessonLocal.lessonRecord.id,
+                                            tmdid=lessonLocal.lessonRecord.tmdid,
+                                            sid= lessonLocal.lessonRecord.subjectId,
+                                            cid= lessonLocal.lessonRecord.courseId,
+                                            hrate=studentLessonData.cooperation,
+                                            crate=studentLessonData.achieve,
+                                            trate=studentLessonData.attitude,
+                                            xrate=studentLessonData.cowork,
+                                            prate=studentLessonData.appraise
+                                        };
+                                        studentSemester.lessons.Add(lesson);
+                                    }
+                                    else
+                                    {
+                                        if (studentLessonData.attend==1)
+                                        {
+                                            les.attend=1;
+                                        }
+                                        else
+                                        {
+                                            les.attend=0;
+                                        }
+                                    }
+                                }
+                               
+                                string oid = $"{semester.studyYear}-{semester.currSemester.id}-{studentSemester.stuid}";
+                                string ocode = $"OverallEducation-{school.id}";
+
+                                var student = studentsBase.Find(x => x.id.Equals(studentSemester.stuid));
+                                if (student!=null)
+                                {
+                                    OverallEducation overallEducation = overallEducations.Find(x => x.id.Equals(oid) && x.code.Equals(ocode));
+                                    if (overallEducation== null)
+                                    {
+
+                                        overallEducation = new OverallEducation
+                                        {
+                                            id =oid,
+                                            code = $"OverallEducation-{school.id}",
+                                            pk = "OverallEducation",
+                                            ttl = -1,
+                                            name = student.name,
+                                            classId = student?.classId,
+                                            schoolCode = $"{school.id}",
+                                            semesterId = semester.currSemester.id,
+                                            year = semester.studyYear,
+                                            periodId = $"{period.id}",
+                                            stuYear = student.year,
+                                            studentId = student.id,
+                                            lessonScore= new List<StudentLessonRecord>()
+                                        };
+                                        overallEducations.Add(overallEducation);
+                                    }
+                                    var hasrecord = overallEducation.lessonScore.Find(x => x.lessonId.Equals(lessonLocal.lessonRecord.id));
+                                    if (hasrecord==null)
+                                    {
+                                        hasrecord=   new StudentLessonRecord();
+                                        overallEducation.lessonScore.Add(hasrecord);
+                                    }
+                                    hasrecord.gscore = studentLessonData.gscore;
+                                    hasrecord.pscore = studentLessonData.pscore;
+                                    hasrecord.tscore = studentLessonData.tscore;
+                                    hasrecord.tmdid = lessonLocal.lessonRecord.tmdid;
+                                    hasrecord.school = school.id;
+                                    hasrecord.scope = lessonLocal.lessonRecord.scope;
+                                    hasrecord.lessonId = lessonLocal.lessonRecord.id;
+                                    hasrecord.courseId = lessonLocal.lessonRecord.courseId;
+                                    periodId = period?.id;
+                                    hasrecord.subjectId = lessonLocal.lessonRecord.subjectId;
+                                    hasrecord.time= lessonLocal.lessonRecord.startTime;
+                                    hasrecord.attend=les.attend;
+                                    if (lesson!=null)
+                                    {
+                                        hasrecord.hrate=lesson.hrate;
+                                        hasrecord.crate=lesson.crate;
+                                        hasrecord.trate=lesson.trate;
+                                        hasrecord.xrate=lesson.xrate;
+                                        hasrecord.prate=lesson.prate;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            return (studentLessonDatas, lessonItems, codeBools);
+        }
+
         /// <summary>
         /// 生成学生student-analysis.json
         /// </summary>
@@ -2446,7 +2644,7 @@ namespace HTEX.Lib.ETL.Lesson
                     {
                         //是IES大陆正式站历史课例数据，自2024-03-01至2024-10-08日，互动指数或学法指数黄灯或绿灯，不包含醍摩豆学校及测试学校，课例时长超过5分钟的有效课例(10,680笔数据) 的IRS互动+抢权+挑人的次数集合，
                         //通过“2倍标准差规则” 移除异常值后得到的集合，再通过K-Means聚类算法得到高低位阶互动频次两个集合，并根据当前课例互动次数位阶的集合的质心值，该值定为m值
-                        IEnumerable<double> all = lessonDataAnalysis.levelInteract.SelectMany(x => x.Value);
+                        //IEnumerable<double> all = lessonDataAnalysis.levelInteract.SelectMany(x => x.Value);
                         var currMacth = lessonDataAnalysis.levelInteract.FindAll(x => x.Value.Min()<=n  && x.Value.Max()>=n);
                         KeyValuePair<double, List<double>> curr = new KeyValuePair<double, List<double>>();
                         if (currMacth!=null  && currMacth.Count()>0)
@@ -2457,10 +2655,11 @@ namespace HTEX.Lib.ETL.Lesson
                         {
                             curr =  lessonDataAnalysis.levelInteract.MaxBy(x => x.Key);
                         }
-                        var p = LessonETLService.GetPersent(all, n);
+                        var p = LessonETLService.GetPersent(lessonDataAnalysis.interactNormal, n);
                         var l = n<lessonDataAnalysis.interactPass ? lessonDataAnalysis.interactLow : n>lessonDataAnalysis.interactGood ? lessonDataAnalysis.interactHigh : lessonDataAnalysis.interactMedium;
                         //出题系数=当前互动次数与互动通过次数之间的比例*当前互动次数与互动中位数之间的比例*当前互动次数与互动高阶互动频次的比例
-                        var m = (n*1.0/l) *(p.persent/100)  *  (curr.Value.Count*1.0/all.Count());
+                        // var m = (n*1.0/l) *(p.persent/100)  *  (curr.Value.Count*1.0/all.Count());
+                        var m = (n*1.0/l) *(p.persent/100)  *  (lessonDataAnalysis.interactNormal.Where(x => x.Value>= curr.Value[0]&&  x.Value<=curr.Value[1] ).Sum(x=>x.Value)*1.0/lessonDataAnalysis.interactNormal.Sum(x=>x.Value));
                         //学生作答次数
                         var w = studentLessonData.interactRecord.interactRecords.Where(x => x.resultWeight>=InteractWeight.T1).Count()*1.0;
                         //作答正确数（包括部分正确）