update

HTEX.DataETL/Controllers/LessonRecordController.cs

@@ -242,8 +242,19 @@ namespace HTEX.DataETL.Controllers
                     }
                     else
                     {
-                        System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
-                        System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
+                        try
+                        {
+                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
+                        }
+                        catch (Exception ex) { }
+                        try
+                        {
+                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
+                        }
+                        catch (Exception ex)
+                        {
+
+                        }
                     }
                     var studata = await LessonETLService.DoStudentLessonData(Constant.objectiveTypes, _azureStorage, item, _dingDing, _azureCosmos.GetCosmosClient(), "China", _azureRedis, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
                     if (techCount!= null) 
@@ -254,8 +265,18 @@ namespace HTEX.DataETL.Controllers
                             await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
                         }
                         else {
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
+                            try {
+                                System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
+                            }
+                            catch (Exception ex) { }
+                            try
+                            {
+                                System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
+                            }
+                            catch (Exception ex) 
+                            {
+                            
+                            }
                         }
                     }
                     index++;
@@ -368,35 +389,46 @@ namespace HTEX.DataETL.Controllers
                 List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
                 if (lessonRecords.IsNotEmpty())
                 {
-
                     await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
                     {
                         string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
-                        if (item.lessonBase!=null && item.lessonBase.student!=null)
-                        {
-                            TechCount techCount = new TechCount
-                            {
-                                lessonId=item.lessonRecord?.id,
-                                examCount = item.examDatas.Count,
-                                taskCount = item.taskDatas.Count,
-                                irsCount = item.irsDatas.Count,
-                                coworkCount = item.coworkDatas.Count,
-                                smartRatingCount =item.smartRatingDatas.Count,
-                                timeCount=item.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList()
-                            };
-                            if (techCount.examCount>0 || techCount.taskCount>0 ||techCount.irsCount>0 || techCount.coworkCount>0 || techCount.smartRatingCount>0) 
-                            {
-                                await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord.id}-count.json", techCount.ToJsonString());
-                                await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
-                            }
+                        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
+                        //if (item.lessonBase!=null && item.lessonBase.student!=null)
+                        //{
+                        //    TechCount techCount = new TechCount
+                        //    {
+                        //        yearMonth=yearMonthPath,
+                        //        lessonId=item.lessonRecord?.id,
+                        //        examCount = item.examDatas.Count,
+                        //        taskCount = item.taskDatas.Count,
+                        //        irsCount = item.irsDatas.Count,
+                        //        coworkCount = item.coworkDatas.Count,
+                        //        smartRatingCount =item.smartRatingDatas.Count,
+                        //        timeCount=item.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList()
+                        //    };
+                        //    if (techCount.examCount>0 || techCount.taskCount>0 ||techCount.irsCount>0 || techCount.coworkCount>0 || techCount.smartRatingCount>0
+                                
+                        //        )
+                        //    {
+                        //        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord.id}-count.json", techCount.ToJsonString());
+                               
+                        //    }
+                        //    else {
+                        //        try
+                        //        {
+                        //            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
+                        //        }
+                        //        catch (Exception ex) { }
+                              
+                        //    }
                            
-                        }
-                        else
-                        {
+                        //}
+                        //else
+                        //{
 
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
-                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
-                        }
+                        //    System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
+                        //    System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
+                        //}
                     }
                 }
                 List<TechCount> techCounts = new List<TechCount>();

TEAMModelOS.Extension/HTEX.Lib/ETL/Lesson/LessonETLService.cs

@@ -130,234 +130,238 @@ namespace HTEX.Lib.ETL.Lesson
             var codeBools=  GetCodeBools(studentLessonDatas);
             List<StudentLessonItem> lessonItems = new List<StudentLessonItem>();
             string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
-            try
+            if (codeBools.FindAll(x => x.value).IsNotEmpty()) 
             {
-                
-                bool exists= await  azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis.json").ExistsAsync();
-                if (exists) 
+                try
                 {
-                    if (lessonDataAnalysis==null)
+
+                    bool exists = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis.json").ExistsAsync();
+                    if (exists)
                     {
-                        BlobDownloadResult blobDownload = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis.json").DownloadContentAsync();
-                        lessonDataAnalysis = blobDownload.Content.ToObjectFromJson<LessonDataAnalysisCluster>();
-                    }
+                        if (lessonDataAnalysis==null)
+                        {
+                            BlobDownloadResult blobDownload = await azureStorage.GetBlobContainerClient("0-public").GetBlobClient($"/lesson/analysis/analysis.json").DownloadContentAsync();
+                            lessonDataAnalysis = blobDownload.Content.ToObjectFromJson<LessonDataAnalysisCluster>();
+                        }
 
-                    lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis, codeBools);
-                    XmlDocument xmlDocument = new XmlDocument();
-                    var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
-                    xmlDocument.Load($"{runtimePath}\\summary.xml");
-                    await LessonETLService.ExportToExcelAzureBlob(lessonItems,azureStorage, owner,$"{lessonLocal.lessonRecord.id}/student-analysis.xlsx", xmlDocument);
-                   // await _dingDing.SendBotMsg($"学生课中报告已经生成：{owner},records/{lessonLocal.lessonRecord.id}/student-analysis.xlsx",GroupNames.成都开发測試群組);
-                }
-            }
-            catch (Exception ex)
-            {
-                if (!ex.Message.Contains("The specified blob does not exist"))
-                {
-                    await _dingDing.SendBotMsg($"{Environment.GetEnvironmentVariable("Option:Location")},lesson/analysis/analysis.json转换异常，{ex.Message}{ex.StackTrace}", GroupNames.成都开发測試群組);
+                        lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis, codeBools);
+                        XmlDocument xmlDocument = new XmlDocument();
+                        var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
+                        xmlDocument.Load($"{runtimePath}\\summary.xml");
+                        await LessonETLService.ExportToExcelAzureBlob(lessonItems, azureStorage, owner, $"{lessonLocal.lessonRecord.id}/student-analysis.xlsx", xmlDocument);
+                        // await _dingDing.SendBotMsg($"学生课中报告已经生成：{owner},records/{lessonLocal.lessonRecord.id}/student-analysis.xlsx",GroupNames.成都开发測試群組);
+                    }
                 }
-            }
-            await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
-            if (lessonLocal.lessonRecord.scope.Equals("school")&& !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.school)  && location.Equals("China",StringComparison.OrdinalIgnoreCase))
-            {
-                School school = schools.Find(x=>x.id.Equals(lessonLocal.lessonRecord.school));
-                if (school==null)
+                catch (Exception ex)
                 {
-                    ResponseMessage response = await client.GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemStreamAsync(lessonLocal.lessonRecord.school, new PartitionKey("Base"));
-                    if (response.IsSuccessStatusCode)
+                    if (!ex.Message.Contains("The specified blob does not exist"))
                     {
-                        school = JsonDocument.Parse(response.Content).RootElement.ToObject<School>();
-                        schools.Add(school);
+                        await _dingDing.SendBotMsg($"{Environment.GetEnvironmentVariable("Option:Location")},lesson/analysis/analysis.json转换异常，{ex.Message}{ex.StackTrace}", GroupNames.成都开发測試群組);
                     }
                 }
-                if (school!=null)
+                await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
+                if (lessonLocal.lessonRecord.scope.Equals("school")&& !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.school)  && location.Equals("China", StringComparison.OrdinalIgnoreCase))
                 {
-                    var sdtus= studentLessonDatas.Where(y =>!studentsBase.Where(z=>z.schoolId.Equals(school.id)).Select(x => x.id).Contains(y.id) );
-                    if (sdtus!=null && sdtus.Count()>0)
+                    School school = schools.Find(x => x.id.Equals(lessonLocal.lessonRecord.school));
+                    if (school==null)
                     {
-                        var result = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<Student>($"select value c from c where c.id in ({string.Join(",", sdtus.Select(d => $"'{d.id}'"))})", $"Base-{school.id}");
-                        if (result.list.Any())
+                        ResponseMessage response = await client.GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemStreamAsync(lessonLocal.lessonRecord.school, new PartitionKey("Base"));
+                        if (response.IsSuccessStatusCode)
                         {
-                            studentsBase.AddRange(result.list);
+                            school = JsonDocument.Parse(response.Content).RootElement.ToObject<School>();
+                            schools.Add(school);
                         }
                     }
-                    string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : school.period.FirstOrDefault()?.id;
-                    var period = school.period.Find(x => x.id.Equals(periodId));
-                    if (period!=null) 
+                    if (school!=null)
                     {
-                        var semester = SchoolService.GetSemester(period, lessonLocal.lessonRecord.startTime);
-                        string pre_id = $"{semester.studyYear}-{semester.currSemester.id}";
-                        string code = $"StudentSemesterRecord-{school.id}";
-                        foreach (var studentLessonData in studentLessonDatas)
+                        var sdtus = studentLessonDatas.Where(y => !studentsBase.Where(z => z.schoolId.Equals(school.id)).Select(x => x.id).Contains(y.id));
+                        if (sdtus!=null && sdtus.Count()>0)
                         {
-                            StudentSemesterRecord studentSemester = studentSemesterRecords.Find(x=>x.id.Equals($"{pre_id}-{studentLessonData.id}") && x.code.Equals(code));
-                            if (studentSemester==null) 
+                            var result = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<Student>($"select value c from c where c.id in ({string.Join(",", sdtus.Select(d => $"'{d.id}'"))})", $"Base-{school.id}");
+                            if (result.list.Any())
                             {
-                                ResponseMessage responseMessage = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).ReadItemStreamAsync($"{pre_id}-{studentLessonData.id}", new PartitionKey(code));
-                                if (responseMessage.IsSuccessStatusCode)
-                                {
-                                    studentSemester= JsonDocument.Parse(responseMessage.Content).ToObject<StudentSemesterRecord>();
-                                }
-                                else
-                                {
-                                    studentSemester= new StudentSemesterRecord
-                                    {
-                                        id= $"{pre_id}-{studentLessonData.id}",
-                                        code=code,
-                                        stuid= studentLessonData.id,
-                                        userType=Constant.ScopeStudent,
-                                        school=school.id,
-                                        studyYear=semester.studyYear,
-                                        semesterId=semester.currSemester.id,
-                                        period= period?.id,
-                                        pk="StudentSemesterRecord"
-                                    };
-                                }
-                                studentSemesterRecords.Add(studentSemester);
+                                studentsBase.AddRange(result.list);
                             }
-                            
-                            var les =  studentSemester.les.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
-                            if (les==null)
-                            {
-                                  les = new StuLesson()
-                                {
-                                    id= lessonLocal.lessonRecord.id,
-                                    time= lessonLocal.lessonRecord.startTime,
-                                    attend=0
-                                };
-                                studentSemester.les.Add(les);
-                            }
-                            else {
-                                les.time= lessonLocal.lessonRecord.startTime;
-                            }
-                            var lesson = studentSemester.lessons.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
-                            if (lesson!=null)
+                        }
+                        string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : school.period.FirstOrDefault()?.id;
+                        var period = school.period.Find(x => x.id.Equals(periodId));
+                        if (period!=null)
+                        {
+                            var semester = SchoolService.GetSemester(period, lessonLocal.lessonRecord.startTime);
+                            string pre_id = $"{semester.studyYear}-{semester.currSemester.id}";
+                            string code = $"StudentSemesterRecord-{school.id}";
+                            foreach (var studentLessonData in studentLessonDatas)
                             {
-                                if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0)
-                                {
-                                    les.attend=1;
-                                    lesson.tmdid = lessonLocal.lessonRecord.tmdid;
-                                    lesson.sid = lessonLocal.lessonRecord.subjectId;
-                                    lesson.cid = lessonLocal.lessonRecord.courseId;
-                                    lesson.hrate = studentLessonData.cooperation;
-                                    lesson.crate = studentLessonData.achieve;
-                                    lesson.trate = studentLessonData.attitude;
-                                    lesson.xrate = studentLessonData.cowork;
-                                    lesson.prate = studentLessonData.appraise;
-                                }
-                                else
+                                StudentSemesterRecord studentSemester = studentSemesterRecords.Find(x => x.id.Equals($"{pre_id}-{studentLessonData.id}") && x.code.Equals(code));
+                                if (studentSemester==null)
                                 {
-                                    if (studentLessonData.attend==1)
+                                    ResponseMessage responseMessage = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).ReadItemStreamAsync($"{pre_id}-{studentLessonData.id}", new PartitionKey(code));
+                                    if (responseMessage.IsSuccessStatusCode)
                                     {
-                                        les.attend=1;
+                                        studentSemester= JsonDocument.Parse(responseMessage.Content).ToObject<StudentSemesterRecord>();
                                     }
                                     else
                                     {
-                                        les.attend=0;
+                                        studentSemester= new StudentSemesterRecord
+                                        {
+                                            id= $"{pre_id}-{studentLessonData.id}",
+                                            code=code,
+                                            stuid= studentLessonData.id,
+                                            userType=Constant.ScopeStudent,
+                                            school=school.id,
+                                            studyYear=semester.studyYear,
+                                            semesterId=semester.currSemester.id,
+                                            period= period?.id,
+                                            pk="StudentSemesterRecord"
+                                        };
                                     }
+                                    studentSemesterRecords.Add(studentSemester);
                                 }
-                            }
-                            else
-                            {
-                                if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0) 
+
+                                var les = studentSemester.les.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
+                                if (les==null)
                                 {
-                                    les.attend=1;
-                                    lesson = new StuLessonLite
+                                    les = new StuLesson()
                                     {
-                                        id=lessonLocal.lessonRecord.id,
-                                        tmdid=lessonLocal.lessonRecord.tmdid,
-                                        sid= lessonLocal.lessonRecord.subjectId,
-                                        cid= lessonLocal.lessonRecord.courseId,
-                                        hrate=studentLessonData.cooperation,
-                                        crate=studentLessonData.achieve,
-                                        trate=studentLessonData.attitude,
-                                        xrate=studentLessonData.cowork,
-                                        prate=studentLessonData.appraise
+                                        id= lessonLocal.lessonRecord.id,
+                                        time= lessonLocal.lessonRecord.startTime,
+                                        attend=0
                                     };
-                                    studentSemester.lessons.Add(lesson);
+                                    studentSemester.les.Add(les);
                                 }
                                 else
                                 {
-                                    if (studentLessonData.attend==1)
+                                    les.time= lessonLocal.lessonRecord.startTime;
+                                }
+                                var lesson = studentSemester.lessons.Find(x => x.id.Equals(lessonLocal.lessonRecord.id));
+                                if (lesson!=null)
+                                {
+                                    if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0)
                                     {
                                         les.attend=1;
+                                        lesson.tmdid = lessonLocal.lessonRecord.tmdid;
+                                        lesson.sid = lessonLocal.lessonRecord.subjectId;
+                                        lesson.cid = lessonLocal.lessonRecord.courseId;
+                                        lesson.hrate = studentLessonData.cooperation;
+                                        lesson.crate = studentLessonData.achieve;
+                                        lesson.trate = studentLessonData.attitude;
+                                        lesson.xrate = studentLessonData.cowork;
+                                        lesson.prate = studentLessonData.appraise;
                                     }
                                     else
                                     {
-                                        les.attend=0;
+                                        if (studentLessonData.attend==1)
+                                        {
+                                            les.attend=1;
+                                        }
+                                        else
+                                        {
+                                            les.attend=0;
+                                        }
                                     }
                                 }
-                            }
-                            //await client.GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
-                            string oid = $"{semester.studyYear}-{semester.currSemester.id}-{studentSemester.stuid}";
-                            string ocode = $"OverallEducation-{school.id}";
-                           
-                            var student=   studentsBase.Find(x => x.id.Equals(studentSemester.stuid));
-                            if (student!=null)
-                            {
-                                OverallEducation overallEducation = overallEducations.Find(x => x.id.Equals(oid) && x.code.Equals(ocode));
-                                if (overallEducation== null)
+                                else
                                 {
-                                    ResponseMessage oresponse = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).ReadItemStreamAsync(oid, new PartitionKey(ocode));
-                                    if (!oresponse.IsSuccessStatusCode)
+                                    if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 ||  studentLessonData.appraise>0)
                                     {
-                                        overallEducation = new OverallEducation
+                                        les.attend=1;
+                                        lesson = new StuLessonLite
                                         {
-                                            id =oid,
-                                            code = $"OverallEducation-{school.id}",
-                                            pk = "OverallEducation",
-                                            ttl = -1,
-                                            name = student.name,
-                                            classId = student?.classId,
-                                            schoolCode = $"{school.id}",
-                                            semesterId = semester.currSemester.id,
-                                            year = semester.studyYear,
-                                            periodId = $"{period.id}",
-                                            stuYear = student.year,
-                                            studentId = student.id,
-                                            lessonScore= new List<StudentLessonRecord>()
+                                            id=lessonLocal.lessonRecord.id,
+                                            tmdid=lessonLocal.lessonRecord.tmdid,
+                                            sid= lessonLocal.lessonRecord.subjectId,
+                                            cid= lessonLocal.lessonRecord.courseId,
+                                            hrate=studentLessonData.cooperation,
+                                            crate=studentLessonData.achieve,
+                                            trate=studentLessonData.attitude,
+                                            xrate=studentLessonData.cowork,
+                                            prate=studentLessonData.appraise
                                         };
+                                        studentSemester.lessons.Add(lesson);
                                     }
                                     else
                                     {
-                                        overallEducation = JsonDocument.Parse(oresponse.Content).RootElement.ToObject<OverallEducation>();
+                                        if (studentLessonData.attend==1)
+                                        {
+                                            les.attend=1;
+                                        }
+                                        else
+                                        {
+                                            les.attend=0;
+                                        }
                                     }
-                                    overallEducations.Add(overallEducation);
                                 }
-                               
-                                var hasrecord = overallEducation.lessonScore.Find(x => x.lessonId.Equals(lessonLocal.lessonRecord.id));
-                                if (hasrecord==null)
-                                {
-                                    hasrecord=   new StudentLessonRecord();
-                                    overallEducation.lessonScore.Add(hasrecord);
-                                }
-                                hasrecord.gscore = studentLessonData.gscore;
-                                hasrecord.pscore = studentLessonData.pscore;
-                                hasrecord.tscore = studentLessonData.tscore;
-                                hasrecord.tmdid = lessonLocal.lessonRecord.tmdid;
-                                hasrecord.school = school.id;
-                                hasrecord.scope = lessonLocal.lessonRecord.scope;
-                                hasrecord.lessonId = lessonLocal.lessonRecord.id;
-                                hasrecord.courseId = lessonLocal.lessonRecord.courseId;
-                                periodId = period?.id;
-                                hasrecord.subjectId = lessonLocal.lessonRecord.subjectId;
-                                hasrecord.time= lessonLocal.lessonRecord.startTime;
-                                hasrecord.attend=les.attend;
-                                if (lesson!=null) 
+                                //await client.GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
+                                string oid = $"{semester.studyYear}-{semester.currSemester.id}-{studentSemester.stuid}";
+                                string ocode = $"OverallEducation-{school.id}";
+
+                                var student = studentsBase.Find(x => x.id.Equals(studentSemester.stuid));
+                                if (student!=null)
                                 {
-                                    hasrecord.hrate=lesson.hrate;
-                                    hasrecord.crate=lesson.crate;
-                                    hasrecord.trate=lesson.trate;
-                                    hasrecord.xrate=lesson.xrate;
-                                    hasrecord.prate=lesson.prate;
+                                    OverallEducation overallEducation = overallEducations.Find(x => x.id.Equals(oid) && x.code.Equals(ocode));
+                                    if (overallEducation== null)
+                                    {
+                                        ResponseMessage oresponse = await client.GetContainer(Constant.TEAMModelOS, Constant.Student).ReadItemStreamAsync(oid, new PartitionKey(ocode));
+                                        if (!oresponse.IsSuccessStatusCode)
+                                        {
+                                            overallEducation = new OverallEducation
+                                            {
+                                                id =oid,
+                                                code = $"OverallEducation-{school.id}",
+                                                pk = "OverallEducation",
+                                                ttl = -1,
+                                                name = student.name,
+                                                classId = student?.classId,
+                                                schoolCode = $"{school.id}",
+                                                semesterId = semester.currSemester.id,
+                                                year = semester.studyYear,
+                                                periodId = $"{period.id}",
+                                                stuYear = student.year,
+                                                studentId = student.id,
+                                                lessonScore= new List<StudentLessonRecord>()
+                                            };
+                                        }
+                                        else
+                                        {
+                                            overallEducation = JsonDocument.Parse(oresponse.Content).RootElement.ToObject<OverallEducation>();
+                                        }
+                                        overallEducations.Add(overallEducation);
+                                    }
+
+                                    var hasrecord = overallEducation.lessonScore.Find(x => x.lessonId.Equals(lessonLocal.lessonRecord.id));
+                                    if (hasrecord==null)
+                                    {
+                                        hasrecord=   new StudentLessonRecord();
+                                        overallEducation.lessonScore.Add(hasrecord);
+                                    }
+                                    hasrecord.gscore = studentLessonData.gscore;
+                                    hasrecord.pscore = studentLessonData.pscore;
+                                    hasrecord.tscore = studentLessonData.tscore;
+                                    hasrecord.tmdid = lessonLocal.lessonRecord.tmdid;
+                                    hasrecord.school = school.id;
+                                    hasrecord.scope = lessonLocal.lessonRecord.scope;
+                                    hasrecord.lessonId = lessonLocal.lessonRecord.id;
+                                    hasrecord.courseId = lessonLocal.lessonRecord.courseId;
+                                    periodId = period?.id;
+                                    hasrecord.subjectId = lessonLocal.lessonRecord.subjectId;
+                                    hasrecord.time= lessonLocal.lessonRecord.startTime;
+                                    hasrecord.attend=les.attend;
+                                    if (lesson!=null)
+                                    {
+                                        hasrecord.hrate=lesson.hrate;
+                                        hasrecord.crate=lesson.crate;
+                                        hasrecord.trate=lesson.trate;
+                                        hasrecord.xrate=lesson.xrate;
+                                        hasrecord.prate=lesson.prate;
+                                    }
+                                    //if (overallEducation!=null) 
+                                    //{
+                                    //    await client.GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code));
+                                    //    string key = $"OverallEducation:{school.id}:{period.id}:{semester.studyYear}:{semester.currSemester.id}:{student?.classId}";
+                                    //    await azureRedis.GetRedisClient(8).HashSetAsync(key, studentLessonData.id, overallEducation.ToJsonString());
+                                    //    await azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
+                                    //}
                                 }
-                                //if (overallEducation!=null) 
-                                //{
-                                //    await client.GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code));
-                                //    string key = $"OverallEducation:{school.id}:{period.id}:{semester.studyYear}:{semester.currSemester.id}:{student?.classId}";
-                                //    await azureRedis.GetRedisClient(8).HashSetAsync(key, studentLessonData.id, overallEducation.ToJsonString());
-                                //    await azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
-                                //}
                             }
                         }
                     }
@@ -496,151 +500,149 @@ namespace HTEX.Lib.ETL.Lesson
 
         public static async IAsyncEnumerable<TechCount> GetTeachCount(List<LessonRecord> lessonRecords, List<string> filesLessons, string pathLessons, List<string> ignore, List<string> objectiveTypes, AzureStorageFactory azureStorage,bool force)
         {
+            List<TechCount> techCounts = new List<TechCount>();
             foreach (var item in filesLessons)
             {
                 if (item.EndsWith("-local.json"))
                 {
-                    TechCount count = null;
-
                     string lessonId = item.Split("\\").Last().Replace("-local.json", "");
                     string countFile = item.Replace("-local.json", "-count.json");
-                    if (System.IO.File.Exists(countFile))
+                    TechCount count = null;
+                    if (!force)
                     {
                         if (System.IO.File.Exists(countFile))
                         {
                             string contjson = await System.IO.File.ReadAllTextAsync(countFile);
                             count = contjson.ToObject<TechCount>();
                         }
+                        else
+                        {
+                            force=true;
+                            
+                        }
                     }
-                    if (string.IsNullOrWhiteSpace(count?.yearMonth)|| force)
+                    if (force) 
                     {
                         string localjson = await System.IO.File.ReadAllTextAsync(item);
                         var lessonLocal = localjson.ToObject<LessonLocal>();
-                        count = new TechCount();
-                        count.lessonId=item.Split("\\").Last().Replace("-local.json", "");
-                        count.examCount= lessonLocal.examDatas.Count;
-                        count.taskCount= lessonLocal.taskDatas.Count;
-                        count.irsCount= lessonLocal.irsDatas.Count;
-                        count.interactNormalCount=count.irsCount;
-                        count.coworkCount= lessonLocal.coworkDatas.Count;
-                        count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
-                        count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
-
-
-                        if (lessonLocal.lessonRecord!=null)
+                        List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
+                        studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
+                        studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal.lessonRecord.id);
+                        studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+                        studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
+                        studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+                        studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
+                        var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
+                        studentLessonDatas= pickupData.studentLessonDatas;
+                        var codeBools = LessonETLService.GetCodeBools(studentLessonDatas);
+                        if (codeBools.FindAll(x => x.value).IsNotEmpty()) 
                         {
+                            if (count==null) { count= new TechCount(); }
+                            count.lessonId=item.Split("\\").Last().Replace("-local.json", "");
+                            count.examCount= lessonLocal.examDatas.Count;
+                            count.taskCount= lessonLocal.taskDatas.Count;
+                            count.irsCount= lessonLocal.irsDatas.Count;
+                            count.interactNormalCount=count.irsCount;
+                            count.coworkCount= lessonLocal.coworkDatas.Count;
+                            count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
+                            count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
+
+
+                            if (lessonLocal.lessonRecord!=null)
+                            {
 
-                            count.yearMonth=DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
+                                count.yearMonth=DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
 
-                            if (lessonLocal?.lessonBase?.summary!=null)
-                            {
-                                count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
-                                count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
-                                count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
-                                count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
-                                count.examCountBase=lessonLocal.lessonBase.summary.examCount;
-                                count.interactNormalCountBase= count.interactNormalCount;
-                            }
-                            if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
-                            {
-                                count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
-                                count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
-                                count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
-                            }
-                            ///处理学生数据
-                            {
-                                List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
-                                studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
-                                studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal.lessonRecord.id);
-                                studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-                                studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
-                                studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-                                studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
-                                var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
-                                studentLessonDatas= pickupData.studentLessonDatas;
-                                count.pickup= pickupData.pickup ;
-                                //  var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId)  &&  !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) &&  x.lessonId.Equals(lessonLocal.lessonRecord.id));
-                                int sumUpload = 0;
-                                int taskCount = 0;
-                                int maxUpload = 0;
-                                HashSet<string> pickUp = new HashSet<string>();
-                                foreach (var stu in studentLessonDatas)
+                                if (lessonLocal?.lessonBase?.summary!=null)
+                                {
+                                    count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
+                                    count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
+                                    count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
+                                    count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
+                                    count.examCountBase=lessonLocal.lessonBase.summary.examCount;
+                                    count.interactNormalCountBase= count.interactNormalCount;
+                                }
+                                if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
+                                {
+                                    count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
+                                    count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
+                                    count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
+                                }
                                 {
-                                    var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
-                                    if (countS.Count()>0)
+
+                                    count.pickup= pickupData.pickup;
+                                    //  var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId)  &&  !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) &&  x.lessonId.Equals(lessonLocal.lessonRecord.id));
+                                    int sumUpload = 0;
+                                    int taskCount = 0;
+                                    int maxUpload = 0;
+                                    HashSet<string> pickUp = new HashSet<string>();
+                                    foreach (var stu in studentLessonDatas)
                                     {
-                                        int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
-                                        if (stuUploadmax> maxUpload)
+                                        var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
+                                        if (countS.Count()>0)
                                         {
-                                            maxUpload=stuUploadmax;
+                                            int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
+                                            if (stuUploadmax> maxUpload)
+                                            {
+                                                maxUpload=stuUploadmax;
+                                            }
                                         }
-                                    }
-                                    int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);
+                                        int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);
 
-                                    sumUpload+=stuUpload;
-                                    if (stu.taskRecord.itemRecords.Count()> taskCount)
-                                    {
-                                        taskCount=stu.taskRecord.itemRecords.Count();
-                                    }
-                                    var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
-                                    if (stu_scores!=null  && stu_scores.Count()>0)
-                                    {
-                                        count.stuCowork.AddRange(stu_scores);
+                                        sumUpload+=stuUpload;
+                                        if (stu.taskRecord.itemRecords.Count()> taskCount)
+                                        {
+                                            taskCount=stu.taskRecord.itemRecords.Count();
+                                        }
+                                        var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
+                                        if (stu_scores!=null  && stu_scores.Count()>0)
+                                        {
+                                            count.stuCowork.AddRange(stu_scores);
+                                        }
+                                        var grp_scores = stu.group_coworkScore.Where(x => x>0);
+                                        if (grp_scores!=null  && grp_scores.Count()>0)
+                                        {
+                                            count.groupCowork.AddRange(grp_scores);
+                                        }
+                                        //if (stu.pickups.IsNotEmpty())
+                                        //{
+                                        //    foreach (var pickup in stu.pickups)
+                                        //    {
+                                        //        pickUp.Add(pickup);
+                                        //    }
+                                        //}
                                     }
-                                    var grp_scores = stu.group_coworkScore.Where(x => x>0);
-                                    if (grp_scores!=null  && grp_scores.Count()>0)
+                                    if (studentLessonDatas.Count>0&& taskCount>0  && maxUpload>0)
                                     {
-                                        count.groupCowork.AddRange(grp_scores);
+                                        var avgUpload = sumUpload*1.0/(studentLessonDatas.Count *taskCount);
+                                        count.upload.Add(new List<double>() { avgUpload, maxUpload });
                                     }
-                                    //if (stu.pickups.IsNotEmpty())
+                                    //if (pickUp.Count>0)
                                     //{
-                                    //    foreach (var pickup in stu.pickups)
+                                    //    count.pickup.AddRange(pickUp.ToList());
+                                    //}
+                                    string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
+                                    await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
+                                    //if (force)
+                                    //{
+                                       
+                                    //}
+                                    //else
+                                    //{
+                                    //    if (lessonRecords.FindAll(x => x.id.Equals(lessonLocal.lessonRecord.id)).IsNotEmpty())
                                     //    {
-                                    //        pickUp.Add(pickup);
+                                    //        // await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json", studentLessonDatas.ToJsonString());
+                                    //        //只有不是强制更新的时候再去刷新线上的json文件。
+
+                                    //        await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
+                                    //        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
                                     //    }
                                     //}
                                 }
-                                if (studentLessonDatas.Count>0&& taskCount>0  && maxUpload>0)
-                                {
-                                    var avgUpload = sumUpload*1.0/(studentLessonDatas.Count *taskCount);
-                                    count.upload.Add(new List<double>() { avgUpload, maxUpload });
-                                }
-                                //if (pickUp.Count>0)
-                                //{
-                                //    count.pickup.AddRange(pickUp.ToList());
-                                //}
-                                string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
-                                // if (!azureStorage.GetBlobContainerClient(owner).GetBlobClient($"records/{lessonLocal.lessonRecord.id}/student-analysis.json").Exists())
-                                {
-
-
-                                }
-                                // if (!System.IO.File.Exists($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json"))
-                                {
-
-                                }
-                                if (force)
-                                {
-                                    await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
-                                }
-                                else 
-                                {
-                                    if (lessonRecords.FindAll(x => x.id.Equals(lessonLocal.lessonRecord.id)).IsNotEmpty())
-                                    {
-                                        // await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json", studentLessonDatas.ToJsonString());
-                                        //只有不是强制更新的时候再去刷新线上的json文件。
-                                      
-                                        await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
-                                        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
-                                    }
-                                }
-                               
                             }
                         }
                     }
 
-
-
                     if (count!=null)
                     {
                         yield return count;