TEAMModelOS/HTEX.DataETL/Controllers/LessonRecordController.cs

using DocumentFormat.OpenXml.Office2010.Excel;
using HTEX.Lib.ETL;
using HTEX.Lib.ETL.Lesson;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.Cosmos;
using Microsoft.OData.UriParser;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Xml;
using TEAMModelOS.SDK;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Helper.Common.FileHelper;
using TEAMModelOS.SDK.Models;
using TEAMModelOS.SDK.Models.Cosmos.BI;
using TEAMModelOS.SDK.Models.Cosmos.OpenEntity;

namespace HTEX.DataETL.Controllers
{
    [ApiController]
    [Route("lesson-record")]
    public class LessonRecordController : ControllerBase
    {
        private readonly ILogger<LessonRecordController> _logger;
        private readonly AzureCosmosFactory _azureCosmos;
        private readonly AzureStorageFactory _azureStorage;
        private readonly IConfiguration _configuration;
        private readonly IWebHostEnvironment _webHostEnvironment;

        public LessonRecordController(ILogger<LessonRecordController> logger, AzureCosmosFactory azureCosmos, AzureStorageFactory azureStorage
            , IConfiguration configuration, IWebHostEnvironment environment)
        {
            _logger = logger;
            _azureCosmos = azureCosmos;
            _azureStorage = azureStorage;
            _configuration = configuration;
            _webHostEnvironment = environment;
        }
        [HttpPost("process-local")]
        public async Task<IActionResult> ProcessLocal(JsonElement json)
        {
            List<StudentLessonData> studentLessonDatas = new List<StudentLessonData>();
            string? id = json.GetProperty("id").GetString();
            if (!string.IsNullOrWhiteSpace(id))
            {
                string? lessonPath= _configuration.GetValue<string>("LessonPath");
                string? path = $"{lessonPath}\\locals\\{id}";
                var files = FileHelper.ListAllFiles(path);
                // var sampleJson =System.IO. File.ReadAllTextAsync(path);
                LessonBase? lessonBase = null;
                List<LocalStudent> localStudents = new List<LocalStudent>();
                List<TaskData> taskDatas = new List<TaskData>();
                List<SmartRatingData> smartRatingDatas = new List<SmartRatingData>();
                List<IRSData> irsDatas = new List<IRSData>();
                List<CoworkData> coworkDatas = new List<CoworkData>();
                List<ExamData> examDatas = new List<ExamData>();
                TimeLineData? timeLineData = null;
                foreach (var item in files)
                {

                    if (item.Contains("IES\\base.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        jsons = jsons.Replace("\"Uncall\"", "0").Replace("Uncall", "0");
                        lessonBase = jsons.ToObject<LessonBase>();
                        var data = LessonETLService.GetBaseData(lessonBase);
                        localStudents = data.studentLessonDatas;
                    }

                    if (item.Contains("IES\\TimeLine.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        timeLineData = jsons.ToObject<TimeLineData>();
                    }
                    if (item.Contains("IES\\Task.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        taskDatas = jsons.ToObject<List<TaskData>>();
                    }

                    if (item.Contains("IES\\SmartRating.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        smartRatingDatas = jsons.ToObject<List<SmartRatingData>>();
                    }

                    if (item.Contains("IES\\IRS.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        irsDatas = jsons.ToObject<List<IRSData>>();
                    }
                    if (item.Contains("IES\\Cowork.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        coworkDatas = jsons.ToObject<List<CoworkData>>();
                    }
                    try
                    {
                        if (item.Contains($"\\{id}\\Exam\\") && item.EndsWith("Exam.json"))
                        {
                            string examsFile = item;
                            if (examsFile.EndsWith("Exam.json"))
                            {
                                ExamData? examData = null;
                                string jsons = await System.IO.File.ReadAllTextAsync(item);
                                jsons = jsons.Replace("\"publish\": \"0\"", "\"publish\": 0").Replace("\"publish\": \"1\"", "\"publish\": 1");
                                examData = jsons.ToObject<ExamData>();

                                if (examData != null && examData.exam.papers.IsNotEmpty())
                                {
                                    string paperId = examData.exam.papers.First().id;

                                    string paperPath = $"{path}\\ExamPaper\\{paperId}\\index.json";
                                    string jsonp = await System.IO.File.ReadAllTextAsync(paperPath);
                                    LessonPaper lessonPaper = jsonp.ToObject<LessonPaper>();
                                    examData.paper = lessonPaper;
                                    examDatas.Add(examData);
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, ex.Message);
                    }
                }
                if (lessonBase!=null  && timeLineData!=null)
                {
                    studentLessonDatas = localStudents.ToJsonString().ToObject<List<StudentLessonData>>();
                    studentLessonDatas = LessonETLService.GetIRSData(lessonBase, timeLineData, irsDatas, studentLessonDatas, examDatas, id);
                    studentLessonDatas = LessonETLService.GetCoworkData(lessonBase, timeLineData, coworkDatas, studentLessonDatas,id);
                    studentLessonDatas = LessonETLService.GetExamData(lessonBase, timeLineData, examDatas, studentLessonDatas, Constant.objectiveTypes, id);
                    studentLessonDatas = LessonETLService.GetSmartRatingData(lessonBase, timeLineData, smartRatingDatas, studentLessonDatas, id);
                    studentLessonDatas = LessonETLService.GetTaskData(lessonBase, timeLineData, taskDatas, studentLessonDatas,id);
                    await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString());
                    string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis.json");
                    LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
                    var lessonItems=  LessonETLService.ProcessStudentDataV1(studentLessonDatas, lessonDataAnalysis);
                    XmlDocument xmlDocument = new XmlDocument();
                    var runtimePath=  System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
                    xmlDocument.Load($"{runtimePath}\\summary.xml");
                    await  LessonETLService. ExportToExcel(lessonItems, $"{path}\\analysis.xlsx", xmlDocument);
                }
            }
            return Ok();
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="json"></param>
        /// <returns></returns>
        [HttpPost("process-history-students")]
        public async Task<IActionResult> ProcessHistoryStudents(JsonElement json)
        {
            string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
            string? pathLessons = $"{lessonBasePath}\\lessons";
            string? pathAnalysis = $"{lessonBasePath}\\analysis";
            string jsons = await System.IO.File.ReadAllTextAsync($"{pathAnalysis}\\analysis.json");
            LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
            List<string>  filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
            var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
            XmlDocument xmlDocument = new XmlDocument();
            xmlDocument.Load($"{runtimePath}\\summary.xml");
            List<string> noStujson = new List<string>();
            foreach (string fileLesson in filesLessons) 
            {
                try {
                    var stuFile = fileLesson.Replace("-local.json", "-stu.json");
                    var excleFile = fileLesson.Replace("-local.json", "-stu.xlsx");
                    if (System.IO.File.Exists(stuFile))
                    {
                        string stuJson = await System.IO.File.ReadAllTextAsync(stuFile);
                        List<StudentLessonData> studentLessonDatas = stuJson.ToObject<List<StudentLessonData>>();
                        var lessonItems = LessonETLService.ProcessStudentDataV1(studentLessonDatas, lessonDataAnalysis);
                        await LessonETLService.ExportToExcel(lessonItems, excleFile, xmlDocument);
                    }
                    else
                    {
                        noStujson.Add(stuFile);
                    }
                }
                catch (Exception ex) {
                   throw new Exception($"{fileLesson},{ex.Message}",ex);
                }
            }
            return Ok(noStujson);
        }



        /// <summary>
        /// 课例数据ETL处理过程
        /// </summary>
        /// <param name="json"></param>
        /// <returns></returns>
        [HttpPost("process-history")]
        public async Task<IActionResult> ProcessHistory(JsonElement json) 
        {
            
            List<string> localIds = new List<string>();
            string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
            string? pathLessons = $"{lessonBasePath}\\lessons";
            string? pathAnalysis = $"{lessonBasePath}\\analysis";
            var filesLessons = FileHelper.ListAllFiles(pathLessons);
            foreach (var file in filesLessons)
            {
                if (file.EndsWith("-local.json"))
                {
                    string lessonId = file.Split("\\").Last().Replace("-local.json", "");
                    localIds.Add(lessonId);
                }
            }
            bool loadLocal = true;
            List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
            var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
            long stime = 1690819200000;//2023-08-01 00:00:00
            foreach (var file in filesAnalysis) {
                //读取每月的数据
                if (file.EndsWith("-m-analysis.json"))
                {
                    string jsons = await System.IO.File.ReadAllTextAsync(file);
                    LessonDataAnalysisMonth lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisMonth>();
                    lessonDataAnalysisMonths.Add(lessonDataAnalysis);
                }
            }
            if (lessonDataAnalysisMonths.IsNotEmpty()) {
                var maxUpdateTime = lessonDataAnalysisMonths.Max(x => x.updateTime);
                if (maxUpdateTime>0)
                {
                    //更新周期是一周
                    if (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()- maxUpdateTime>604800000)
                    {
                        stime=maxUpdateTime;
                        loadLocal =true;
                    }
                    else
                    {
                        stime=maxUpdateTime;
                        loadLocal=false;
                    }
                }
            }
            HashSet<string> yearMonth = new HashSet<string>();
            long newest = 0;
            if (loadLocal ||(json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
            {
                List<LessonRecord> lessonRecords = new List<LessonRecord>();
                var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
                    .GetList<LessonRecord>($"SELECT value c FROM   c  where c.startTime>={stime} and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' and c.school<>'hbcn' and c.school<>'habook' and (c.tLevel>0 or c.pLevel>0) ", null);
                if (resultSchool.list.IsNotEmpty())
                {
                    newest= resultSchool.list.Max(x => x.startTime);
                    lessonRecords.AddRange(resultSchool.list);
                }
                else {
                    newest=stime;
                }
                var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
                   .GetList<LessonRecord>($"SELECT value c FROM   c  where c.startTime>={stime} and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord'  and (c.tLevel>0 or c.pLevel>0) ", null);
                if (resultTeacher.list.IsNotEmpty())
                {
                    long max = resultTeacher.list.Max(x => x.startTime);
                    if (max<newest)
                    {
                        newest=max;
                    }
                    lessonRecords.AddRange(resultTeacher.list);
                }
                List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
                if (lessonRecords.IsNotEmpty())
                {
                  
                    await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
                    {
                        string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
                        if (item.lessonBase!=null && item.lessonBase.student!=null)
                        {
                            TechCount techCount = new TechCount
                            {
                                lessonId=item.lessonRecord?.id,
                                examCount = item.examDatas.Count,
                                taskCount = item.taskDatas.Count,
                                irsCount = item.irsDatas.Count,
                                coworkCount = item.coworkDatas.Count,
                                smartRatingCount =item.smartRatingDatas.Count,
                                timeCount=item.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList()
                            };
                            await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord.id}-count.json", techCount.ToJsonString());
                            await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
                        }
                        else
                        {

                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
                            System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
                        }
                    }
                }
                List<TechCount> techCounts = new List<TechCount>();
                filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
                await foreach (var item in  GetTeachCount(lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage))
                {
                    techCounts.Add(item);
                }
                var yearMonthDatas = techCounts.GroupBy(x => x.yearMonth).Select(x => new { key = x.Key, list = x.ToList() });
               // lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
                LessonDataAnalysisCluster lessonDataAnalysisCluster = new LessonDataAnalysisCluster();
                foreach (var yearMonthData in yearMonthDatas) 
                {
                    if (string.IsNullOrWhiteSpace(yearMonthData.key)) 
                    {
                        Console.WriteLine(yearMonthData.list.Select(x => x.lessonId).ToJsonString());
                    }
                    LessonDataAnalysisMonth lessonDataAnalysisMonth= new LessonDataAnalysisMonth() { updateTime= newest, yearMonth= yearMonthData.key };
                    lessonDataAnalysisMonth.pscore  = yearMonthData.list.SelectMany(x => x.pscore).ToList();
                    lessonDataAnalysisMonth.tscore  = yearMonthData.list.SelectMany(x => x.tscore).ToList();
                    lessonDataAnalysisMonth.gscore  = yearMonthData.list.SelectMany(x => x.gscore).ToList();
                    lessonDataAnalysisMonth.irs= yearMonthData.list.Where(x => x.irsCount>0).Select(x =>(double)x.irsCount).ToList();
                    lessonDataAnalysisMonth.interactNormal= yearMonthData.list.Where(x => x.interactNormalCount > 0).Select(x => (double)x.interactNormalCount).ToList();
                    lessonDataAnalysisMonth.task = yearMonthData.list.Where(x => x.taskCount > 0).Select(x => (double)x.taskCount).ToList();
                    lessonDataAnalysisMonth.stuCowork=yearMonthData.list.SelectMany(x => x.stuCowork).ToList();
                    lessonDataAnalysisMonth.groupCowork=yearMonthData.list.SelectMany(x => x.groupCowork).ToList();
                    System.IO.File.WriteAllText(Path.Combine(pathAnalysis, $"{yearMonthData.key}-m-analysis.json"), lessonDataAnalysisMonth.ToJsonString());
                    // lessonDataAnalysisMonths.Add( lessonDataAnalysisMonth);
                    if (lessonDataAnalysisMonth.task.IsNotEmpty()) 
                    {
                        lessonDataAnalysisCluster.task.AddRange(lessonDataAnalysisMonth.task);
                    }
                    if (lessonDataAnalysisMonth.irs.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.irs.AddRange(lessonDataAnalysisMonth.irs);
                    }
                    if (lessonDataAnalysisMonth.interactNormal.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.interactNormal.AddRange(lessonDataAnalysisMonth.interactNormal);
                    }
                    if (lessonDataAnalysisMonth.pscore.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.pscore.AddRange(lessonDataAnalysisMonth.pscore);
                    }
                    if (lessonDataAnalysisMonth.gscore.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.gscore.AddRange(lessonDataAnalysisMonth.gscore);
                    }
                    if (lessonDataAnalysisMonth.tscore.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.tscore.AddRange(lessonDataAnalysisMonth.tscore);
                    }
                    if (lessonDataAnalysisMonth.stuCowork.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.stuCowork.AddRange(lessonDataAnalysisMonth.stuCowork);
                    }
                    if (lessonDataAnalysisMonth.groupCowork.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.groupCowork.AddRange(lessonDataAnalysisMonth.groupCowork);
                    }
                }

                //标准差偏差N倍，视为异常数据
                int thresholdMultiplier = 2;
                lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
                lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
                lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
                lessonDataAnalysisCluster.irs = LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.irs.OrderBy(x => x), thresholdMultiplier);
                lessonDataAnalysisCluster.interactNormal=LessonETLService. CleanDataBySDThreshold(lessonDataAnalysisCluster.interactNormal.OrderBy(x => x), thresholdMultiplier);
                lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x=>x), thresholdMultiplier);
                lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
                var d = lessonDataAnalysisCluster.interactNormal.Select(x => x).OrderBy(x => x).ToArray();
                //  设置0 为自动规划聚类，11 则为自动规划后得到的数字。
                var clusterInteract = KMeansService.KMeansOptimized(d,11,10);
                //foreach (var item in clusterInteract)
                //{
                //    Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()}");
                //}
                foreach (var item in clusterInteract.OrderBy(x => x.Key))
                {
                    lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
                    //Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
                }
                System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
            }
            return Ok(new { yearMonth });
        }

        private static async IAsyncEnumerable<TechCount> GetTeachCount(List<LessonRecord> lessonRecords,List<string> filesLessons, string pathLessons,  List<string> ignore, List<string> objectiveTypes,AzureStorageFactory azureStorage)
        {
            foreach (var item in filesLessons) 
            {
                if (item.EndsWith("-local.json"))
                {
                    TechCount count = null;

                    string lessonId = item.Split("\\").Last().Replace("-local.json", "");
                    string countFile = item.Replace("-local.json", "-count.json");
                    if (System.IO.File.Exists(countFile))
                    {
                        if (System.IO.File.Exists(countFile))
                        {
                            string contjson = await System.IO.File.ReadAllTextAsync(countFile);
                            count = contjson.ToObject<TechCount>();
                        }
                    }
                    if (string.IsNullOrWhiteSpace(count?.yearMonth))
                    {
                        string localjson = await System.IO.File.ReadAllTextAsync(item);
                        var lessonLocal = localjson.ToObject<LessonLocal>();
                        count = new TechCount();
                        count.lessonId=item.Split("\\").Last().Replace("-local.json", "");
                        count.examCount= lessonLocal.examDatas.Count;
                        count.taskCount= lessonLocal.taskDatas.Count;
                        count.irsCount= lessonLocal.irsDatas.Count;
                        count.interactNormalCount=count.irsCount;
                        count.coworkCount= lessonLocal.coworkDatas.Count;
                        count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
                        count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event)  &&  !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
                        
                        
                        if (lessonLocal.lessonRecord!=null)
                        {

                            count.yearMonth=DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
                           
                            if (lessonLocal?.lessonBase?.summary!=null)
                            {
                                count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
                                count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
                                count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
                                count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
                                count.examCountBase=lessonLocal.lessonBase.summary.examCount;
                                count.interactNormalCountBase= count.interactNormalCount;
                            }
                            if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
                            {
                                count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
                                count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
                                count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
                            }
                            ///处理学生数据
                            {
                                List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
                                studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal.lessonRecord.id);
                                studentLessonDatas =  LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
                                studentLessonDatas =  LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
                                studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
                                studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
                                //  var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId)  &&  !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) &&  x.lessonId.Equals(lessonLocal.lessonRecord.id));
                                int sumUpload = 0;
                                int taskCount = 0;
                                int maxUpload = 0;
                                //HashSet<string> pickUp = new HashSet<string>();
                                foreach (var stu in studentLessonDatas)
                                {
                                    var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
                                    if (countS.Count()>0)
                                    {
                                        int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
                                        if (stuUploadmax> maxUpload)
                                        {
                                            maxUpload=stuUploadmax;
                                        }
                                    }
                                    int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);

                                    sumUpload+=stuUpload;
                                    if (stu.taskRecord.itemRecords.Count()> taskCount)
                                    {
                                        taskCount=stu.taskRecord.itemRecords.Count();
                                    }
                                    var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
                                    if (stu_scores!=null  && stu_scores.Count()>0)
                                    {
                                        count.stuCowork.AddRange(stu_scores);
                                    }
                                    var grp_scores = stu.group_coworkScore.Where(x => x>0);
                                    if (grp_scores!=null  && grp_scores.Count()>0)
                                    {
                                        count.groupCowork.AddRange(grp_scores);
                                    }
                                    //if (stu.pickups.IsNotEmpty())
                                    //{
                                    //    foreach (var pickup in stu.pickups)
                                    //    {
                                    //        pickUp.Add(pickup);
                                    //    }
                                    //}
                                }
                                if (studentLessonDatas.Count>0&& taskCount>0  && maxUpload>0)
                                {
                                    var avgUpload = sumUpload*1.0/(studentLessonDatas.Count *taskCount);
                                    count.upload.Add(new List<double>() { avgUpload, maxUpload });
                                }
                                //if (pickUp.Count>0)
                                //{
                                //    count.pickup.Add(pickUp.ToList());
                                //}
                                string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
                                // if (!azureStorage.GetBlobContainerClient(owner).GetBlobClient($"records/{lessonLocal.lessonRecord.id}/student-analysis.json").Exists())
                                {


                                }
                                // if (!System.IO.File.Exists($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json"))
                                {

                                }
                                if (lessonRecords.FindAll(x => x.id.Equals(lessonLocal.lessonRecord.id)).IsNotEmpty())
                                {
                                    await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json", studentLessonDatas.ToJsonString());
                                    await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
                                    await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
                                }
                            }
                        }
                    }

                    
                
                    if (count!=null) 
                    {
                        yield return count;
                    }
                }
            }
           
        }
    }
}