TEAMModelOS/HTEX.DataETL/Controllers/LessonRecordController.cs

using DocumentFormat.OpenXml.Drawing.Charts;
using DocumentFormat.OpenXml.Spreadsheet;
using HTEX.Lib.ETL.Lesson;
using MathNet.Numerics.Distributions;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.Cosmos;
using Microsoft.Extensions.Logging;
using StackExchange.Redis;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Reflection;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Xml;
using TEAMModelOS.SDK;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Helper.Common.FileHelper;
using TEAMModelOS.SDK.Models;
using TEAMModelOS.SDK.Models.Cosmos.BI;
using TEAMModelOS.SDK.Models.Cosmos.Common;
using TEAMModelOS.SDK.Models.Cosmos.OpenEntity;

namespace HTEX.DataETL.Controllers
{
    [ApiController]
    [Route("lesson-record")]
    public class LessonRecordController : ControllerBase
    {
        private readonly ILogger<LessonRecordController> _logger;
        private readonly AzureCosmosFactory _azureCosmos;
        private readonly AzureStorageFactory _azureStorage;
        private readonly IConfiguration _configuration;
        private readonly IWebHostEnvironment _webHostEnvironment;
        private readonly DingDing  _dingDing;
        private readonly AzureRedisFactory _azureRedis;



        public LessonRecordController(ILogger<LessonRecordController> logger, AzureCosmosFactory azureCosmos, AzureStorageFactory azureStorage   , IConfiguration configuration, IWebHostEnvironment environment,DingDing dingDing,AzureRedisFactory azureRedis )
        {
            _logger = logger;
            _azureCosmos = azureCosmos;
            _azureStorage = azureStorage;
            _configuration = configuration;
            _webHostEnvironment = environment;
            _dingDing = dingDing;
            _azureRedis = azureRedis;

        }
         
        [HttpPost("process-local")]
        public async Task<IActionResult> ProcessLocal(JsonElement json)
        {
            List<StudentLessonData> studentLessonDatas = new List<StudentLessonData>();
            string? id = json.GetProperty("id").GetString();
            if (!string.IsNullOrWhiteSpace(id))
            {
                string? lessonPath = _configuration.GetValue<string>("LessonPath");
                string? path = $"{lessonPath}\\locals\\{id}";
                var files = FileHelper.ListAllFiles(path);
                // var sampleJson =System.IO. File.ReadAllTextAsync(path);
                LessonBase? lessonBase = null;
                List<LocalStudent> localStudents = new List<LocalStudent>();
                List<TaskData> taskDatas = new List<TaskData>();
                List<SmartRatingData> smartRatingDatas = new List<SmartRatingData>();
                List<IRSData> irsDatas = new List<IRSData>();
                List<CoworkData> coworkDatas = new List<CoworkData>();
                List<ExamData> examDatas = new List<ExamData>();
                TimeLineData? timeLineData = null;
                foreach (var item in files)
                {

                    if (item.Contains("IES\\base.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        jsons = jsons.Replace("\"Uncall\"", "0").Replace("Uncall", "0");
                        lessonBase = jsons.ToObject<LessonBase>();
                        var data = LessonETLService.GetBaseData(lessonBase);
                        localStudents = data.studentLessonDatas;
                    }

                    if (item.Contains("IES\\TimeLine.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        timeLineData = jsons.ToObject<TimeLineData>();
                    }
                    if (item.Contains("IES\\Task.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        taskDatas = jsons.ToObject<List<TaskData>>();
                    }

                    if (item.Contains("IES\\SmartRating.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        smartRatingDatas = jsons.ToObject<List<SmartRatingData>>();
                    }

                    if (item.Contains("IES\\IRS.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        irsDatas = jsons.ToObject<List<IRSData>>();
                    }
                    if (item.Contains("IES\\Cowork.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        coworkDatas = jsons.ToObject<List<CoworkData>>();
                    }
                    try
                    {
                        if (item.Contains($"\\{id}\\Exam\\") && item.EndsWith("Exam.json"))
                        {
                            string examsFile = item;
                            if (examsFile.EndsWith("Exam.json"))
                            {
                                ExamData? examData = null;
                                string jsons = await System.IO.File.ReadAllTextAsync(item);
                                jsons = jsons.Replace("\"publish\": \"0\"", "\"publish\": 0").Replace("\"publish\": \"1\"", "\"publish\": 1");
                                examData = jsons.ToObject<ExamData>();

                                if (examData != null && examData.exam.papers.IsNotEmpty())
                                {
                                    string paperId = examData.exam.papers.First().id;

                                    string paperPath = $"{path}\\ExamPaper\\{paperId}\\index.json";
                                    string jsonp = await System.IO.File.ReadAllTextAsync(paperPath);
                                    LessonPaper lessonPaper = jsonp.ToObject<LessonPaper>();
                                    examData.paper = lessonPaper;
                                    examDatas.Add(examData);
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, ex.Message);
                    }
                }
                if (lessonBase!=null  && timeLineData!=null)
                {
                    studentLessonDatas = localStudents.ToJsonString().ToObject<List<StudentLessonData>>();
                    studentLessonDatas = LessonETLService.GetBaseInfo(lessonBase!, studentLessonDatas, id);
                    studentLessonDatas = LessonETLService.GetIRSData(lessonBase, timeLineData, irsDatas, studentLessonDatas, examDatas, id);
                    studentLessonDatas = LessonETLService.GetCoworkData(lessonBase, timeLineData, coworkDatas, studentLessonDatas, id);
                    studentLessonDatas = LessonETLService.GetExamData(lessonBase, timeLineData, examDatas, studentLessonDatas, Constant.objectiveTypes, id);
                    studentLessonDatas = LessonETLService.GetSmartRatingData(lessonBase, timeLineData, smartRatingDatas, studentLessonDatas, id);
                    studentLessonDatas = LessonETLService.GetTaskData(lessonBase, timeLineData, taskDatas, studentLessonDatas, id);
                    var pickupData = LessonETLService.GetPickupData(lessonBase, timeLineData, studentLessonDatas, id);
                    studentLessonDatas= pickupData.studentLessonDatas;
                    var codeBools= LessonETLService.GetCodeBools(studentLessonDatas);
                    await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString());
                    string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis-model.json");
                    LessonDataAnalysisModel lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisModel>();
                    

                    var studentLessons = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis,codeBools);
                    XmlDocument xmlDocument = new XmlDocument();
                    var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
                    xmlDocument.Load($"{runtimePath}\\summary.xml");
                    await LessonETLService.ExportToExcelLocal(studentLessons, $"{path}\\analysis.xlsx", xmlDocument);
                }
            }
            return Ok();
        }

        
        /// <summary>
        /// 
        /// </summary>
        /// <param name="json"></param>
        /// <returns></returns>
        [HttpPost("process-history-students")]
        public async Task<IActionResult> ProcessHistoryStudents(JsonElement json)
        {
            string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
            string? pathLessons = $"{lessonBasePath}\\lessons";
            string? pathAnalysis = $"{lessonBasePath}\\analysis";
            string jsons = await System.IO.File.ReadAllTextAsync($"{pathAnalysis}\\analysis-model.json");
            LessonDataAnalysisModel lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisModel>();
            List<string> filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
            List<string> filesStudata = FileHelper.ListAllFiles(pathLessons, "-sdata.json");
            var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
            XmlDocument xmlDocument = new XmlDocument();
            xmlDocument.Load($"{runtimePath}\\summary.xml");
            List<School> schools = new List<School>();
           // await Parallel.ForEachAsync(filesLessons, async (fileLesson, _) =>
            List<string> localIds = new List<string>();
            foreach (var file in filesLessons)
            {
                string lessonId = file.Split("\\").Last().Replace("-local.json", "");
                localIds.Add(lessonId);
            }
            List<LessonLocal> lessonLocals = new List<LessonLocal>();
            
            List<LessonRecord> lessonRecords = new List<LessonRecord>();
            string recordsPtah = $"{lessonBasePath}\\records\\records.json";
            long stime = 0;//2023-09-01 00:00:00
            long etime = 0;//2024-11-13 23:59:59
            //if (System.IO.File.Exists(recordsPtah))
            //{
            //    string jsonData = await System.IO.File.ReadAllTextAsync(recordsPtah);
            //    lessonRecords= jsonData.ToObject<List<LessonRecord>>();
            //    stime= lessonRecords.Max(x => x.startTime);
            //    etime= lessonRecords.Max(x => x.startTime);
            //}
            if (stime==0) 
            {
                stime = 1693497600000;//2023-09-01 00:00:00
            }
            if (etime==0) 
            {
                etime = 1731513599000;//2024-11-13 23:59:59
            }
            List<LessonLocal> locals=  await LessonETLService.FixLocalData(localIds, _azureCosmos, _azureStorage, pathLessons, stime, etime);
            if (locals.IsNotEmpty()) 
            {
                lessonLocals.AddRange(locals);
            }

            int u = 0;
            Parallel.ForEach(filesLessons, file =>
            {
                string jsonp = System.IO.File.ReadAllText(file);
                var lessonLocal = jsonp.ToObject<LessonLocal>();
                if (lessonLocal.lessonBase!=null && lessonLocal.lessonBase.student.IsNotEmpty())
                {
                    lessonLocals.Add(lessonLocal);
                }
                else
                {
                    System.IO.File.Delete(file);
                    System.IO.File.Delete(file.Replace("-local.json", "-count.json"));
                    u++;
                }
            });
            List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
            if (lessonLocals.IsNotEmpty()) 
            {
                lessonRecords=lessonLocals.Where(x => x.lessonRecord.scope.Equals("school")).Select(x =>x.lessonRecord).ToList();
            }
            if (lessonRecords.IsNotEmpty())
            {
                await System.IO.File.WriteAllTextAsync(recordsPtah, lessonRecords.ToJsonString());
                List<StudentSemesterRecord>  studentSemesterRecords= new List<StudentSemesterRecord>();
                List<OverallEducation> overallEducations= new List<OverallEducation>();
                List<Student> studentsBase = new List<Student>();
                string schoolPtah = $"{lessonBasePath}\\schools\\school.json";
                if (System.IO.File.Exists(schoolPtah))
                {
                    string jsonData = await System.IO.File.ReadAllTextAsync(schoolPtah);
                    schools= jsonData.ToObject<List<School>>();
                }
                var schoolGroup = lessonLocals.Where(x => !string.IsNullOrWhiteSpace(x.lessonRecord?.school)).GroupBy(x => x.lessonRecord?.school).Select(x => new { key = x.Key, list = x.ToList() });
                var newschoolIds = schoolGroup.Select(x=>x.key).ExceptBy(schools.Select(x => x.id), x => x);
                if (newschoolIds!=null && newschoolIds.Count()>0) 
                {
                    string schoolSql = $"select value c from c where c.id in ({string.Join(",", schoolGroup.Select(x => $"'{x.key}'"))})";
                    var schoolResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).GetList<School>(schoolSql, "Base");
                    if (schoolResults.list.IsNotEmpty())
                    {
                        schools.AddRange(schoolResults.list);
                    }
                }
                await System.IO.File.WriteAllTextAsync($"{lessonBasePath}\\schools\\school.json", schools.ToJsonString());
                foreach (var group in schoolGroup) 
                {
                    string students_path = $"{lessonBasePath}\\students\\{group.key}\\students.json";
                    string records_path = $"{lessonBasePath}\\students\\{group.key}\\records.json";
                    string overall_path = $"{lessonBasePath}\\students\\{group.key}\\overall.json";
                    if (!Directory.Exists($"{lessonBasePath}\\students\\{group.key}")) 
                    {
                        Directory.CreateDirectory($"{lessonBasePath}\\students\\{group.key}");
                    }
                    var studentIds = group.list.SelectMany(x => x.studentLessonDatas).Where(x=>!string.IsNullOrWhiteSpace(x.id)).Select(x => x.id).Distinct();
                    List<Student> schoolStudent = new List<Student>();
                    if (System.IO.File.Exists(students_path)) 
                    {
                        string jsonData = await System.IO.File.ReadAllTextAsync(students_path);
                        schoolStudent= jsonData.ToObject<List<Student>>();
                        studentsBase.AddRange(schoolStudent);
                    }
                    var newIds= studentIds.ExceptBy(schoolStudent.Where(x=>x.schoolId.Equals(group.key)).Select(x=>x.id),x=>x);
                    if (newIds!=null  && newIds.Count()>0)
                    {
                        string studentSql = $"select value c from c where c.id in ({string.Join(",", newIds.Select(x => $"'{x}'"))})";
                        var studentResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<Student>(studentSql, $"Base-{group.key}");
                        if (studentResults.list.IsNotEmpty())
                        {
                            schoolStudent.AddRange(studentResults.list);
                            studentsBase.AddRange(studentResults.list);
                        }
                    }
                    await System.IO.File.WriteAllTextAsync(students_path, schoolStudent.ToJsonString());

                    List<StudentSemesterRecord>  schoolStudentSemesterRecords=new List<StudentSemesterRecord>();
                    if (System.IO.File.Exists(records_path))
                    {
                        string jsonData = await System.IO.File.ReadAllTextAsync(records_path);
                        schoolStudentSemesterRecords= jsonData.ToObject<List<StudentSemesterRecord>>();
                        studentSemesterRecords.AddRange(schoolStudentSemesterRecords);
                    }
                    var newstuIds = studentIds.ExceptBy(studentSemesterRecords.Where(x => x.school.Equals(group.key)).Select(x => x.stuid), x => x);
                    if (newstuIds!=null  && newstuIds.Count()>0) 
                    {
                        string studentSemesterRecordSql = $"select value c from c where c.stuid in ({string.Join(",", newstuIds.Distinct().Select(x => $"'{x}'"))}) and c.studyYear>=2023";
                        var studentSemesterRecordResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<StudentSemesterRecord>(studentSemesterRecordSql, $"StudentSemesterRecord-{group.key}");
                        if (studentSemesterRecordResults.list.IsNotEmpty())
                        {
                            schoolStudentSemesterRecords.AddRange(studentSemesterRecordResults.list);
                            studentSemesterRecords.AddRange(studentSemesterRecordResults.list);
                        }
                    }
                   
                    await System.IO.File.WriteAllTextAsync(records_path, schoolStudentSemesterRecords.ToJsonString());


                    List<OverallEducation> schoolOverallEducations = new List<OverallEducation>();

                    if (System.IO.File.Exists(overall_path))
                    {
                        string jsonData = await System.IO.File.ReadAllTextAsync(overall_path);
                        schoolOverallEducations= jsonData.ToObject<List<OverallEducation>>();
                        overallEducations.AddRange(schoolOverallEducations);
                    }
                    var newstuoIds = studentIds.ExceptBy(schoolOverallEducations.Where(x => x.schoolCode.Equals(group.key)).Select(x => x.studentId), x => x);
                    if (newstuIds!=null  && newstuIds.Count()>0) 
                    {
                        string overallEducationSql = $"select value c from c where c.studentId in ({string.Join(",", newstuoIds.Select(x => $"'{x}'"))}) and c.year>=2023";
                        var overallEducationResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<OverallEducation>(overallEducationSql, $"OverallEducation-{group.key}");
                        if (overallEducationResults.list.IsNotEmpty())
                        {
                            schoolOverallEducations.AddRange(overallEducationResults.list);
                            overallEducations.AddRange(overallEducationResults.list);
                        }
                    }
                    await System.IO.File.WriteAllTextAsync(overall_path, schoolOverallEducations.ToJsonString());
                }
                
                //List<(string id,string owner, List<StudentLessonData> studentLessonData)> studentLessonDatas= new List<(string id, string owner, List<StudentLessonData>)>();
                ConcurrentQueue<LessonStudentResult> lessonItems = new ConcurrentQueue<LessonStudentResult>();
                int v = 0;
                Parallel.ForEach(filesStudata, file =>
                {
                    string jsonp = System.IO.File.ReadAllText(file);
                    var studentResult = jsonp.ToObject<LessonStudentResult>();
                    lessonItems.Enqueue(studentResult);
                    v++;
                });

                int n = 0;
                lessonLocals= lessonLocals.ExceptBy(lessonItems.Select(x => x.id), y => y.lessonRecord.id).ToList();
                Parallel.ForEach(lessonLocals, (item, con) =>
                {
                    try
                    {
                        var studata = LessonETLService.DoStudentLessonDataV2(Constant.objectiveTypes, item, "China", studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
                        if (studata.codeBools.FindAll(x => x.value).IsNotEmpty())
                        {
                            string owner = item.lessonRecord.scope.Equals("school") ? item.lessonRecord.school : item.lessonRecord.tmdid;
                            //studentLessonDatas.Add((item.lessonRecord.id, owner, studata.studentLessonDatas));
                            LessonStudentResult result = new LessonStudentResult { id= item.lessonRecord.id, owner= owner, studentLessons= studata.lessonItems, codeBools= studata.codeBools, lessonDatas= studata.studentLessonDatas };
                            string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
                            System.IO.File.WriteAllText($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-sdata.json", item.ToJsonString());
                            lessonItems.Enqueue(result);
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine($"{ex.Message},{ex.StackTrace}");
                        throw new Exception(ex.Message, ex);
                    }
                    n++;
                });
                foreach (var group in schoolGroup)
                {
                    string students_path = $"{lessonBasePath}\\students\\{group.key}\\students.json";
                    string records_path = $"{lessonBasePath}\\students\\{group.key}\\records.json";
                    string overall_path = $"{lessonBasePath}\\students\\{group.key}\\overall.json";
                    if (!Directory.Exists($"{lessonBasePath}\\students\\{group.key}"))
                    {
                        Directory.CreateDirectory($"{lessonBasePath}\\students\\{group.key}");
                    }
                    var schoolStudent = studentsBase.FindAll(x => x.schoolId.Equals(group.key));
                    var schoolStudentSemesterRecords=  studentSemesterRecords.FindAll(x => x.school.Equals(group.key));
                    var schoolOverallEducations = overallEducations.FindAll(x => x.schoolCode.Equals(group.key));
                    await System.IO.File.WriteAllTextAsync(students_path, schoolStudent.ToJsonString());
                    await System.IO.File.WriteAllTextAsync(records_path, schoolStudentSemesterRecords.ToJsonString());
                    await System.IO.File.WriteAllTextAsync(overall_path, schoolOverallEducations.ToJsonString());
                }
                int m = 0;
                var grpdata = studentSemesterRecords.GroupBy(x => x.code).Select(x => new { key = x.Key, list = x.ToList() });
                foreach (var group in grpdata)
                {
                    var pages = group.list.Page(100);
                   
                    foreach (var page in pages)
                    {
                        List<Task<ItemResponse<StudentSemesterRecord>>> list = new();
                        foreach (var studentSemester in page) 
                        {
                            list.Add(_azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code)))  ;
                            m++;
                        }
                       await Task.WhenAll(list);
                    }
                }
                int k = 0;
                var gpover = overallEducations.GroupBy(x => x.code).Select(x => new { key = x.Key, list = x.ToList() });
                foreach (var item in gpover)
                {
                    var pages = item.list.Page(100);
                    foreach (var page in pages)
                    {
                        List<Task<ItemResponse<OverallEducation>>> list = new();
                        foreach (var overallEducation in page) 
                        {
                            list.Add(_azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code)));
                             
                            //string key = $"OverallEducation:{overallEducation.schoolCode}:{overallEducation.periodId}:{overallEducation.year}:{overallEducation.semesterId}:{overallEducation?.classId}";
                            //await _azureRedis.GetRedisClient(8).HashSetAsync(key, overallEducation.studentId, overallEducation.ToJsonString());
                            //await _azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
                          
                        }
                       await Task.WhenAll(list);
                    }
                }

                int p = 0;
                // 获取类的属性
                PropertyInfo[] properties = typeof(StudentLessonItem).GetProperties();
                List<string> summaryes= new List<string>();
                for (int i = 0; i < properties.Length; i++)
                {
                    string summary = Regex.Replace(LessonETLService.GetPropertySummary(properties[i], xmlDocument), @"\s+", "");
                    summaryes.Add(summary);
                }
                await Parallel.ForEachAsync(lessonItems, async (lessonItem, _) => {
                    await _azureStorage.GetBlobContainerClient(lessonItem.owner).UploadFileByContainer(lessonItem.lessonDatas.ToJsonString(), "records", $"{lessonItem.id}/student-analysis.json");
                    await LessonETLService.ExportToExcelAzureBlob(lessonItem.studentLessons, _azureStorage, lessonItem.owner, $"{lessonItem.id}/student-analysis.xlsx", xmlDocument, summaryes, properties);
                    p++;
                });
                return Ok(new { p, m, k,n,u});
            }
            return Ok(new {});
        }

        

        /// <summary>
        /// 课例数据ETL处理过程
        /// </summary>
        /// <param name="json"></param>
        /// <returns></returns>
        [HttpPost("process-history")]
        public async Task<IActionResult> ProcessHistory(JsonElement json)
        {
            Stopwatch stopwatch= new Stopwatch();
            List<string> localIds = new List<string>();
            string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
            string? pathLessons = $"{lessonBasePath}\\lessons";
            string? pathAnalysis = $"{lessonBasePath}\\analysis";
            var filesLessons = FileHelper.ListAllFiles(pathLessons);
            foreach (var file in filesLessons)
            {
                if (file.EndsWith("-local.json"))
                {
                    string lessonId = file.Split("\\").Last().Replace("-local.json", "");
                    localIds.Add(lessonId);
                }
            }
            bool loadLocal = true;
            var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
            long stime = 1693497600000;//2023-09-01 00:00:00
            bool force = false;
            if ((json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
            {
                force= _force.GetBoolean();
            }
            if (force) 
            {
                List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
                foreach (var file in filesAnalysis)
                {
                    //读取每月的数据
                    if (file.EndsWith("-m-analysis.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(file);
                        LessonDataAnalysisMonth lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisMonth>();
                        lessonDataAnalysisMonths.Add(lessonDataAnalysis);
                    }
                }
                if (lessonDataAnalysisMonths.IsNotEmpty())
                {
                    var maxUpdateTime = lessonDataAnalysisMonths.Max(x => x.updateTime);
                    if (maxUpdateTime>0)
                    {
                        //更新周期是一周
                        if (DateTimeOffset.Now.ToUnixTimeMilliseconds()- maxUpdateTime>604800000)
                        {
                            stime=maxUpdateTime;
                            loadLocal =true;
                        }
                        else
                        {
                            stime=maxUpdateTime;
                            loadLocal=false;
                        }
                    }
                }
            }
           
            // if (loadLocal ||force)
            {
                
                List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
                await LessonETLService.FixLocalData(localIds, _azureCosmos, _azureStorage, pathLessons, stime, DateTimeOffset.Now.ToUnixTimeMilliseconds());
                List<TechCount> techCounts = new List<TechCount>();
                filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");

              
                List<LessonLocal> lessonLocals = new List<LessonLocal>();
                int u = 0;
                if (force) 
                {
                    stopwatch.Start();
                     Parallel.ForEach(filesLessons, file =>
                    {
                        string jsonp =   System.IO.File.ReadAllText(file);
                        var lessonLocal = jsonp.ToObject<LessonLocal>();

                        if (lessonLocal.lessonBase!=null && lessonLocal.lessonBase.student.IsNotEmpty())
                        {
                            lessonLocals.Add(lessonLocal);
                        }
                        else
                        {
                            System.IO.File.Delete(file);
                            System.IO.File.Delete(file.Replace("-local.json", "-count.json"));
                            u++;
                        }
                    });
                    stopwatch.Stop();
                }
                _logger.LogInformation($"Loaded {lessonLocals.Count} lessons in {stopwatch.Elapsed.TotalSeconds} seconds");
                int  index = 0;
                await Parallel.ForEachAsync(filesLessons, async (item, _) => {
                    TechCount techCount=  await  LessonETLService.GetTeachCount(_azureCosmos, item, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force, lessonLocals);
                    if (techCount != null) {
                        techCounts.Add(techCount);
                    }
                    index++;
                });
                long newest = lessonLocals.Max(x=>x.lessonRecord.startTime);
                await LessonETLService.GenAnalysisData(pathAnalysis, newest, techCounts,_azureStorage);
            }
            return Ok(new {   });
        }
    }
}