TEAMModelOS/HTEX.DataETL/Controllers/LessonRecordController.cs

using DocumentFormat.OpenXml.Drawing.Charts;
using HTEX.Lib.ETL.Lesson;
using MathNet.Numerics.Distributions;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.Cosmos;
using StackExchange.Redis;
using System.Collections.Generic;
using System.Text.Json;
using System.Xml;
using TEAMModelOS.SDK;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Helper.Common.FileHelper;
using TEAMModelOS.SDK.Models;
using TEAMModelOS.SDK.Models.Cosmos.Common;
using TEAMModelOS.SDK.Models.Cosmos.OpenEntity;

namespace HTEX.DataETL.Controllers
{
    [ApiController]
    [Route("lesson-record")]
    public class LessonRecordController : ControllerBase
    {
        private readonly ILogger<LessonRecordController> _logger;
        private readonly AzureCosmosFactory _azureCosmos;
        private readonly AzureStorageFactory _azureStorage;
        private readonly IConfiguration _configuration;
        private readonly IWebHostEnvironment _webHostEnvironment;
        private readonly DingDing  _dingDing;
        private readonly AzureRedisFactory _azureRedis;



        public LessonRecordController(ILogger<LessonRecordController> logger, AzureCosmosFactory azureCosmos, AzureStorageFactory azureStorage   , IConfiguration configuration, IWebHostEnvironment environment,DingDing dingDing,AzureRedisFactory azureRedis )
        {
            _logger = logger;
            _azureCosmos = azureCosmos;
            _azureStorage = azureStorage;
            _configuration = configuration;
            _webHostEnvironment = environment;
            _dingDing = dingDing;
            _azureRedis = azureRedis;

        }
         
        [HttpPost("process-local")]
        public async Task<IActionResult> ProcessLocal(JsonElement json)
        {
            List<StudentLessonData> studentLessonDatas = new List<StudentLessonData>();
            string? id = json.GetProperty("id").GetString();
            if (!string.IsNullOrWhiteSpace(id))
            {
                string? lessonPath = _configuration.GetValue<string>("LessonPath");
                string? path = $"{lessonPath}\\locals\\{id}";
                var files = FileHelper.ListAllFiles(path);
                // var sampleJson =System.IO. File.ReadAllTextAsync(path);
                LessonBase? lessonBase = null;
                List<LocalStudent> localStudents = new List<LocalStudent>();
                List<TaskData> taskDatas = new List<TaskData>();
                List<SmartRatingData> smartRatingDatas = new List<SmartRatingData>();
                List<IRSData> irsDatas = new List<IRSData>();
                List<CoworkData> coworkDatas = new List<CoworkData>();
                List<ExamData> examDatas = new List<ExamData>();
                TimeLineData? timeLineData = null;
                foreach (var item in files)
                {

                    if (item.Contains("IES\\base.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        jsons = jsons.Replace("\"Uncall\"", "0").Replace("Uncall", "0");
                        lessonBase = jsons.ToObject<LessonBase>();
                        var data = LessonETLService.GetBaseData(lessonBase);
                        localStudents = data.studentLessonDatas;
                    }

                    if (item.Contains("IES\\TimeLine.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        timeLineData = jsons.ToObject<TimeLineData>();
                    }
                    if (item.Contains("IES\\Task.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        taskDatas = jsons.ToObject<List<TaskData>>();
                    }

                    if (item.Contains("IES\\SmartRating.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        smartRatingDatas = jsons.ToObject<List<SmartRatingData>>();
                    }

                    if (item.Contains("IES\\IRS.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        irsDatas = jsons.ToObject<List<IRSData>>();
                    }
                    if (item.Contains("IES\\Cowork.json"))
                    {
                        string jsons = await System.IO.File.ReadAllTextAsync(item);
                        coworkDatas = jsons.ToObject<List<CoworkData>>();
                    }
                    try
                    {
                        if (item.Contains($"\\{id}\\Exam\\") && item.EndsWith("Exam.json"))
                        {
                            string examsFile = item;
                            if (examsFile.EndsWith("Exam.json"))
                            {
                                ExamData? examData = null;
                                string jsons = await System.IO.File.ReadAllTextAsync(item);
                                jsons = jsons.Replace("\"publish\": \"0\"", "\"publish\": 0").Replace("\"publish\": \"1\"", "\"publish\": 1");
                                examData = jsons.ToObject<ExamData>();

                                if (examData != null && examData.exam.papers.IsNotEmpty())
                                {
                                    string paperId = examData.exam.papers.First().id;

                                    string paperPath = $"{path}\\ExamPaper\\{paperId}\\index.json";
                                    string jsonp = await System.IO.File.ReadAllTextAsync(paperPath);
                                    LessonPaper lessonPaper = jsonp.ToObject<LessonPaper>();
                                    examData.paper = lessonPaper;
                                    examDatas.Add(examData);
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, ex.Message);
                    }
                }
                if (lessonBase!=null  && timeLineData!=null)
                {
                    studentLessonDatas = localStudents.ToJsonString().ToObject<List<StudentLessonData>>();
                    studentLessonDatas = LessonETLService.GetBaseInfo(lessonBase!, studentLessonDatas, id);
                    studentLessonDatas = LessonETLService.GetIRSData(lessonBase, timeLineData, irsDatas, studentLessonDatas, examDatas, id);
                    studentLessonDatas = LessonETLService.GetCoworkData(lessonBase, timeLineData, coworkDatas, studentLessonDatas, id);
                    studentLessonDatas = LessonETLService.GetExamData(lessonBase, timeLineData, examDatas, studentLessonDatas, Constant.objectiveTypes, id);
                    studentLessonDatas = LessonETLService.GetSmartRatingData(lessonBase, timeLineData, smartRatingDatas, studentLessonDatas, id);
                    studentLessonDatas = LessonETLService.GetTaskData(lessonBase, timeLineData, taskDatas, studentLessonDatas, id);
                    var pickupData = LessonETLService.GetPickupData(lessonBase, timeLineData, studentLessonDatas, id);
                    studentLessonDatas= pickupData.studentLessonDatas;
                    var codeBools= LessonETLService.GetCodeBools(studentLessonDatas);
                    await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString());
                    string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis-model.json");
                    LessonDataAnalysisModel lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisModel>();
                    

                    var studentLessons = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis,codeBools);
                    XmlDocument xmlDocument = new XmlDocument();
                    var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
                    xmlDocument.Load($"{runtimePath}\\summary.xml");
                    await LessonETLService.ExportToExcelLocal(studentLessons, $"{path}\\analysis.xlsx", xmlDocument);
                }
            }
            return Ok();
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="json"></param>
        /// <returns></returns>
        [HttpPost("process-fix-history-students")]
        public async Task<IActionResult> ProcessFixHistoryStudents(JsonElement json)
        {
            string? pathAnalysis = $"F:\\lesson-local\\analysis";
            try
            {
                string jsons = await System.IO.File.ReadAllTextAsync($"F:\\lesson-local\\analysis\\analysis-model.json");
                var s = JsonSerializer.Deserialize<LessonDataAnalysisModel>(jsons);
                LessonDataAnalysisModel lessonDataAnalysis = JsonDocument.Parse(jsons).RootElement.ToObject<LessonDataAnalysisModel>();
                var per = LessonETLService.GetPersent(lessonDataAnalysis.irs, 2);

            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
            return Ok();
        }
        /// <summary>
        /// 
        /// </summary>
        /// <param name="json"></param>
        /// <returns></returns>
        [HttpPost("process-history-students")]
        public async Task<IActionResult> ProcessHistoryStudents(JsonElement json)
        {

          
            string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
            string? pathLessons = $"{lessonBasePath}\\lessons";
            string? pathAnalysis = $"{lessonBasePath}\\analysis";
            string jsons = await System.IO.File.ReadAllTextAsync($"{pathAnalysis}\\analysis-model.json");
            LessonDataAnalysisModel lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisModel>();
            List<string> filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
            var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
            XmlDocument xmlDocument = new XmlDocument();
            xmlDocument.Load($"{runtimePath}\\summary.xml");
            List<School> schools = new List<School>();
           // await Parallel.ForEachAsync(filesLessons, async (fileLesson, _) =>
            List<string> localIds = new List<string>();
            foreach (var file in filesLessons)
            {
                if (file.EndsWith("-local.json"))
                {
                    string lessonId = file.Split("\\").Last().Replace("-local.json", "");
                    localIds.Add(lessonId);
                }
            }
            long stime = 1693497600000;//2023-08-01 00:00:00
            long etime = 1731513599000;//2023-08-01 00:00:00
            var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
                 .GetList<LessonRecord>($"SELECT value c FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and c.startTime>={stime} and c.startTime<={etime} and   c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ");
            List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
            int index = 0;
            if (resultSchool.list.IsNotEmpty())
            {
                List<StudentSemesterRecord>  studentSemesterRecords= new List<StudentSemesterRecord>();
                List<OverallEducation> overallEducations= new List<OverallEducation>();
                List<Student> studentsBase = new List<Student>();
                List<LessonLocal>lessonLocals = new List<LessonLocal>();
                await Parallel.ForEachAsync(resultSchool.list, async (lessonRecord, cancellationToken) =>
                {
                    var  item = await LessonETLService.GetLessonLocal(lessonRecord,localIds,_azureStorage,pathLessons);
                    lessonLocals.Add(item);
                    index++;
                });
                
                var  schoolGroup =   lessonLocals.Where(x=>!string.IsNullOrWhiteSpace(x.lessonRecord?.school)).GroupBy(x => x.lessonRecord?.school).Select(x => new { key=x.Key ,list=x.ToList()});
                string schoolSql = $"select value c from c where c.id in ({string.Join(",", schoolGroup.Select(x => $"'{x.key}'"))})";
                var schoolResults= await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).GetList<School>(schoolSql, "Base");
                if (schoolResults.list.IsNotEmpty()) 
                {
                    schools.AddRange(schoolResults.list);
                }
                foreach (var group in schoolGroup) 
                {
                    var studentIds = group.list.SelectMany(x => x.studentLessonDatas).Where(x=>!string.IsNullOrWhiteSpace(x.id)).Select(x => x.id).Distinct();
                    string studentSql = $"select value c from c where c.id in ({string.Join(",", studentIds.Select(x => $"'{x}'"))})";
                    var  studentResults=  await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<Student>(studentSql, $"Base-{group.key}");
                    if (studentResults.list.IsNotEmpty()) 
                    {
                        studentsBase.AddRange(studentResults.list);
                    }
                    string studentSemesterRecordSql = $"select value c from c where c.stuid in ({string.Join(",", studentIds.Select(x => $"'{x}'"))}) and c.studyYear>=2023";
                    var studentSemesterRecordResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<StudentSemesterRecord>(studentSemesterRecordSql, $"StudentSemesterRecord-{group.key}");
                    if (studentSemesterRecordResults.list.IsNotEmpty())
                    {
                        studentSemesterRecords.AddRange(studentSemesterRecordResults.list);
                    }
                    string overallEducationSql = $"select value c from c where c.studentId in ({string.Join(",", studentIds.Select(x => $"'{x}'"))}) and c.year>=2023";
                    var overallEducationResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<OverallEducation>(overallEducationSql, $"OverallEducation-{group.key}");
                    if (overallEducationResults.list.IsNotEmpty())
                    {
                        overallEducations.AddRange(overallEducationResults.list);
                    }
                }
                List<(string id,string owner, List<StudentLessonData> studentLessonData)> studentLessonDatas= new List<(string id, string owner, List<StudentLessonData>)>();
                List<(string id, string owner, List<StudentLessonItem> studentLessons,List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)> lessonItems = new List<(string id, string owner, List<StudentLessonItem> studentLessons, List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)>();
                int n = 0;
                foreach (var  item in lessonLocals) 
                {
                    var studata = await LessonETLService.DoStudentLessonData(Constant.objectiveTypes, _azureStorage, item, _dingDing, _azureCosmos.GetCosmosClient(), "China", _azureRedis, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
                    if (studata.codeBools.FindAll(x=>x.value).IsNotEmpty()) 
                    {
                        string owner = item.lessonRecord.scope.Equals("school") ? item.lessonRecord.school : item.lessonRecord.tmdid;
                        studentLessonDatas.Add((item.lessonRecord.id, owner, studata.studentLessonDatas));
                        lessonItems.Add((item.lessonRecord.id, owner, studata.lessonItems,studata.codeBools,studata.studentLessonDatas));
                    }
                    n++;
                }
                int p = 0;
                //XmlDocument xmlDocument = new XmlDocument();
                //var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
                //xmlDocument.Load($"{runtimePath}\\summary.xml");
                await Parallel.ForEachAsync(lessonItems, async (lessonItem, cancellationToken) =>
                {
                    lessonItem.studentLessons = LessonETLService.ProcessStudentDataV2(lessonItem.studentLessonData, lessonDataAnalysis, lessonItem.codeBools);
                    await LessonETLService.ExportToExcelAzureBlob(lessonItem.studentLessons, _azureStorage, lessonItem.owner, $"{lessonItem.id}/student-analysis.xlsx", xmlDocument);
                    p++;
                });
                int s = 0;
                await Parallel.ForEachAsync(studentLessonDatas, async (studentLessonData, cancellationToken) =>
                {
                    await _azureStorage.GetBlobContainerClient(studentLessonData.owner).UploadFileByContainer(studentLessonData.studentLessonData.ToJsonString(), "records", $"{studentLessonData.id}/student-analysis.json");
                    s++;
                });

                int m   =0;
                await Parallel.ForEachAsync(studentSemesterRecords, async (studentSemester, cancellationToken) => 
                {
                    await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code));
                    m++;
                });
                int k = 0;
                await Parallel.ForEachAsync(overallEducations, async (overallEducation, cancellationToken) =>
                {
                    await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(overallEducation, partitionKey: new PartitionKey(overallEducation.code));
                    //string key = $"OverallEducation:{overallEducation.schoolCode}:{overallEducation.periodId}:{overallEducation.year}:{overallEducation.semesterId}:{overallEducation?.classId}";
                    //await _azureRedis.GetRedisClient(8).HashSetAsync(key, overallEducation.studentId, overallEducation.ToJsonString());
                    //await _azureRedis.GetRedisClient(8).KeyExpireAsync(key, new TimeSpan(180 *24, 0, 0));
                    k++;
                });
            }
            return Ok();
        }

        

        /// <summary>
        /// 课例数据ETL处理过程
        /// </summary>
        /// <param name="json"></param>
        /// <returns></returns>
        [HttpPost("process-history")]
        public async Task<IActionResult> ProcessHistory(JsonElement json)
        {

            List<string> localIds = new List<string>();
            string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
            string? pathLessons = $"{lessonBasePath}\\lessons";
            string? pathAnalysis = $"{lessonBasePath}\\analysis";
            var filesLessons = FileHelper.ListAllFiles(pathLessons);
            foreach (var file in filesLessons)
            {
                if (file.EndsWith("-local.json"))
                {
                    string lessonId = file.Split("\\").Last().Replace("-local.json", "");
                    localIds.Add(lessonId);
                }
            }
            bool loadLocal = true;
            List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
            var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
            long stime = 1693497600000;//2023-09-01 00:00:00
            foreach (var file in filesAnalysis)
            {
                //读取每月的数据
                if (file.EndsWith("-m-analysis.json"))
                {
                    string jsons = await System.IO.File.ReadAllTextAsync(file);
                    LessonDataAnalysisMonth lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisMonth>();
                    lessonDataAnalysisMonths.Add(lessonDataAnalysis);
                }
            }
            if (lessonDataAnalysisMonths.IsNotEmpty())
            {
                var maxUpdateTime = lessonDataAnalysisMonths.Max(x => x.updateTime);
                if (maxUpdateTime>0)
                {
                    //更新周期是一周
                    if (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()- maxUpdateTime>604800000)
                    {
                        stime=maxUpdateTime;
                        loadLocal =true;
                    }
                    else
                    {
                        stime=maxUpdateTime;
                        loadLocal=false;
                    }
                }
            }
            HashSet<string> yearMonth = new HashSet<string>();
            long newest = 0;
            bool force = false;
            if ((json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
            {
                force= _force.GetBoolean();
            }
            // if (loadLocal ||force)
            {
                List<LessonRecord> lessonRecords = new List<LessonRecord>();
                var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
                    .GetList<LessonRecord>($"SELECT value c FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime} and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord' ", null);
                if (resultSchool.list.IsNotEmpty())
                {
                    newest= resultSchool.list.Max(x => x.startTime);
                    lessonRecords.AddRange(resultSchool.list);
                }
                else
                {
                    newest=stime;
                }
                var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
                   .GetList<LessonRecord>($"SELECT value c FROM   c  where   ( c.analysis>=0 or IS_DEFINED(c.analysis) = false ) and  c.startTime>={stime} and c.expire<=0  and c.status<>404  and c.duration>300 and c.pk='LessonRecord'   ", null);
                if (resultTeacher.list.IsNotEmpty())
                {
                    long max = resultTeacher.list.Max(x => x.startTime);
                    if (max<newest)
                    {
                        newest=max;
                    }
                    lessonRecords.AddRange(resultTeacher.list);
                }
                List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
                if (lessonRecords.IsNotEmpty())
                {
                    await Parallel.ForEachAsync(lessonRecords, async(record, _)=>{
                        LessonLocal  item = await LessonETLService.GetLessonLocal(record, localIds, _azureStorage, pathLessons);
                        string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
                        item.lessonRecord.learningCategory= item.lessonBase?.summary?.learningCategory;
                        if (item.lessonRecord.learningCategory == null)
                        {

                            item.lessonRecord.learningCategory = new LearningCategory();
                        }
                        await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
                    });


                   //await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
                   // {
                   //     string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
                   //     item.lessonRecord.learningCategory= item.lessonBase?.summary?.learningCategory;
                   //     if (item.lessonRecord.learningCategory == null)
                   //     {

                   //         item. lessonRecord.learningCategory = new LearningCategory();
                   //     }
                   //     await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
                   // }
                }
                List<TechCount> techCounts = new List<TechCount>();
                filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
                //await foreach (var item in LessonETLService.GetTeachCount(_azureCosmos,lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force))
                //{
                //    techCounts.Add(item);
                //}

                await Parallel.ForEachAsync(filesLessons, async (item, _) => {
                    TechCount techCount=  await  LessonETLService.GetTeachCount(_azureCosmos, item, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force);
                    if (techCount != null) {
                        techCounts.Add(techCount);
                    }
                });

                await LessonETLService.GenAnalysisData(pathAnalysis, newest, techCounts,_azureStorage);
            }
            return Ok(new { yearMonth });
        }


    }
}