using HTEX.Lib.ETL.Lesson; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.Cosmos; using System.Text.Json; using System.Xml; using TEAMModelOS.SDK; using TEAMModelOS.SDK.DI; using TEAMModelOS.SDK.Extension; using TEAMModelOS.SDK.Helper.Common.FileHelper; using TEAMModelOS.SDK.Models; using TEAMModelOS.SDK.Models.Cosmos.Common; namespace HTEX.DataETL.Controllers { [ApiController] [Route("lesson-record")] public class LessonRecordController : ControllerBase { private readonly ILogger _logger; private readonly AzureCosmosFactory _azureCosmos; private readonly AzureStorageFactory _azureStorage; private readonly IConfiguration _configuration; private readonly IWebHostEnvironment _webHostEnvironment; public LessonRecordController(ILogger logger, AzureCosmosFactory azureCosmos, AzureStorageFactory azureStorage , IConfiguration configuration, IWebHostEnvironment environment) { _logger = logger; _azureCosmos = azureCosmos; _azureStorage = azureStorage; _configuration = configuration; _webHostEnvironment = environment; } [HttpPost("process-local")] public async Task ProcessLocal(JsonElement json) { List studentLessonDatas = new List(); string? id = json.GetProperty("id").GetString(); if (!string.IsNullOrWhiteSpace(id)) { string? lessonPath = _configuration.GetValue("LessonPath"); string? path = $"{lessonPath}\\locals\\{id}"; var files = FileHelper.ListAllFiles(path); // var sampleJson =System.IO. File.ReadAllTextAsync(path); LessonBase? lessonBase = null; List localStudents = new List(); List taskDatas = new List(); List smartRatingDatas = new List(); List irsDatas = new List(); List coworkDatas = new List(); List examDatas = new List(); TimeLineData? timeLineData = null; foreach (var item in files) { if (item.Contains("IES\\base.json")) { string jsons = await System.IO.File.ReadAllTextAsync(item); jsons = jsons.Replace("\"Uncall\"", "0").Replace("Uncall", "0"); lessonBase = jsons.ToObject(); var data = LessonETLService.GetBaseData(lessonBase); localStudents = data.studentLessonDatas; } if (item.Contains("IES\\TimeLine.json")) { string jsons = await System.IO.File.ReadAllTextAsync(item); timeLineData = jsons.ToObject(); } if (item.Contains("IES\\Task.json")) { string jsons = await System.IO.File.ReadAllTextAsync(item); taskDatas = jsons.ToObject>(); } if (item.Contains("IES\\SmartRating.json")) { string jsons = await System.IO.File.ReadAllTextAsync(item); smartRatingDatas = jsons.ToObject>(); } if (item.Contains("IES\\IRS.json")) { string jsons = await System.IO.File.ReadAllTextAsync(item); irsDatas = jsons.ToObject>(); } if (item.Contains("IES\\Cowork.json")) { string jsons = await System.IO.File.ReadAllTextAsync(item); coworkDatas = jsons.ToObject>(); } try { if (item.Contains($"\\{id}\\Exam\\") && item.EndsWith("Exam.json")) { string examsFile = item; if (examsFile.EndsWith("Exam.json")) { ExamData? examData = null; string jsons = await System.IO.File.ReadAllTextAsync(item); jsons = jsons.Replace("\"publish\": \"0\"", "\"publish\": 0").Replace("\"publish\": \"1\"", "\"publish\": 1"); examData = jsons.ToObject(); if (examData != null && examData.exam.papers.IsNotEmpty()) { string paperId = examData.exam.papers.First().id; string paperPath = $"{path}\\ExamPaper\\{paperId}\\index.json"; string jsonp = await System.IO.File.ReadAllTextAsync(paperPath); LessonPaper lessonPaper = jsonp.ToObject(); examData.paper = lessonPaper; examDatas.Add(examData); } } } } catch (Exception ex) { _logger.LogError(ex, ex.Message); } } if (lessonBase!=null && timeLineData!=null) { studentLessonDatas = localStudents.ToJsonString().ToObject>(); studentLessonDatas = LessonETLService.GetBaseInfo(lessonBase!, studentLessonDatas, id); studentLessonDatas = LessonETLService.GetIRSData(lessonBase, timeLineData, irsDatas, studentLessonDatas, examDatas, id); studentLessonDatas = LessonETLService.GetCoworkData(lessonBase, timeLineData, coworkDatas, studentLessonDatas, id); studentLessonDatas = LessonETLService.GetExamData(lessonBase, timeLineData, examDatas, studentLessonDatas, Constant.objectiveTypes, id); studentLessonDatas = LessonETLService.GetSmartRatingData(lessonBase, timeLineData, smartRatingDatas, studentLessonDatas, id); studentLessonDatas = LessonETLService.GetTaskData(lessonBase, timeLineData, taskDatas, studentLessonDatas, id); var pickupData = LessonETLService.GetPickupData(lessonBase, timeLineData, studentLessonDatas, id); studentLessonDatas= pickupData.studentLessonDatas; await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString()); string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis.json"); LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject(); var lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis); XmlDocument xmlDocument = new XmlDocument(); var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); xmlDocument.Load($"{runtimePath}\\summary.xml"); await LessonETLService.ExportToExcelLocal(lessonItems, $"{path}\\analysis.xlsx", xmlDocument); } } return Ok(); } /// /// /// /// /// [HttpPost("process-fix-history-students")] public async Task ProcessFixHistoryStudents(JsonElement json) { string? lessonBasePath = _configuration.GetValue("LessonPath"); string studentsPath = $"{lessonBasePath}\\students"; List studentsPs = FileHelper.ListAllFiles(studentsPath); string? pathLessons = $"{lessonBasePath}\\lessons"; List filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json"); int index = 0; foreach (var stu in studentsPs) { string stujson = await System.IO.File.ReadAllTextAsync(stu); var studentSemester = stujson.ToObject(); await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester, new PartitionKey(studentSemester.code)); index++; } return Ok(); } /// /// /// /// /// [HttpPost("process-history-students")] public async Task ProcessHistoryStudents(JsonElement json) { string? lessonBasePath = _configuration.GetValue("LessonPath"); string? pathLessons = $"{lessonBasePath}\\lessons"; string? pathAnalysis = $"{lessonBasePath}\\analysis"; string jsons = await System.IO.File.ReadAllTextAsync($"{pathAnalysis}\\analysis.json"); LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject(); List filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json"); var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); XmlDocument xmlDocument = new XmlDocument(); xmlDocument.Load($"{runtimePath}\\summary.xml"); List students = new List(); List schools = new List(); // await Parallel.ForEachAsync(filesLessons, async (fileLesson, _) => foreach (var fileLesson in filesLessons) { try { string localjson = await System.IO.File.ReadAllTextAsync(fileLesson); var lessonLocal = localjson.ToObject(); List studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject>(); studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id); studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal?.lessonRecord?.id); studentLessonDatas = LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id); studentLessonDatas = LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, Constant.objectiveTypes, lessonLocal.lessonRecord.id); studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id); studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id); var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id); studentLessonDatas= pickupData.studentLessonDatas; string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid; var lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis); if (lessonLocal.lessonRecord.scope.Equals("school")&& !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.school)) { var school = schools.Find(x => x.id.Equals(lessonLocal.lessonRecord.school)); if (school==null) { ResponseMessage response = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemStreamAsync(lessonLocal.lessonRecord.school, new PartitionKey("Base")); if (response.IsSuccessStatusCode) { school = JsonDocument.Parse(response.Content).RootElement.ToObject(); } } string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : school.period.FirstOrDefault()?.id; var period = school.period.Find(x => x.id.Equals(periodId)); var semester = SchoolService.GetSemester(period, lessonLocal.lessonRecord.startTime); string pre_id = $"{semester.studyYear}-{semester.currSemester.id}"; string code = $"StudentSemesterRecord-{school.id}"; foreach (var studentLessonData in studentLessonDatas) { StudentSemesterRecord studentSemester = students.Find(x => x.stuid.Equals(studentLessonData.id) &&x.id.Equals($"{pre_id}-{studentLessonData.id}") && x.code.Equals(code)); if (studentSemester==null) { studentSemester= new StudentSemesterRecord { id= $"{pre_id}-{studentLessonData.id}", code=code, stuid= studentLessonData.id, userType=Constant.ScopeStudent, school=school.id, studyYear=semester.studyYear, semesterId=semester.currSemester.id, period= period?.id, pk="StudentSemesterRecord" }; students.Add(studentSemester); } string lessonId = string.Empty; StuLesson lesson = new StuLesson() { id= lessonLocal.lessonRecord.id, time= lessonLocal.lessonRecord.startTime, attend=0 }; if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 || studentLessonData.appraise>0) { lesson.attend=1; studentSemester.lessons.Add(new StuLessonLite { id=lessonLocal.lessonRecord.id, tmdid=lessonLocal.lessonRecord.tmdid, sid= lessonLocal.lessonRecord.subjectId, cid= lessonLocal.lessonRecord.courseId, hrate=studentLessonData.cooperation, crate=studentLessonData.achieve, trate=studentLessonData.attitude, xrate=studentLessonData.cowork, prate=studentLessonData.appraise }); } else { if (studentLessonData.attend==1) { lesson.attend=1; } else { lesson.attend=0; } } studentSemester.les.Add(lesson); // studentSemester.lessonIds.Add(lessonId); } } // await LessonETLService.ExportToExcelAzureBlob(lessonItems, _azureStorage, owner, $"{lessonLocal.lessonRecord.id}/student-analysis.xlsx", xmlDocument); // await _azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json"); } catch (Exception ex) { throw new Exception($"{fileLesson}", ex); } } string studentsPath = $"{lessonBasePath}\\students"; foreach (var stu in students) { await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(stu, new PartitionKey(stu.code)); // string path = $"{studentsPath}\\{stu.school}\\{stu.studyYear}\\{stu.semesterId}"; // if (!Directory.Exists(path)) { // Directory.CreateDirectory(path); } // await System.IO.File.WriteAllTextAsync($"{path}\\{stu.stuid}.json", stu.ToJsonString()); } string schoolsPath = $"{lessonBasePath}\\schools"; await System.IO.File.WriteAllTextAsync($"{schoolsPath}\\school.json", schools.ToJsonString()); return Ok(); } /// /// 课例数据ETL处理过程 /// /// /// [HttpPost("process-history")] public async Task ProcessHistory(JsonElement json) { List localIds = new List(); string? lessonBasePath = _configuration.GetValue("LessonPath"); string? pathLessons = $"{lessonBasePath}\\lessons"; string? pathAnalysis = $"{lessonBasePath}\\analysis"; var filesLessons = FileHelper.ListAllFiles(pathLessons); foreach (var file in filesLessons) { if (file.EndsWith("-local.json")) { string lessonId = file.Split("\\").Last().Replace("-local.json", ""); localIds.Add(lessonId); } } bool loadLocal = true; List lessonDataAnalysisMonths = new List(); var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis); long stime = 1690819200000;//2023-08-01 00:00:00 foreach (var file in filesAnalysis) { //读取每月的数据 if (file.EndsWith("-m-analysis.json")) { string jsons = await System.IO.File.ReadAllTextAsync(file); LessonDataAnalysisMonth lessonDataAnalysis = jsons.ToObject(); lessonDataAnalysisMonths.Add(lessonDataAnalysis); } } if (lessonDataAnalysisMonths.IsNotEmpty()) { var maxUpdateTime = lessonDataAnalysisMonths.Max(x => x.updateTime); if (maxUpdateTime>0) { //更新周期是一周 if (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()- maxUpdateTime>604800000) { stime=maxUpdateTime; loadLocal =true; } else { stime=maxUpdateTime; loadLocal=false; } } } HashSet yearMonth = new HashSet(); long newest = 0; bool force = false; if ((json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True))) { force= _force.GetBoolean(); } // if (loadLocal ||force) { List lessonRecords = new List(); var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School) .GetList($"SELECT value c FROM c where c.startTime>={stime} and c.expire<=0 and c.status<>404 and c.duration>300 and c.pk='LessonRecord' and c.school<>'hbcn' and c.school<>'habook' and (c.tLevel>0 or c.pLevel>0) ", null); if (resultSchool.list.IsNotEmpty()) { newest= resultSchool.list.Max(x => x.startTime); lessonRecords.AddRange(resultSchool.list); } else { newest=stime; } var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher) .GetList($"SELECT value c FROM c where c.startTime>={stime} and c.expire<=0 and c.status<>404 and c.duration>300 and c.pk='LessonRecord' and (c.tLevel>0 or c.pLevel>0) ", null); if (resultTeacher.list.IsNotEmpty()) { long max = resultTeacher.list.Max(x => x.startTime); if (max ignore = new List() { "PgJump", "PgRcv", "PgAdd" }; if (lessonRecords.IsNotEmpty()) { await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons)) { string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM"); if (item.lessonBase!=null && item.lessonBase.student!=null) { TechCount techCount = new TechCount { lessonId=item.lessonRecord?.id, examCount = item.examDatas.Count, taskCount = item.taskDatas.Count, irsCount = item.irsDatas.Count, coworkCount = item.coworkDatas.Count, smartRatingCount =item.smartRatingDatas.Count, timeCount=item.sokratesDatas.Where(x => !ignore.Contains(x.Event) && !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList() }; await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord.id}-count.json", techCount.ToJsonString()); await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString()); } else { System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json"); System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json"); } } } List techCounts = new List(); filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json"); await foreach (var item in LessonETLService.GetTeachCount(lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force)) { techCounts.Add(item); } LessonETLService.GenAnalysisData(pathAnalysis, newest, techCounts); } return Ok(new { yearMonth }); } } }