LessonRecordController.cs 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. using Azure.Storage.Blobs.Models;
  2. using DocumentFormat.OpenXml.Office2010.Excel;
  3. using HTEX.Lib.ETL;
  4. using HTEX.Lib.ETL.Lesson;
  5. using Microsoft.AspNetCore.Mvc;
  6. using Microsoft.Azure.Cosmos;
  7. using Microsoft.OData.UriParser;
  8. using System.CodeDom;
  9. using System.Collections.Generic;
  10. using System.IO;
  11. using System.Runtime.InteropServices;
  12. using System.Text.Json;
  13. using System.Xml;
  14. using TEAMModelOS.SDK;
  15. using TEAMModelOS.SDK.DI;
  16. using TEAMModelOS.SDK.Extension;
  17. using TEAMModelOS.SDK.Helper.Common.FileHelper;
  18. using TEAMModelOS.SDK.Models;
  19. using TEAMModelOS.SDK.Models.Cosmos.BI;
  20. using TEAMModelOS.SDK.Models.Cosmos.Common;
  21. using TEAMModelOS.SDK.Models.Cosmos.OpenEntity;
  22. namespace HTEX.DataETL.Controllers
  23. {
  24. [ApiController]
  25. [Route("lesson-record")]
  26. public class LessonRecordController : ControllerBase
  27. {
  28. private readonly ILogger<LessonRecordController> _logger;
  29. private readonly AzureCosmosFactory _azureCosmos;
  30. private readonly AzureStorageFactory _azureStorage;
  31. private readonly IConfiguration _configuration;
  32. private readonly IWebHostEnvironment _webHostEnvironment;
  33. public LessonRecordController(ILogger<LessonRecordController> logger, AzureCosmosFactory azureCosmos, AzureStorageFactory azureStorage
  34. , IConfiguration configuration, IWebHostEnvironment environment)
  35. {
  36. _logger = logger;
  37. _azureCosmos = azureCosmos;
  38. _azureStorage = azureStorage;
  39. _configuration = configuration;
  40. _webHostEnvironment = environment;
  41. }
  42. [HttpPost("process-local")]
  43. public async Task<IActionResult> ProcessLocal(JsonElement json)
  44. {
  45. List<StudentLessonData> studentLessonDatas = new List<StudentLessonData>();
  46. string? id = json.GetProperty("id").GetString();
  47. if (!string.IsNullOrWhiteSpace(id))
  48. {
  49. string? lessonPath= _configuration.GetValue<string>("LessonPath");
  50. string? path = $"{lessonPath}\\locals\\{id}";
  51. var files = FileHelper.ListAllFiles(path);
  52. // var sampleJson =System.IO. File.ReadAllTextAsync(path);
  53. LessonBase? lessonBase = null;
  54. List<LocalStudent> localStudents = new List<LocalStudent>();
  55. List<TaskData> taskDatas = new List<TaskData>();
  56. List<SmartRatingData> smartRatingDatas = new List<SmartRatingData>();
  57. List<IRSData> irsDatas = new List<IRSData>();
  58. List<CoworkData> coworkDatas = new List<CoworkData>();
  59. List<ExamData> examDatas = new List<ExamData>();
  60. TimeLineData? timeLineData = null;
  61. foreach (var item in files)
  62. {
  63. if (item.Contains("IES\\base.json"))
  64. {
  65. string jsons = await System.IO.File.ReadAllTextAsync(item);
  66. jsons = jsons.Replace("\"Uncall\"", "0").Replace("Uncall", "0");
  67. lessonBase = jsons.ToObject<LessonBase>();
  68. var data = LessonETLService.GetBaseData(lessonBase);
  69. localStudents = data.studentLessonDatas;
  70. }
  71. if (item.Contains("IES\\TimeLine.json"))
  72. {
  73. string jsons = await System.IO.File.ReadAllTextAsync(item);
  74. timeLineData = jsons.ToObject<TimeLineData>();
  75. }
  76. if (item.Contains("IES\\Task.json"))
  77. {
  78. string jsons = await System.IO.File.ReadAllTextAsync(item);
  79. taskDatas = jsons.ToObject<List<TaskData>>();
  80. }
  81. if (item.Contains("IES\\SmartRating.json"))
  82. {
  83. string jsons = await System.IO.File.ReadAllTextAsync(item);
  84. smartRatingDatas = jsons.ToObject<List<SmartRatingData>>();
  85. }
  86. if (item.Contains("IES\\IRS.json"))
  87. {
  88. string jsons = await System.IO.File.ReadAllTextAsync(item);
  89. irsDatas = jsons.ToObject<List<IRSData>>();
  90. }
  91. if (item.Contains("IES\\Cowork.json"))
  92. {
  93. string jsons = await System.IO.File.ReadAllTextAsync(item);
  94. coworkDatas = jsons.ToObject<List<CoworkData>>();
  95. }
  96. try
  97. {
  98. if (item.Contains($"\\{id}\\Exam\\") && item.EndsWith("Exam.json"))
  99. {
  100. string examsFile = item;
  101. if (examsFile.EndsWith("Exam.json"))
  102. {
  103. ExamData? examData = null;
  104. string jsons = await System.IO.File.ReadAllTextAsync(item);
  105. jsons = jsons.Replace("\"publish\": \"0\"", "\"publish\": 0").Replace("\"publish\": \"1\"", "\"publish\": 1");
  106. examData = jsons.ToObject<ExamData>();
  107. if (examData != null && examData.exam.papers.IsNotEmpty())
  108. {
  109. string paperId = examData.exam.papers.First().id;
  110. string paperPath = $"{path}\\ExamPaper\\{paperId}\\index.json";
  111. string jsonp = await System.IO.File.ReadAllTextAsync(paperPath);
  112. LessonPaper lessonPaper = jsonp.ToObject<LessonPaper>();
  113. examData.paper = lessonPaper;
  114. examDatas.Add(examData);
  115. }
  116. }
  117. }
  118. }
  119. catch (Exception ex)
  120. {
  121. _logger.LogError(ex, ex.Message);
  122. }
  123. }
  124. if (lessonBase!=null && timeLineData!=null)
  125. {
  126. studentLessonDatas = localStudents.ToJsonString().ToObject<List<StudentLessonData>>();
  127. studentLessonDatas = LessonETLService.GetBaseInfo( lessonBase!, studentLessonDatas, id);
  128. studentLessonDatas = LessonETLService.GetIRSData(lessonBase, timeLineData, irsDatas, studentLessonDatas, examDatas, id);
  129. studentLessonDatas = LessonETLService.GetCoworkData(lessonBase, timeLineData, coworkDatas, studentLessonDatas,id);
  130. studentLessonDatas = LessonETLService.GetExamData(lessonBase, timeLineData, examDatas, studentLessonDatas, Constant.objectiveTypes, id);
  131. studentLessonDatas = LessonETLService.GetSmartRatingData(lessonBase, timeLineData, smartRatingDatas, studentLessonDatas, id);
  132. studentLessonDatas = LessonETLService.GetTaskData(lessonBase, timeLineData, taskDatas, studentLessonDatas,id);
  133. var pickupData = LessonETLService.GetPickupData(lessonBase, timeLineData, studentLessonDatas, id);
  134. studentLessonDatas= pickupData.studentLessonDatas;
  135. await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString());
  136. string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis.json");
  137. LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
  138. var lessonItems= LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis);
  139. XmlDocument xmlDocument = new XmlDocument();
  140. var runtimePath= System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
  141. xmlDocument.Load($"{runtimePath}\\summary.xml");
  142. await LessonETLService. ExportToExcelLocal(lessonItems, $"{path}\\analysis.xlsx", xmlDocument);
  143. }
  144. }
  145. return Ok();
  146. }
  147. /// <summary>
  148. ///
  149. /// </summary>
  150. /// <param name="json"></param>
  151. /// <returns></returns>
  152. [HttpPost("process-fix-history-students")]
  153. public async Task<IActionResult> ProcessFixHistoryStudents(JsonElement json)
  154. {
  155. string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
  156. string studentsPath = $"{lessonBasePath}\\students";
  157. List<string> studentsPs = FileHelper.ListAllFiles(studentsPath);
  158. string? pathLessons = $"{lessonBasePath}\\lessons";
  159. List<string> filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
  160. int index = 0;
  161. foreach (var stu in studentsPs)
  162. {
  163. string stujson = await System.IO.File.ReadAllTextAsync(stu);
  164. var studentSemester = stujson.ToObject<StudentSemesterRecord>();
  165. await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).UpsertItemAsync(studentSemester,new PartitionKey(studentSemester.code));
  166. index++;
  167. }
  168. return Ok();
  169. }
  170. /// <summary>
  171. ///
  172. /// </summary>
  173. /// <param name="json"></param>
  174. /// <returns></returns>
  175. [HttpPost("process-history-students")]
  176. public async Task<IActionResult> ProcessHistoryStudents(JsonElement json)
  177. {
  178. string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
  179. string? pathLessons = $"{lessonBasePath}\\lessons";
  180. string? pathAnalysis = $"{lessonBasePath}\\analysis";
  181. string jsons = await System.IO.File.ReadAllTextAsync($"{pathAnalysis}\\analysis.json");
  182. LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
  183. List<string> filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
  184. var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
  185. XmlDocument xmlDocument = new XmlDocument();
  186. xmlDocument.Load($"{runtimePath}\\summary.xml");
  187. List<StudentSemesterRecord> students = new List<StudentSemesterRecord>();
  188. List<School> schools = new List<School>();
  189. await Parallel.ForEachAsync(filesLessons, async (fileLesson, _) =>
  190. {
  191. try {
  192. string localjson = await System.IO.File.ReadAllTextAsync(fileLesson);
  193. var lessonLocal = localjson.ToObject<LessonLocal>();
  194. List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
  195. studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
  196. studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal?.lessonRecord?.id);
  197. studentLessonDatas = LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
  198. studentLessonDatas = LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, Constant.objectiveTypes, lessonLocal.lessonRecord.id);
  199. studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
  200. studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
  201. var pickupData = LessonETLService.GetPickupData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, studentLessonDatas, lessonLocal.lessonRecord.id);
  202. studentLessonDatas= pickupData.studentLessonDatas;
  203. string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
  204. var lessonItems = LessonETLService.ProcessStudentDataV2(studentLessonDatas, lessonDataAnalysis);
  205. if (lessonLocal.lessonRecord.scope.Equals("school")&& !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.school))
  206. {
  207. var school = schools.Find(x => x.id.Equals(lessonLocal.lessonRecord.school));
  208. if (school==null)
  209. {
  210. ResponseMessage response = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemStreamAsync(lessonLocal.lessonRecord.school, new PartitionKey("Base"));
  211. if (response.IsSuccessStatusCode)
  212. {
  213. school = JsonDocument.Parse(response.Content).RootElement.ToObject<School>();
  214. }
  215. }
  216. string? periodId = !string.IsNullOrWhiteSpace(lessonLocal.lessonRecord.periodId) ? lessonLocal.lessonRecord.periodId : school.period.FirstOrDefault()?.id;
  217. var period = school.period.Find(x => x.id.Equals(periodId));
  218. var semester = SchoolService.GetSemester(period, lessonLocal.lessonRecord.startTime);
  219. string pre_id = $"{semester.studyYear}-{semester.currSemester.id}";
  220. string code = $"StudentSemesterRecord-{school.id}";
  221. foreach (var studentLessonData in studentLessonDatas)
  222. {
  223. StudentSemesterRecord studentSemester = students.Find(x => x.stuid.Equals(studentLessonData.id) &&x.id.Equals($"{pre_id}-{studentLessonData.id}") && x.code.Equals(code));
  224. if (studentSemester==null)
  225. {
  226. studentSemester= new StudentSemesterRecord
  227. {
  228. id= $"{pre_id}-{studentLessonData.id}",
  229. code=code,
  230. stuid= studentLessonData.id,
  231. userType=Constant.ScopeStudent,
  232. school=school.id,
  233. studyYear=semester.studyYear,
  234. semesterId=semester.currSemester.id,
  235. period= period?.id,
  236. pk="StudentSemesterRecord"
  237. };
  238. students.Add(studentSemester);
  239. }
  240. string lessonId = string.Empty;
  241. StuLesson lesson = new StuLesson()
  242. {
  243. id= lessonLocal.lessonRecord.id,
  244. time= lessonLocal.lessonRecord.startTime,
  245. attend=0
  246. };
  247. if (studentLessonData.cooperation>0 || studentLessonData.achieve>0|| studentLessonData.attitude>0 || studentLessonData.cowork>0 || studentLessonData.appraise>0)
  248. {
  249. lesson.attend=1;
  250. studentSemester.lessons.Add(new StuLessonLite
  251. {
  252. tmdid=lessonLocal.lessonRecord.tmdid,
  253. sid= lessonLocal.lessonRecord.subjectId,
  254. cid= lessonLocal.lessonRecord.courseId,
  255. hrate=studentLessonData.cooperation,
  256. crate=studentLessonData.achieve,
  257. trate=studentLessonData.attitude,
  258. xrate=studentLessonData.cowork,
  259. prate=studentLessonData.appraise
  260. });
  261. }
  262. else {
  263. if (studentLessonData.attend==1)
  264. {
  265. lesson.attend=1;
  266. }
  267. else {
  268. lesson.attend=0;
  269. }
  270. }
  271. studentSemester.les.Add(lesson);
  272. // studentSemester.lessonIds.Add(lessonId);
  273. }
  274. }
  275. // await LessonETLService.ExportToExcelAzureBlob(lessonItems, _azureStorage, owner, $"{lessonLocal.lessonRecord.id}/student-analysis.xlsx", xmlDocument);
  276. // await _azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
  277. } catch (Exception ex)
  278. {
  279. throw new Exception($"{fileLesson}", ex);
  280. }
  281. });
  282. string studentsPath = $"{lessonBasePath}\\students";
  283. foreach (var stu in students)
  284. {
  285. string path = $"{studentsPath}\\{stu.school}\\{stu.studyYear}\\{stu.semesterId}";
  286. if (!Directory.Exists(path))
  287. {
  288. Directory.CreateDirectory(path);
  289. }
  290. await System.IO.File.WriteAllTextAsync($"{path}\\{stu.stuid}.json", stu.ToJsonString());
  291. }
  292. string schoolsPath = $"{lessonBasePath}\\schools";
  293. await System.IO.File.WriteAllTextAsync($"{schoolsPath}\\school.json", schools.ToJsonString());
  294. return Ok();
  295. }
  296. /// <summary>
  297. /// 课例数据ETL处理过程
  298. /// </summary>
  299. /// <param name="json"></param>
  300. /// <returns></returns>
  301. [HttpPost("process-history")]
  302. public async Task<IActionResult> ProcessHistory(JsonElement json)
  303. {
  304. List<string> localIds = new List<string>();
  305. string? lessonBasePath = _configuration.GetValue<string>("LessonPath");
  306. string? pathLessons = $"{lessonBasePath}\\lessons";
  307. string? pathAnalysis = $"{lessonBasePath}\\analysis";
  308. var filesLessons = FileHelper.ListAllFiles(pathLessons);
  309. foreach (var file in filesLessons)
  310. {
  311. if (file.EndsWith("-local.json"))
  312. {
  313. string lessonId = file.Split("\\").Last().Replace("-local.json", "");
  314. localIds.Add(lessonId);
  315. }
  316. }
  317. bool loadLocal = true;
  318. List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
  319. var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
  320. long stime = 1690819200000;//2023-08-01 00:00:00
  321. foreach (var file in filesAnalysis) {
  322. //读取每月的数据
  323. if (file.EndsWith("-m-analysis.json"))
  324. {
  325. string jsons = await System.IO.File.ReadAllTextAsync(file);
  326. LessonDataAnalysisMonth lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisMonth>();
  327. lessonDataAnalysisMonths.Add(lessonDataAnalysis);
  328. }
  329. }
  330. if (lessonDataAnalysisMonths.IsNotEmpty()) {
  331. var maxUpdateTime = lessonDataAnalysisMonths.Max(x => x.updateTime);
  332. if (maxUpdateTime>0)
  333. {
  334. //更新周期是一周
  335. if (DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()- maxUpdateTime>604800000)
  336. {
  337. stime=maxUpdateTime;
  338. loadLocal =true;
  339. }
  340. else
  341. {
  342. stime=maxUpdateTime;
  343. loadLocal=false;
  344. }
  345. }
  346. }
  347. HashSet<string> yearMonth = new HashSet<string>();
  348. long newest = 0;
  349. bool force = false;
  350. if ((json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
  351. {
  352. force= _force.GetBoolean();
  353. }
  354. // if (loadLocal ||force)
  355. {
  356. List<LessonRecord> lessonRecords = new List<LessonRecord>();
  357. var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
  358. .GetList<LessonRecord>($"SELECT value c FROM c where c.startTime>={stime} and c.expire<=0 and c.status<>404 and c.duration>300 and c.pk='LessonRecord' and c.school<>'hbcn' and c.school<>'habook' and (c.tLevel>0 or c.pLevel>0) ", null);
  359. if (resultSchool.list.IsNotEmpty())
  360. {
  361. newest= resultSchool.list.Max(x => x.startTime);
  362. lessonRecords.AddRange(resultSchool.list);
  363. }
  364. else
  365. {
  366. newest=stime;
  367. }
  368. var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
  369. .GetList<LessonRecord>($"SELECT value c FROM c where c.startTime>={stime} and c.expire<=0 and c.status<>404 and c.duration>300 and c.pk='LessonRecord' and (c.tLevel>0 or c.pLevel>0) ", null);
  370. if (resultTeacher.list.IsNotEmpty())
  371. {
  372. long max = resultTeacher.list.Max(x => x.startTime);
  373. if (max<newest)
  374. {
  375. newest=max;
  376. }
  377. lessonRecords.AddRange(resultTeacher.list);
  378. }
  379. List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
  380. if (lessonRecords.IsNotEmpty())
  381. {
  382. await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
  383. {
  384. string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
  385. if (item.lessonBase!=null && item.lessonBase.student!=null)
  386. {
  387. TechCount techCount = new TechCount
  388. {
  389. lessonId=item.lessonRecord?.id,
  390. examCount = item.examDatas.Count,
  391. taskCount = item.taskDatas.Count,
  392. irsCount = item.irsDatas.Count,
  393. coworkCount = item.coworkDatas.Count,
  394. smartRatingCount =item.smartRatingDatas.Count,
  395. timeCount=item.sokratesDatas.Where(x => !ignore.Contains(x.Event) && !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList()
  396. };
  397. await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord.id}-count.json", techCount.ToJsonString());
  398. await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json", item.ToJsonString());
  399. }
  400. else
  401. {
  402. System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-local.json");
  403. System.IO.File.Delete($"{pathLessons}\\MM{yearMonthPath}\\{item.lessonRecord!.id}-count.json");
  404. }
  405. }
  406. }
  407. List<TechCount> techCounts = new List<TechCount>();
  408. filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
  409. await foreach (var item in LessonETLService.GetTeachCount(lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force))
  410. {
  411. techCounts.Add(item);
  412. }
  413. LessonETLService. GenAnalysisData(pathAnalysis, newest, techCounts);
  414. }
  415. return Ok(new { yearMonth });
  416. }
  417. }
  418. }