|
|
@@ -4,6 +4,7 @@ using MathNet.Numerics.Distributions;
|
|
|
using Microsoft.AspNetCore.Mvc;
|
|
|
using Microsoft.Azure.Cosmos;
|
|
|
using StackExchange.Redis;
|
|
|
+using System.Collections.Generic;
|
|
|
using System.Text.Json;
|
|
|
using System.Xml;
|
|
|
using TEAMModelOS.SDK;
|
|
|
@@ -199,7 +200,6 @@ namespace HTEX.DataETL.Controllers
|
|
|
var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
|
|
|
XmlDocument xmlDocument = new XmlDocument();
|
|
|
xmlDocument.Load($"{runtimePath}\\summary.xml");
|
|
|
- List<StudentSemesterRecord> students = new List<StudentSemesterRecord>();
|
|
|
List<School> schools = new List<School>();
|
|
|
// await Parallel.ForEachAsync(filesLessons, async (fileLesson, _) =>
|
|
|
List<string> localIds = new List<string>();
|
|
|
@@ -229,7 +229,7 @@ namespace HTEX.DataETL.Controllers
|
|
|
lessonLocals.Add(item);
|
|
|
index++;
|
|
|
});
|
|
|
- int n = 0;
|
|
|
+
|
|
|
var schoolGroup = lessonLocals.Where(x=>!string.IsNullOrWhiteSpace(x.lessonRecord?.school)).GroupBy(x => x.lessonRecord?.school).Select(x => new { key=x.Key ,list=x.ToList()});
|
|
|
string schoolSql = $"select value c from c where c.id in ({string.Join(",", schoolGroup.Select(x => $"'{x.key}'"))})";
|
|
|
var schoolResults= await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).GetList<School>(schoolSql, "Base");
|
|
|
@@ -237,12 +237,59 @@ namespace HTEX.DataETL.Controllers
|
|
|
{
|
|
|
schools.AddRange(schoolResults.list);
|
|
|
}
|
|
|
-
|
|
|
+ foreach (var group in schoolGroup)
|
|
|
+ {
|
|
|
+ var studentIds = group.list.SelectMany(x => x.studentLessonDatas).Where(x=>!string.IsNullOrWhiteSpace(x.id)).Select(x => x.id).Distinct();
|
|
|
+ string studentSql = $"select value c from c where c.id in ({string.Join(",", studentIds.Select(x => $"'{x}'"))})";
|
|
|
+ var studentResults= await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<Student>(studentSql, $"Base-{group.key}");
|
|
|
+ if (studentResults.list.IsNotEmpty())
|
|
|
+ {
|
|
|
+ studentsBase.AddRange(studentResults.list);
|
|
|
+ }
|
|
|
+ string studentSemesterRecordSql = $"select value c from c where c.stuid in ({string.Join(",", studentIds.Select(x => $"'{x}'"))}) and c.studyYear>=2023";
|
|
|
+ var studentSemesterRecordResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<StudentSemesterRecord>(studentSemesterRecordSql, $"StudentSemesterRecord-{group.key}");
|
|
|
+ if (studentSemesterRecordResults.list.IsNotEmpty())
|
|
|
+ {
|
|
|
+ studentSemesterRecords.AddRange(studentSemesterRecordResults.list);
|
|
|
+ }
|
|
|
+ string overallEducationSql = $"select value c from c where c.studentId in ({string.Join(",", studentIds.Select(x => $"'{x}'"))}) and c.year>=2023";
|
|
|
+ var overallEducationResults = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Student).GetList<OverallEducation>(overallEducationSql, $"OverallEducation-{group.key}");
|
|
|
+ if (overallEducationResults.list.IsNotEmpty())
|
|
|
+ {
|
|
|
+ overallEducations.AddRange(overallEducationResults.list);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ List<(string id,string owner, List<StudentLessonData> studentLessonData)> studentLessonDatas= new List<(string id, string owner, List<StudentLessonData>)>();
|
|
|
+ List<(string id, string owner, List<StudentLessonItem> studentLessons,List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)> lessonItems = new List<(string id, string owner, List<StudentLessonItem> studentLessons, List<CodeBool> codeBools, List<StudentLessonData> studentLessonData)>();
|
|
|
+ int n = 0;
|
|
|
foreach (var item in lessonLocals)
|
|
|
{
|
|
|
var studata = await LessonETLService.DoStudentLessonData(Constant.objectiveTypes, _azureStorage, item, _dingDing, _azureCosmos.GetCosmosClient(), "China", _azureRedis, studentSemesterRecords, overallEducations, lessonDataAnalysis, studentsBase, schools);
|
|
|
+ if (studata.codeBools.FindAll(x=>x.value).IsNotEmpty())
|
|
|
+ {
|
|
|
+ string owner = item.lessonRecord.scope.Equals("school") ? item.lessonRecord.school : item.lessonRecord.tmdid;
|
|
|
+ studentLessonDatas.Add((item.lessonRecord.id, owner, studata.studentLessonDatas));
|
|
|
+ lessonItems.Add((item.lessonRecord.id, owner, studata.lessonItems,studata.codeBools,studata.studentLessonDatas));
|
|
|
+ }
|
|
|
n++;
|
|
|
}
|
|
|
+ int p = 0;
|
|
|
+ //XmlDocument xmlDocument = new XmlDocument();
|
|
|
+ //var runtimePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
|
|
|
+ //xmlDocument.Load($"{runtimePath}\\summary.xml");
|
|
|
+ await Parallel.ForEachAsync(lessonItems, async (lessonItem, cancellationToken) =>
|
|
|
+ {
|
|
|
+ lessonItem.studentLessons = LessonETLService.ProcessStudentDataV2(lessonItem.studentLessonData, lessonDataAnalysis, lessonItem.codeBools);
|
|
|
+ await LessonETLService.ExportToExcelAzureBlob(lessonItem.studentLessons, _azureStorage, lessonItem.owner, $"{lessonItem.id}/student-analysis.xlsx", xmlDocument);
|
|
|
+ p++;
|
|
|
+ });
|
|
|
+ int s = 0;
|
|
|
+ await Parallel.ForEachAsync(studentLessonDatas, async (studentLessonData, cancellationToken) =>
|
|
|
+ {
|
|
|
+ await _azureStorage.GetBlobContainerClient(studentLessonData.owner).UploadFileByContainer(studentLessonData.studentLessonData.ToJsonString(), "records", $"{studentLessonData.id}/student-analysis.json");
|
|
|
+ s++;
|
|
|
+ });
|
|
|
+
|
|
|
int m =0;
|
|
|
await Parallel.ForEachAsync(studentSemesterRecords, async (studentSemester, cancellationToken) =>
|
|
|
{
|
CrazyIter_Bin