|
|
@@ -135,6 +135,9 @@ namespace HTEX.DataETL.Controllers
|
|
|
studentLessonDatas = LessonETLService.GetExamData(lessonBase, timeLineData, examDatas, studentLessonDatas, Constant.objectiveTypes, id);
|
|
|
studentLessonDatas = LessonETLService.GetSmartRatingData(lessonBase, timeLineData, smartRatingDatas, studentLessonDatas, id);
|
|
|
studentLessonDatas = LessonETLService.GetTaskData(lessonBase, timeLineData, taskDatas, studentLessonDatas,id);
|
|
|
+ var pickupData = LessonETLService.GetPickupData(lessonBase, timeLineData, studentLessonDatas, id);
|
|
|
+ studentLessonDatas= pickupData.studentLessonDatas;
|
|
|
+
|
|
|
await System.IO.File.WriteAllTextAsync(Path.Combine(path, $"student-analysis.json"), studentLessonDatas.ToJsonString());
|
|
|
string jsons = await System.IO.File.ReadAllTextAsync($"{lessonPath}\\analysis\\analysis.json");
|
|
|
LessonDataAnalysisCluster lessonDataAnalysis = jsons.ToObject<LessonDataAnalysisCluster>();
|
|
|
@@ -246,7 +249,12 @@ namespace HTEX.DataETL.Controllers
|
|
|
}
|
|
|
HashSet<string> yearMonth = new HashSet<string>();
|
|
|
long newest = 0;
|
|
|
- if (loadLocal ||(json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
|
|
|
+ bool force = false;
|
|
|
+ if ((json.TryGetProperty("force", out JsonElement _force)&& _force.ValueKind.Equals(JsonValueKind.True)))
|
|
|
+ {
|
|
|
+ force= _force.GetBoolean();
|
|
|
+ }
|
|
|
+ // if (loadLocal ||force)
|
|
|
{
|
|
|
List<LessonRecord> lessonRecords = new List<LessonRecord>();
|
|
|
var resultSchool = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School)
|
|
|
@@ -256,7 +264,8 @@ namespace HTEX.DataETL.Controllers
|
|
|
newest= resultSchool.list.Max(x => x.startTime);
|
|
|
lessonRecords.AddRange(resultSchool.list);
|
|
|
}
|
|
|
- else {
|
|
|
+ else
|
|
|
+ {
|
|
|
newest=stime;
|
|
|
}
|
|
|
var resultTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher)
|
|
|
@@ -273,7 +282,7 @@ namespace HTEX.DataETL.Controllers
|
|
|
List<string> ignore = new List<string>() { "PgJump", "PgRcv", "PgAdd" };
|
|
|
if (lessonRecords.IsNotEmpty())
|
|
|
{
|
|
|
-
|
|
|
+
|
|
|
await foreach (var item in LessonETLService.GetLessonLocal(lessonRecords, localIds, _azureStorage, pathLessons))
|
|
|
{
|
|
|
string yearMonthPath = DateTimeOffset.FromUnixTimeMilliseconds(item.lessonRecord.startTime).ToString("yyyyMM");
|
|
|
@@ -302,231 +311,15 @@ namespace HTEX.DataETL.Controllers
|
|
|
}
|
|
|
List<TechCount> techCounts = new List<TechCount>();
|
|
|
filesLessons = FileHelper.ListAllFiles(pathLessons, "-local.json");
|
|
|
- await foreach (var item in GetTeachCount(lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage))
|
|
|
+ await foreach (var item in LessonETLService.GetTeachCount(lessonRecords, filesLessons, pathLessons, ignore, Constant.objectiveTypes, _azureStorage, force))
|
|
|
{
|
|
|
techCounts.Add(item);
|
|
|
}
|
|
|
- var yearMonthDatas = techCounts.GroupBy(x => x.yearMonth).Select(x => new { key = x.Key, list = x.ToList() });
|
|
|
- // lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
|
|
|
- LessonDataAnalysisCluster lessonDataAnalysisCluster = new LessonDataAnalysisCluster();
|
|
|
- foreach (var yearMonthData in yearMonthDatas)
|
|
|
- {
|
|
|
- if (string.IsNullOrWhiteSpace(yearMonthData.key))
|
|
|
- {
|
|
|
- Console.WriteLine(yearMonthData.list.Select(x => x.lessonId).ToJsonString());
|
|
|
- }
|
|
|
- LessonDataAnalysisMonth lessonDataAnalysisMonth= new LessonDataAnalysisMonth() { updateTime= newest, yearMonth= yearMonthData.key };
|
|
|
- lessonDataAnalysisMonth.pscore = yearMonthData.list.SelectMany(x => x.pscore).ToList();
|
|
|
- lessonDataAnalysisMonth.tscore = yearMonthData.list.SelectMany(x => x.tscore).ToList();
|
|
|
- lessonDataAnalysisMonth.gscore = yearMonthData.list.SelectMany(x => x.gscore).ToList();
|
|
|
- lessonDataAnalysisMonth.irs= yearMonthData.list.Where(x => x.irsCount>0).Select(x =>(double)x.irsCount).ToList();
|
|
|
- lessonDataAnalysisMonth.interactNormal= yearMonthData.list.Where(x => x.interactNormalCount > 0).Select(x => (double)x.interactNormalCount).ToList();
|
|
|
- lessonDataAnalysisMonth.task = yearMonthData.list.Where(x => x.taskCount > 0).Select(x => (double)x.taskCount).ToList();
|
|
|
- lessonDataAnalysisMonth.stuCowork=yearMonthData.list.SelectMany(x => x.stuCowork).ToList();
|
|
|
- lessonDataAnalysisMonth.groupCowork=yearMonthData.list.SelectMany(x => x.groupCowork).ToList();
|
|
|
- System.IO.File.WriteAllText(Path.Combine(pathAnalysis, $"{yearMonthData.key}-m-analysis.json"), lessonDataAnalysisMonth.ToJsonString());
|
|
|
- // lessonDataAnalysisMonths.Add( lessonDataAnalysisMonth);
|
|
|
- if (lessonDataAnalysisMonth.task.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.task.AddRange(lessonDataAnalysisMonth.task);
|
|
|
- }
|
|
|
- if (lessonDataAnalysisMonth.irs.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.irs.AddRange(lessonDataAnalysisMonth.irs);
|
|
|
- }
|
|
|
- if (lessonDataAnalysisMonth.interactNormal.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.interactNormal.AddRange(lessonDataAnalysisMonth.interactNormal);
|
|
|
- }
|
|
|
- if (lessonDataAnalysisMonth.pscore.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.pscore.AddRange(lessonDataAnalysisMonth.pscore);
|
|
|
- }
|
|
|
- if (lessonDataAnalysisMonth.gscore.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.gscore.AddRange(lessonDataAnalysisMonth.gscore);
|
|
|
- }
|
|
|
- if (lessonDataAnalysisMonth.tscore.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.tscore.AddRange(lessonDataAnalysisMonth.tscore);
|
|
|
- }
|
|
|
- if (lessonDataAnalysisMonth.stuCowork.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.stuCowork.AddRange(lessonDataAnalysisMonth.stuCowork);
|
|
|
- }
|
|
|
- if (lessonDataAnalysisMonth.groupCowork.IsNotEmpty())
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.groupCowork.AddRange(lessonDataAnalysisMonth.groupCowork);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- //标准差偏差N倍，视为异常数据
|
|
|
- int thresholdMultiplier = 2;
|
|
|
- lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
|
|
|
- lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
|
|
|
- lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
|
|
|
- lessonDataAnalysisCluster.irs = LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.irs.OrderBy(x => x), thresholdMultiplier);
|
|
|
- lessonDataAnalysisCluster.interactNormal=LessonETLService. CleanDataBySDThreshold(lessonDataAnalysisCluster.interactNormal.OrderBy(x => x), thresholdMultiplier);
|
|
|
- lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x=>x), thresholdMultiplier);
|
|
|
- lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
|
|
|
- var d = lessonDataAnalysisCluster.interactNormal.Select(x => x).OrderBy(x => x).ToArray();
|
|
|
- // 设置0 为自动规划聚类，11 则为自动规划后得到的数字。
|
|
|
- var clusterInteract = KMeansService.KMeansOptimized(d,11,10);
|
|
|
- //foreach (var item in clusterInteract)
|
|
|
- //{
|
|
|
- // Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()}");
|
|
|
- //}
|
|
|
- foreach (var item in clusterInteract.OrderBy(x => x.Key))
|
|
|
- {
|
|
|
- lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
|
|
|
- //Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
|
|
|
- }
|
|
|
- System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
|
|
|
+ LessonETLService. GenAnalysisData(pathAnalysis, newest, techCounts);
|
|
|
}
|
|
|
return Ok(new { yearMonth });
|
|
|
}
|
|
|
|
|
|
- private static async IAsyncEnumerable<TechCount> GetTeachCount(List<LessonRecord> lessonRecords,List<string> filesLessons, string pathLessons, List<string> ignore, List<string> objectiveTypes,AzureStorageFactory azureStorage)
|
|
|
- {
|
|
|
- foreach (var item in filesLessons)
|
|
|
- {
|
|
|
- if (item.EndsWith("-local.json"))
|
|
|
- {
|
|
|
- TechCount count = null;
|
|
|
-
|
|
|
- string lessonId = item.Split("\\").Last().Replace("-local.json", "");
|
|
|
- string countFile = item.Replace("-local.json", "-count.json");
|
|
|
- if (System.IO.File.Exists(countFile))
|
|
|
- {
|
|
|
- if (System.IO.File.Exists(countFile))
|
|
|
- {
|
|
|
- string contjson = await System.IO.File.ReadAllTextAsync(countFile);
|
|
|
- count = contjson.ToObject<TechCount>();
|
|
|
- }
|
|
|
- }
|
|
|
- if (string.IsNullOrWhiteSpace(count?.yearMonth))
|
|
|
- {
|
|
|
- string localjson = await System.IO.File.ReadAllTextAsync(item);
|
|
|
- var lessonLocal = localjson.ToObject<LessonLocal>();
|
|
|
- count = new TechCount();
|
|
|
- count.lessonId=item.Split("\\").Last().Replace("-local.json", "");
|
|
|
- count.examCount= lessonLocal.examDatas.Count;
|
|
|
- count.taskCount= lessonLocal.taskDatas.Count;
|
|
|
- count.irsCount= lessonLocal.irsDatas.Count;
|
|
|
- count.interactNormalCount=count.irsCount;
|
|
|
- count.coworkCount= lessonLocal.coworkDatas.Count;
|
|
|
- count.smartRatingCount= lessonLocal.smartRatingDatas.Count;
|
|
|
- count.timeCount=lessonLocal.sokratesDatas.Where(x => !ignore.Contains(x.Event) && !x.Event.Contains("End", StringComparison.OrdinalIgnoreCase)).GroupBy(x => x.Event).Select(x => new CodeLong() { code=x.Key, value= x.ToList().Count }).ToList();
|
|
|
-
|
|
|
-
|
|
|
- if (lessonLocal.lessonRecord!=null)
|
|
|
- {
|
|
|
-
|
|
|
- count.yearMonth=DateTimeOffset.FromUnixTimeMilliseconds(lessonLocal.lessonRecord.startTime).ToString("yyyyMM");
|
|
|
-
|
|
|
- if (lessonLocal?.lessonBase?.summary!=null)
|
|
|
- {
|
|
|
- count.smartRatingCountBase=lessonLocal.lessonBase.summary.smartRatingCount;
|
|
|
- count.irsCountBase=lessonLocal.lessonBase.summary.interactionCount;
|
|
|
- count.taskCountBase=lessonLocal.lessonBase.summary.collateTaskCount;
|
|
|
- count.coworkCountBase=lessonLocal.lessonBase.summary.coworkTaskCount;
|
|
|
- count.examCountBase=lessonLocal.lessonBase.summary.examCount;
|
|
|
- count.interactNormalCountBase= count.interactNormalCount;
|
|
|
- }
|
|
|
- if (lessonLocal?.lessonBase?.report?.clientSummaryList!=null)
|
|
|
- {
|
|
|
- count.pscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.score>0).Select(x => x.score);
|
|
|
- count.gscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.groupScore>0).Select(x => x.groupScore);
|
|
|
- count.tscore= lessonLocal.lessonBase.report.clientSummaryList.Where(x => x.interactScore>0).Select(x => x.interactScore);
|
|
|
- }
|
|
|
- ///处理学生数据
|
|
|
- {
|
|
|
- List<StudentLessonData> studentLessonDatas = lessonLocal.studentLessonDatas.ToJsonString().ToObject<List<StudentLessonData>>();
|
|
|
- studentLessonDatas = LessonETLService.GetBaseInfo(lessonLocal.lessonBase!, studentLessonDatas, lessonLocal?.lessonRecord?.id);
|
|
|
- studentLessonDatas = LessonETLService.GetIRSData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.irsDatas, studentLessonDatas, lessonLocal.examDatas, lessonLocal.lessonRecord.id);
|
|
|
- studentLessonDatas = LessonETLService.GetCoworkData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.coworkDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
|
|
|
- studentLessonDatas = LessonETLService.GetExamData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.examDatas, studentLessonDatas, objectiveTypes, lessonLocal.lessonRecord.id);
|
|
|
- studentLessonDatas = LessonETLService.GetSmartRatingData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.smartRatingDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
|
|
|
- studentLessonDatas = LessonETLService.GetTaskData(lessonLocal.lessonBase!, lessonLocal.timeLineData!, lessonLocal.taskDatas, studentLessonDatas, lessonLocal.lessonRecord.id);
|
|
|
- // var techCount = techCounts.Find(x => !string.IsNullOrWhiteSpace(x.lessonId) && !string.IsNullOrWhiteSpace(lessonLocal?.lessonRecord?.id) && x.lessonId.Equals(lessonLocal.lessonRecord.id));
|
|
|
- int sumUpload = 0;
|
|
|
- int taskCount = 0;
|
|
|
- int maxUpload = 0;
|
|
|
- //HashSet<string> pickUp = new HashSet<string>();
|
|
|
- foreach (var stu in studentLessonDatas)
|
|
|
- {
|
|
|
- var countS = stu.taskRecord.itemRecords.Where(x => x.optCount>0);
|
|
|
- if (countS.Count()>0)
|
|
|
- {
|
|
|
- int stuUploadmax = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Max(x => x.optCount);
|
|
|
- if (stuUploadmax> maxUpload)
|
|
|
- {
|
|
|
- maxUpload=stuUploadmax;
|
|
|
- }
|
|
|
- }
|
|
|
- int stuUpload = stu.taskRecord.itemRecords.Where(x => x.optCount>0).Sum(x => x.optCount);
|
|
|
-
|
|
|
- sumUpload+=stuUpload;
|
|
|
- if (stu.taskRecord.itemRecords.Count()> taskCount)
|
|
|
- {
|
|
|
- taskCount=stu.taskRecord.itemRecords.Count();
|
|
|
- }
|
|
|
- var stu_scores = stu.coworkRecord.itemRecords.Where(x => x.itemScore>0).Select(x => x.itemScore);
|
|
|
- if (stu_scores!=null && stu_scores.Count()>0)
|
|
|
- {
|
|
|
- count.stuCowork.AddRange(stu_scores);
|
|
|
- }
|
|
|
- var grp_scores = stu.group_coworkScore.Where(x => x>0);
|
|
|
- if (grp_scores!=null && grp_scores.Count()>0)
|
|
|
- {
|
|
|
- count.groupCowork.AddRange(grp_scores);
|
|
|
- }
|
|
|
- //if (stu.pickups.IsNotEmpty())
|
|
|
- //{
|
|
|
- // foreach (var pickup in stu.pickups)
|
|
|
- // {
|
|
|
- // pickUp.Add(pickup);
|
|
|
- // }
|
|
|
- //}
|
|
|
- }
|
|
|
- if (studentLessonDatas.Count>0&& taskCount>0 && maxUpload>0)
|
|
|
- {
|
|
|
- var avgUpload = sumUpload*1.0/(studentLessonDatas.Count *taskCount);
|
|
|
- count.upload.Add(new List<double>() { avgUpload, maxUpload });
|
|
|
- }
|
|
|
- //if (pickUp.Count>0)
|
|
|
- //{
|
|
|
- // count.pickup.Add(pickUp.ToList());
|
|
|
- //}
|
|
|
- string owner = lessonLocal.lessonRecord.scope.Equals("school") ? lessonLocal.lessonRecord.school : lessonLocal.lessonRecord.tmdid;
|
|
|
- // if (!azureStorage.GetBlobContainerClient(owner).GetBlobClient($"records/{lessonLocal.lessonRecord.id}/student-analysis.json").Exists())
|
|
|
- {
|
|
|
-
|
|
|
-
|
|
|
- }
|
|
|
- // if (!System.IO.File.Exists($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json"))
|
|
|
- {
|
|
|
-
|
|
|
- }
|
|
|
- if (lessonRecords.FindAll(x => x.id.Equals(lessonLocal.lessonRecord.id)).IsNotEmpty())
|
|
|
- {
|
|
|
- await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-stu.json", studentLessonDatas.ToJsonString());
|
|
|
- await azureStorage.GetBlobContainerClient(owner).UploadFileByContainer(studentLessonDatas.ToJsonString(), "records", $"{lessonLocal.lessonRecord.id}/student-analysis.json");
|
|
|
- await System.IO.File.WriteAllTextAsync($"{pathLessons}\\MM{count.yearMonth}\\{lessonLocal.lessonRecord.id}-count.json", count.ToJsonString());
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- if (count!=null)
|
|
|
- {
|
|
|
- yield return count;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
+
|
|
|
}
|
|
|
}
|
CrazyIter_Bin