Program.cs 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272
  1. using Azure;
  2. using Bogus;
  3. using HTEX.Lib.ETL.Lesson;
  4. using HTEX.Test.Controllers;
  5. using HTEX.Test.Service;
  6. using MathNet.Numerics;
  7. using MathNet.Numerics.LinearAlgebra;
  8. using System;
  9. using System.Configuration;
  10. using System.Diagnostics;
  11. using System.IO;
  12. using System.Linq;
  13. using System.Reflection;
  14. using System.Text.RegularExpressions;
  15. using TEAMModelOS.SDK;
  16. using TEAMModelOS.SDK.DI;
  17. using TEAMModelOS.SDK.Extension;
  18. using TEAMModelOS.SDK.Helper.Common.FileHelper;
  19. using TEAMModelOS.SDK.Models;
  20. using TEAMModelOS.SDK.Models.Cosmos.Student;
  21. using static Microsoft.Azure.Amqp.Serialization.SerializableType;
  22. using static System.Runtime.InteropServices.JavaScript.JSType;
  23. namespace HTEX.Test
  24. {
  25. public class Program
  26. {
  27. public static async Task Main(string[] args)
  28. {
  29. //Test2();
  30. await MockDataController.MockData();
  31. //var builder = WebApplication.CreateBuilder(args);
  32. //// Add services to the container.
  33. //builder.Services.AddControllers();
  34. //builder.Services.AddAzureStorage(builder.Configuration.GetValue<string>("Azure:Storage:ConnectionString"));
  35. //builder.Services.AddAzureRedis(builder.Configuration.GetValue<string>("Azure:Redis:ConnectionString"));
  36. //builder.Services.AddAzureCosmos(builder.Configuration.GetValue<string>("Azure:Cosmos:ConnectionString"));
  37. //builder.Services.AddAzureServiceBus(builder.Configuration.GetValue<string>("Azure:ServiceBus:ConnectionString"));
  38. //builder.Services.AddAzureSignalR(builder.Configuration.GetValue<string>("Azure:SignalR:ConnectionString"));
  39. //builder.Services.AddSnowflakeId(Convert.ToInt64(builder.Configuration.GetValue<string>("Option:LocationNum")), 1);
  40. //builder.Services.AddHttpClient();
  41. //var app = builder.Build();
  42. //// Configure the HTTP request pipeline.
  43. //app.UseHttpsRedirection();
  44. //app.UseAuthorization();
  45. //app.MapControllers();
  46. //app.Run();
  47. }
  48. static double CalculateScore(int n, double k = 1)
  49. {
  50. if (n == 0)
  51. {
  52. return 0;
  53. }
  54. else
  55. {
  56. double score = 60 + (40 / (1 + Math.Exp(-k * (n - 1))));
  57. return Math.Max(80, Math.Min(score, 100));
  58. }
  59. }
  60. public static string Test2()
  61. {
  62. string? pathAnalysis = $"F:\\lesson-local\\analysis";
  63. var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
  64. List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
  65. LessonDataAnalysisCluster lessonDataAnalysisCluster = new LessonDataAnalysisCluster();
  66. foreach (var file in filesAnalysis)
  67. {
  68. //读取每月的数据
  69. if (file.EndsWith("-m-analysis.json"))
  70. {
  71. string jsons = System.IO.File.ReadAllText(file);
  72. LessonDataAnalysisMonth lessonDataAnalysisMonth = jsons.ToObject<LessonDataAnalysisMonth>();
  73. lessonDataAnalysisMonths.Add(lessonDataAnalysisMonth);
  74. if (lessonDataAnalysisMonth.task.IsNotEmpty())
  75. {
  76. lessonDataAnalysisCluster.task.AddRange(lessonDataAnalysisMonth.task);
  77. }
  78. if (lessonDataAnalysisMonth.irs.IsNotEmpty())
  79. {
  80. lessonDataAnalysisCluster.irs.AddRange(lessonDataAnalysisMonth.irs);
  81. }
  82. if (lessonDataAnalysisMonth.interactNormal.IsNotEmpty())
  83. {
  84. lessonDataAnalysisCluster.interactNormal.AddRange(lessonDataAnalysisMonth.interactNormal);
  85. }
  86. if (lessonDataAnalysisMonth.pscore.IsNotEmpty())
  87. {
  88. lessonDataAnalysisCluster.pscore.AddRange(lessonDataAnalysisMonth.pscore);
  89. }
  90. if (lessonDataAnalysisMonth.gscore.IsNotEmpty())
  91. {
  92. lessonDataAnalysisCluster.gscore.AddRange(lessonDataAnalysisMonth.gscore);
  93. }
  94. if (lessonDataAnalysisMonth.tscore.IsNotEmpty())
  95. {
  96. lessonDataAnalysisCluster.tscore.AddRange(lessonDataAnalysisMonth.tscore);
  97. }
  98. if (lessonDataAnalysisMonth.stuCowork.IsNotEmpty())
  99. {
  100. lessonDataAnalysisCluster.stuCowork.AddRange(lessonDataAnalysisMonth.stuCowork);
  101. }
  102. if (lessonDataAnalysisMonth.groupCowork.IsNotEmpty())
  103. {
  104. lessonDataAnalysisCluster.groupCowork.AddRange(lessonDataAnalysisMonth.groupCowork);
  105. }
  106. }
  107. }
  108. //标准差偏差N倍,视为异常数据
  109. double thresholdMultiplier =2;
  110. lessonDataAnalysisCluster.task= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.task.OrderBy(x => x), thresholdMultiplier);
  111. lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
  112. lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
  113. lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
  114. lessonDataAnalysisCluster.irs = LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.irs.OrderBy(x => x), thresholdMultiplier);
  115. lessonDataAnalysisCluster.interactNormal=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.interactNormal.OrderBy(x => x), thresholdMultiplier);
  116. lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x => x), thresholdMultiplier);
  117. lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
  118. //超过60 80的
  119. var all = lessonDataAnalysisCluster.interactNormal.Select(x => x).OrderBy(x => x).ToArray();
  120. double n = all.Max()+1;
  121. var clusterInteract = HTEX.Lib.ETL. KMeansService.KMeansOptimized(all,3);
  122. //foreach (var item in clusterInteract.OrderBy(x => x.Key))
  123. //{
  124. // lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
  125. // Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min:{item.Value.Min()}, max:{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
  126. //}
  127. //IEnumerable<double> all = lessonDataAnalysisCluster.clustersInteract.SelectMany(x => x.Value);
  128. int pass = 0;
  129. for (var i = 1; i<n; i++)
  130. {
  131. var p = LessonETLService.GetPersent(all, i);
  132. if (p.persent>=60)
  133. {
  134. pass = i;
  135. break;
  136. }
  137. }
  138. var low = all.Where(x => x<pass).Average();
  139. int good = 0;
  140. for (var i = 1; i<n; i++)
  141. {
  142. var p = LessonETLService.GetPersent(all, i);
  143. if (p.persent>=80)
  144. {
  145. good = i;
  146. break;
  147. }
  148. }
  149. var medium = all.Where(x => x>=pass &&x<good ).Average();
  150. var high = all.Where(x => x>=good).Average();
  151. lessonDataAnalysisCluster.interactGood=good;
  152. lessonDataAnalysisCluster.interactPass=pass;
  153. lessonDataAnalysisCluster.interactHigh=high;
  154. lessonDataAnalysisCluster.interactMedium=medium;
  155. lessonDataAnalysisCluster.interactLow=low;
  156. List<KeyValuePair<double, List<double>>> levelInteract = new List<KeyValuePair<double, List<double>>>()
  157. {
  158. new KeyValuePair<double, List<double>>(low, all.Where(x => x<pass).ToList()),
  159. new KeyValuePair<double, List<double>>(medium, all.Where(x => x>=pass &&x<good).ToList() ),
  160. new KeyValuePair<double, List<double>>(high, all.Where(x => x>=good).ToList())
  161. };
  162. lessonDataAnalysisCluster.levelInteract= levelInteract;
  163. double ss = 0;
  164. for (var i = 1; i<n; i++)
  165. {
  166. // KeyValuePair<double, List<int>> curr = new KeyValuePair<double, List<int>>();
  167. var s = lessonDataAnalysisCluster.levelInteract.FindAll(x => x.Value.Min()<=i && x.Value.Max()>=i).MinBy(x=>x.Key);
  168. var p = LessonETLService.GetPersent(all, i);
  169. var l= i<pass?pass:i<good?pass:good;
  170. var e = (i*1.0/l) *(p.persent) * (s.Value.Count*1.0/all.Count());
  171. ss+= e;
  172. Console.WriteLine($"n: {i},l: {l}, persent: {p.persent},count: {p.count},s-min: {s.Value.Min()},s-max: {s.Value.Max()},value: {e}");
  173. }
  174. //foreach (var s in clusterInteract.OrderBy(x => x.Key))
  175. //{
  176. // clustersDataInteract.Add(s);
  177. //}
  178. //lessonDataAnalysisCluster.clustersInteract= clustersDataInteract;
  179. System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
  180. return "";
  181. }
  182. public static string Test()
  183. {
  184. // 创建一个随机数生成器
  185. Random random = new Random();
  186. // 示例数据集
  187. var data = Enumerable.Range(1, 25000).Select(i => Vector<double>.Build.Dense(i % 10, i % 10)).ToArray();
  188. return "Hello World!";
  189. }
  190. static bool[] MarkAnomalies(List<int> array)
  191. {
  192. if (array.Count == 0) return new bool[0];
  193. double average = array.Average();
  194. double variance = array.Average(x => Math.Pow(x - average, 2));
  195. double standardDeviation = Math.Sqrt(variance);
  196. // 定义异常值的阈值,这里使用2倍标准差
  197. double threshold =3* standardDeviation;
  198. bool[] anomalies = new bool[array.Count];
  199. for (int i = 0; i < array.Count; i++)
  200. {
  201. double deviation = Math.Abs(array[i] - average);
  202. anomalies[i] = deviation > threshold;
  203. }
  204. return anomalies;
  205. }
  206. /// <summary>
  207. /// 计算当前元素在集合中超过了多少百分比的值
  208. /// </summary>
  209. /// <param name="nums"></param>
  210. /// <param name="curr"></param>
  211. /// <returns></returns>
  212. public static double GetPersent(IEnumerable<double> nums, int curr)
  213. {
  214. int count = 0;
  215. foreach (var op in nums.OrderBy(x => x))
  216. {
  217. if (op < curr)
  218. {
  219. count++;
  220. }
  221. else if (op == curr)
  222. {
  223. count++;
  224. }
  225. else
  226. {
  227. break;
  228. }
  229. }
  230. return count *1.0/ nums.Count() * 100;
  231. }
  232. public class KeyCount
  233. {
  234. public int count { get; set;}
  235. public int key { get; set;}
  236. }
  237. class WeightedItem
  238. {
  239. public int Value { get; set; }
  240. public double Weight { get; set; }
  241. }
  242. }
  243. }