123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246 |
- using Azure;
- using Bogus;
- using HTEX.Lib.ETL.Lesson;
- using HTEX.Test.Service;
- using MathNet.Numerics;
- using MathNet.Numerics.LinearAlgebra;
- using System;
- using System.Configuration;
- using System.Diagnostics;
- using System.IO;
- using System.Linq;
- using System.Reflection;
- using System.Text.RegularExpressions;
- using TEAMModelOS.SDK;
- using TEAMModelOS.SDK.DI;
- using TEAMModelOS.SDK.Extension;
- using TEAMModelOS.SDK.Helper.Common.FileHelper;
- using TEAMModelOS.SDK.Models;
- using TEAMModelOS.SDK.Models.Cosmos.Student;
- using static HTEX.Test.Controllers.LessonRecordController;
- using static Microsoft.Azure.Amqp.Serialization.SerializableType;
- using static System.Runtime.InteropServices.JavaScript.JSType;
- namespace HTEX.Test
- {
- public class Program
- {
- public static void Main(string[] args)
- {
- Test2();
- var builder = WebApplication.CreateBuilder(args);
- // Add services to the container.
- builder.Services.AddControllers();
- builder.Services.AddAzureStorage(builder.Configuration.GetValue<string>("Azure:Storage:ConnectionString"));
- builder.Services.AddAzureRedis(builder.Configuration.GetValue<string>("Azure:Redis:ConnectionString"));
- builder.Services.AddAzureCosmos(builder.Configuration.GetValue<string>("Azure:Cosmos:ConnectionString"));
- builder.Services.AddAzureServiceBus(builder.Configuration.GetValue<string>("Azure:ServiceBus:ConnectionString"));
- builder.Services.AddAzureSignalR(builder.Configuration.GetValue<string>("Azure:SignalR:ConnectionString"));
- builder.Services.AddSnowflakeId(Convert.ToInt64(builder.Configuration.GetValue<string>("Option:LocationNum")), 1);
- builder.Services.AddHttpClient();
- var app = builder.Build();
- // Configure the HTTP request pipeline.
- app.UseHttpsRedirection();
- app.UseAuthorization();
- app.MapControllers();
- app.Run();
- }
- public static string Test2()
- {
- string? pathAnalysis = $"F:\\lesson-local\\analysis";
- var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
- List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
- LessonDataAnalysisCluster lessonDataAnalysisCluster = new LessonDataAnalysisCluster();
- foreach (var file in filesAnalysis)
- {
- //读取每月的数据
- if (file.EndsWith("-m-analysis.json"))
- {
- string jsons = System.IO.File.ReadAllText(file);
- LessonDataAnalysisMonth lessonDataAnalysisMonth = jsons.ToObject<LessonDataAnalysisMonth>();
- lessonDataAnalysisMonths.Add(lessonDataAnalysisMonth);
- if (lessonDataAnalysisMonth.task.IsNotEmpty())
- {
- lessonDataAnalysisCluster.task.AddRange(lessonDataAnalysisMonth.task);
- }
- if (lessonDataAnalysisMonth.irs.IsNotEmpty())
- {
- lessonDataAnalysisCluster.irs.AddRange(lessonDataAnalysisMonth.irs);
- }
- if (lessonDataAnalysisMonth.interactNormal.IsNotEmpty())
- {
- lessonDataAnalysisCluster.interactNormal.AddRange(lessonDataAnalysisMonth.interactNormal);
- }
- if (lessonDataAnalysisMonth.pscore.IsNotEmpty())
- {
- lessonDataAnalysisCluster.pscore.AddRange(lessonDataAnalysisMonth.pscore);
- }
- if (lessonDataAnalysisMonth.gscore.IsNotEmpty())
- {
- lessonDataAnalysisCluster.gscore.AddRange(lessonDataAnalysisMonth.gscore);
- }
- if (lessonDataAnalysisMonth.tscore.IsNotEmpty())
- {
- lessonDataAnalysisCluster.tscore.AddRange(lessonDataAnalysisMonth.tscore);
- }
- if (lessonDataAnalysisMonth.stuCowork.IsNotEmpty())
- {
- lessonDataAnalysisCluster.stuCowork.AddRange(lessonDataAnalysisMonth.stuCowork);
- }
- if (lessonDataAnalysisMonth.groupCowork.IsNotEmpty())
- {
- lessonDataAnalysisCluster.groupCowork.AddRange(lessonDataAnalysisMonth.groupCowork);
- }
- }
- }
-
- //标准差偏差N倍,视为异常数据
- double thresholdMultiplier =2;
- lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
- lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
- lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
- lessonDataAnalysisCluster.irs = LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.irs.OrderBy(x => x), thresholdMultiplier);
- lessonDataAnalysisCluster.interactNormal=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.interactNormal.OrderBy(x => x), thresholdMultiplier);
- lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x => x), thresholdMultiplier);
- lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
- //超过60 80的
- var d = lessonDataAnalysisCluster.interactNormal.Select(x => x).OrderBy(x => x).ToArray();
- double n = d.Max()+1;
- var clusterInteract = HTEX.Lib.ETL. KMeansService.KMeansOptimized(d,3);
- foreach (var item in clusterInteract.OrderBy(x => x.Key))
- {
- lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
- Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min:{item.Value.Min()}, max:{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
- }
- IEnumerable<double> all = lessonDataAnalysisCluster.clustersInteract.SelectMany(x => x.Value);
- int pass = 0;
- for (var i = 1; i<n; i++)
- {
- var p = LessonETLService.GetPersent(all, i);
- if (p.persent>=60)
- {
- pass = i;
- break;
- }
- }
- int good = 0;
- for (var i = 1; i<n; i++)
- {
- var p = LessonETLService.GetPersent(all, i);
- if (p.persent>=80)
- {
- good = i;
- break;
- }
- }
- double ss = 0;
- for (var i = 1; i<n; i++)
- {
- // KeyValuePair<double, List<int>> curr = new KeyValuePair<double, List<int>>();
- var s = lessonDataAnalysisCluster.clustersInteract.FindAll(x => x.Value.Min()<=i && x.Value.Max()>=i).MinBy(x=>x.Key);
- var p = LessonETLService.GetPersent(all, i);
- var l= i<pass?pass:i<good?pass:good;
- var e = (i*1.0/l) *(p.persent) * (s.Value.Count*1.0/d.Count());
- ss+= e;
- Console.WriteLine($"n: {i},l: {l}, persent: {p.persent},count: {p.count},s-min: {s.Value.Min()},s-max: {s.Value.Max()},value: {e}");
- }
- //foreach (var s in clusterInteract.OrderBy(x => x.Key))
- //{
- // clustersDataInteract.Add(s);
- //}
- //lessonDataAnalysisCluster.clustersInteract= clustersDataInteract;
- System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
- return "";
- }
- public static string Test()
- {
-
- // 创建一个随机数生成器
- Random random = new Random();
- // 示例数据集
- var data = Enumerable.Range(1, 25000).Select(i => Vector<double>.Build.Dense(i % 10, i % 10)).ToArray();
-
- return "Hello World!";
- }
-
- static bool[] MarkAnomalies(List<int> array)
- {
- if (array.Count == 0) return new bool[0];
- double average = array.Average();
- double variance = array.Average(x => Math.Pow(x - average, 2));
- double standardDeviation = Math.Sqrt(variance);
- // 定义异常值的阈值,这里使用2倍标准差
- double threshold =3* standardDeviation;
- bool[] anomalies = new bool[array.Count];
- for (int i = 0; i < array.Count; i++)
- {
- double deviation = Math.Abs(array[i] - average);
- anomalies[i] = deviation > threshold;
- }
- return anomalies;
- }
-
-
-
- /// <summary>
- /// 计算当前元素在集合中超过了多少百分比的值
- /// </summary>
- /// <param name="nums"></param>
- /// <param name="curr"></param>
- /// <returns></returns>
- public static double GetPersent(IEnumerable<double> nums, int curr)
- {
- int count = 0;
- foreach (var op in nums.OrderBy(x => x))
- {
- if (op < curr)
- {
- count++;
- }
- else if (op == curr)
- {
- count++;
- }
- else
- {
- break;
- }
- }
- return count *1.0/ nums.Count() * 100;
- }
- public class KeyCount
- {
- public int count { get; set;}
- public int key { get; set;}
- }
- class WeightedItem
- {
- public int Value { get; set; }
- public double Weight { get; set; }
- }
- }
- }
|