TEAMModelOS/TEAMModelOS.Extension/HTEX.Test/Program.cs

using Azure;
using Bogus;
using HTEX.Lib.ETL.Lesson;
using HTEX.Test.Service;
using MathNet.Numerics;
using MathNet.Numerics.LinearAlgebra;
using System;
using System.Configuration;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
using TEAMModelOS.SDK;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Helper.Common.FileHelper;
using TEAMModelOS.SDK.Models;
using TEAMModelOS.SDK.Models.Cosmos.Student;
using static HTEX.Test.Controllers.LessonRecordController;
using static Microsoft.Azure.Amqp.Serialization.SerializableType;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace HTEX.Test
{
    public class Program
    {
        public static void Main(string[] args)
        {
            Test2();


            var builder = WebApplication.CreateBuilder(args);

            // Add services to the container.

            builder.Services.AddControllers();
            builder.Services.AddAzureStorage(builder.Configuration.GetValue<string>("Azure:Storage:ConnectionString"));
            builder.Services.AddAzureRedis(builder.Configuration.GetValue<string>("Azure:Redis:ConnectionString"));
            builder.Services.AddAzureCosmos(builder.Configuration.GetValue<string>("Azure:Cosmos:ConnectionString"));
            builder.Services.AddAzureServiceBus(builder.Configuration.GetValue<string>("Azure:ServiceBus:ConnectionString"));
            builder.Services.AddAzureSignalR(builder.Configuration.GetValue<string>("Azure:SignalR:ConnectionString"));
            builder.Services.AddSnowflakeId(Convert.ToInt64(builder.Configuration.GetValue<string>("Option:LocationNum")), 1);
            builder.Services.AddHttpClient();
            var app = builder.Build();

            // Configure the HTTP request pipeline.

            app.UseHttpsRedirection();

            app.UseAuthorization();


            app.MapControllers();

            app.Run();
        }

        public static string Test2() 
        {
            string? pathAnalysis = $"F:\\lesson-local\\analysis";
            var filesAnalysis = FileHelper.ListAllFiles(pathAnalysis);
            List<LessonDataAnalysisMonth> lessonDataAnalysisMonths = new List<LessonDataAnalysisMonth>();
            LessonDataAnalysisCluster lessonDataAnalysisCluster = new LessonDataAnalysisCluster();
            foreach (var file in filesAnalysis)
            {
                //读取每月的数据
                if (file.EndsWith("-m-analysis.json"))
                {
                    string jsons =   System.IO.File.ReadAllText(file);
                    LessonDataAnalysisMonth lessonDataAnalysisMonth = jsons.ToObject<LessonDataAnalysisMonth>();

                    lessonDataAnalysisMonths.Add(lessonDataAnalysisMonth);
                    if (lessonDataAnalysisMonth.task.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.task.AddRange(lessonDataAnalysisMonth.task);
                    }
                    if (lessonDataAnalysisMonth.irs.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.irs.AddRange(lessonDataAnalysisMonth.irs);
                    }
                    if (lessonDataAnalysisMonth.interactNormal.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.interactNormal.AddRange(lessonDataAnalysisMonth.interactNormal);
                    }
                    if (lessonDataAnalysisMonth.pscore.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.pscore.AddRange(lessonDataAnalysisMonth.pscore);
                    }
                    if (lessonDataAnalysisMonth.gscore.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.gscore.AddRange(lessonDataAnalysisMonth.gscore);
                    }
                    if (lessonDataAnalysisMonth.tscore.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.tscore.AddRange(lessonDataAnalysisMonth.tscore);
                    }
                    if (lessonDataAnalysisMonth.stuCowork.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.stuCowork.AddRange(lessonDataAnalysisMonth.stuCowork);
                    }
                    if (lessonDataAnalysisMonth.groupCowork.IsNotEmpty())
                    {
                        lessonDataAnalysisCluster.groupCowork.AddRange(lessonDataAnalysisMonth.groupCowork);
                    }
                }
            }
          
            //标准差偏差N倍，视为异常数据
            double thresholdMultiplier =2;
            lessonDataAnalysisCluster.pscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.pscore.OrderBy(x => x), thresholdMultiplier);
            lessonDataAnalysisCluster.gscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.gscore.OrderBy(x => x), thresholdMultiplier);
            lessonDataAnalysisCluster.tscore= LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.tscore.OrderBy(x => x), thresholdMultiplier);
            lessonDataAnalysisCluster.irs = LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.irs.OrderBy(x => x), thresholdMultiplier);
            lessonDataAnalysisCluster.interactNormal=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.interactNormal.OrderBy(x => x), thresholdMultiplier);
            lessonDataAnalysisCluster.stuCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.stuCowork.OrderBy(x => x), thresholdMultiplier);
            lessonDataAnalysisCluster.groupCowork=LessonETLService.CleanDataBySDThreshold(lessonDataAnalysisCluster.groupCowork.OrderBy(x => x), thresholdMultiplier);
             //超过60 80的


            var d = lessonDataAnalysisCluster.interactNormal.Select(x =>  x).OrderBy(x => x).ToArray();
            double n = d.Max()+1;
            var clusterInteract = HTEX.Lib.ETL. KMeansService.KMeansOptimized(d,3);
            foreach (var item in clusterInteract.OrderBy(x => x.Key))
            {
                lessonDataAnalysisCluster.clustersInteract.Add(new KeyValuePair<double, List<double>>(item.Value.Average(), item.Value));
                Console.WriteLine($"dp:{item.Key} ,avg: {item.Value.Average()}, count: {item.Value.Count}, min：{item.Value.Min()}, max：{item.Value.Max()},weight:{item.Value.Count*1.0/d.Count()}");
            }

            IEnumerable<double> all = lessonDataAnalysisCluster.clustersInteract.SelectMany(x => x.Value);
            int pass = 0;
            for (var i = 1; i<n; i++) 
            {
                var p = LessonETLService.GetPersent(all, i);
                if (p.persent>=60) 
                {
                    pass = i;
                    break;
                }
            }
            int good = 0;
            for (var i = 1; i<n; i++)
            {
                var p = LessonETLService.GetPersent(all, i);
                if (p.persent>=80)
                {
                    good = i;
                    break;
                }
            }
            double ss = 0;
            for (var i = 1; i<n; i++) 
            {
               // KeyValuePair<double, List<int>> curr = new KeyValuePair<double, List<int>>();
                var s = lessonDataAnalysisCluster.clustersInteract.FindAll(x => x.Value.Min()<=i  && x.Value.Max()>=i).MinBy(x=>x.Key);
                var p = LessonETLService.GetPersent(all, i);
                var l= i<pass?pass:i<good?pass:good;
                var e = (i*1.0/l) *(p.persent)  *  (s.Value.Count*1.0/d.Count());
                ss+= e;
                Console.WriteLine($"n: {i},l: {l}, persent: {p.persent},count: {p.count},s-min: {s.Value.Min()},s-max: {s.Value.Max()},value: {e}");
            }

            //foreach (var s in clusterInteract.OrderBy(x => x.Key))
            //{
            //    clustersDataInteract.Add(s);
            //}
            //lessonDataAnalysisCluster.clustersInteract= clustersDataInteract;
            System.IO.File.WriteAllText(Path.Combine(pathAnalysis, "analysis.json"), lessonDataAnalysisCluster.ToJsonString());
            return "";
        }

        public static string Test()
        {
            
            // 创建一个随机数生成器
            Random random = new Random();

            // 示例数据集
            var data = Enumerable.Range(1, 25000).Select(i => Vector<double>.Build.Dense(i % 10, i % 10)).ToArray();
          
            return "Hello World!";
        }
         
    static bool[] MarkAnomalies(List<int> array)
        {
            if (array.Count == 0) return new bool[0];

            double average = array.Average();
            double variance = array.Average(x => Math.Pow(x - average, 2));
            double standardDeviation = Math.Sqrt(variance);

            // 定义异常值的阈值，这里使用2倍标准差
            double threshold =3* standardDeviation;

            bool[] anomalies = new bool[array.Count];
            for (int i = 0; i < array.Count; i++)
            {
                double deviation = Math.Abs(array[i] - average);
                anomalies[i] = deviation > threshold;
            }

            return anomalies;
        }

       
       
        
        /// <summary>
        /// 计算当前元素在集合中超过了多少百分比的值
        /// </summary>
        /// <param name="nums"></param>
        /// <param name="curr"></param>
        /// <returns></returns>
        public static double GetPersent(IEnumerable<double> nums, int curr)
        {
            int count = 0;
            foreach (var op in nums.OrderBy(x => x))
            {
                if (op < curr)
                {
                    count++;
                }
                else if (op == curr)
                {
                    count++;
                }
                else
                {
                    break;
                }
            }
            return count *1.0/ nums.Count() * 100;
        }


        public class KeyCount
        {
            public int count { get; set;}
            public int key { get; set;}
        }
        class WeightedItem
        {
            public int Value { get; set; }
            public double Weight { get; set; }
        }
    }
}