TEAMModelOS/TEAMModelOS.SDK/Models/Service/BI/BILogAnalyseService.cs

using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using TEAMModelOS.SDK.Context.BI;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Models.Cosmos.BI;

namespace TEAMModelOS.SDK.Models.Service.BI
{
    public static class BILogAnalyseService
    {

        /// <summary>
        /// 读取全部的防火墙日志文件并分析保存至
        /// </summary>
        /// <param name="_azureStorage"></param>
        /// <param name="site"></param>
        /// <returns></returns>
        public static async Task<(List<RecCnt> recCnts ,List<string> saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null) 
        {
            var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina);
            if ($"{site}".Equals(BIConst.Global))
            {
                blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global);
            }

            List<RecCnt> recCnts = new();
            List<string> urls = new();
            //地址:    y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
            string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
            await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
            {

                StringBuilder visits = new("[");

                BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
                BlobDownloadInfo download = tempBlobClient.Download();
                var content = download.Content;
                string text;
                using (var streamReader = new StreamReader(content))
                {
                    while ((text = streamReader.ReadLine()) != null)
                    {
                        if (streamReader.EndOfStream)
                            visits.Append($"{text.ToString()}");
                        else
                            visits.Append($"{text.ToString()},");
                    }

                    visits.Append("]");
                    streamReader.Close();
                }

                string input = visits.ToString();
                List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
                DateTimeOffset dtime = DateTimeOffset.UtcNow;
                string cHour = dtime.ToString("yyyyMMddHH");
                string cDay = dtime.ToString("yyyyMMdd");
                if (aGInfos.Count > 0)
                {
                    cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
                    cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
                }

                RecCnt saveCnts = new();

                List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost }).ToList();

                List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
                saveCnts.apiCnt = apiCnt;

                List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
                saveCnts.ipCnt = ipCnt;
                recCnts.Add(saveCnts);

                ////保存存至Blob文件
                var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");

                urls.Add(url);
            }

            return (recCnts, urls);
        }

        /// <summary>
        /// 通过路径获取日志文件并分析结果
        /// </summary>
        /// <param name="_azureStorage"></param>
        /// <param name="path">防火墙路径</param>
        /// <param name="connectStr">连接字串</param>
        /// <returns></returns>
        public static async Task<(List<RecCnt> recCnts, List<string> saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage, IPSearcher _ipSearcher, DingDing _dingDing, string path, string connectName, string timeType = "Hour")
         {
            List<RecCnt> recCnts = new();
            List<string> urls = new();

            DateTimeOffset dtime = DateTimeOffset.UtcNow;
            string cDay = dtime.ToString("yyyyMMdd");

            //天api
            List<RecApiCnt> dayApiCnt = new();
            //天ip
            List<RecIpCnt> dayIpCnt = new();

            //天
            List<MinuteCnt> dayCnts = new();

            try 
            {
                var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-appservicehttplogs", name: connectName);
                await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path))
                {
                    StringBuilder visits = new("[");
                    //BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
                    //BlobDownloadInfo download = tempBlobClient.Download();
                    BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download();

                    var content = download.Content;
                    string text;
                    using (var streamReader = new StreamReader(content))
                    {
                        while ((text = streamReader.ReadLine()) != null)
                        {
                            if (streamReader.EndOfStream)
                                visits.Append($"{text.ToString()}");
                            else
                                visits.Append($"{text.ToString()},");
                        }

                        visits.Append("]");
                        streamReader.Close();
                    }

                    string input = visits.ToString();
                    List<AGInfo> tempAinfos = input.ToObject<List<AGInfo>>();

                    List<AGInfo> tempsert = new List<AGInfo>();
                    List<AGInfo> aGInfos = new List<AGInfo>();

                    tempAinfos.FindAll(x=>x.properties.CsMethod.Equals("POST"))?.ForEach(item =>
                    {
                        string requestUri = item.properties.CsUriStem;
                        if (!string.IsNullOrWhiteSpace(requestUri)) {
                            var isType = StaticValue.suffixName.Where(k => requestUri.Contains(k)).ToList();
                            if (isType.Count == 0)
                                aGInfos.Add(item);
                        }
                        
                    });

                    //foreach (var item in tempAinfos)
                    //{
                    //    string requestUri = item.properties.requestUri;
                    //    var isType = type.Where(k => requestUri.Contains(k)).ToList();
                    //    if (isType.Count == 0)
                    //        aGInfos.Add(item);
                    //}

                    string cHour = dtime.ToString("yyyyMMddHH");
                    string cHH = dtime.ToString("HH");
                    if (aGInfos.Count > 0)
                    {
                        cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
                        cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
                        cHH = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("HH")).First();
                    }

                    RecCnt saveCnts = new();

                    List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost,minute = DateTimeOffset.Parse(s.time).ToString("mm")}).ToList();

                    if (timeType.Equals("Hour"))
                    {
                        //小时
                        List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
                        saveCnts.apiCnt = apiCnt;


                        List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
                        saveCnts.ipCnt = ipCnt;

                        List<MinuteCnt> minCnts = recInfo.GroupBy(a => a.minute).Select(s => new MinuteCnt { minute = s.Key, cnt = s.Count() }).ToList();
                        saveCnts.minCnts = minCnts;

                        var ipcounts = saveCnts.ipCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
                        ipcounts.ForEach(async x => {
                            string region = await _ipSearcher.SearchIpAsync(x.id);
                            if (!string.IsNullOrWhiteSpace(region))
                            {
                                string[] dis = region.Split("·");
                                if (dis.Length >= 2)
                                {
                                    x.code = dis[dis.Length - 1];
                                    x.name = dis[dis.Length - 2];  // 不保留省份
                                                                   //x.name = region.Substring(0, region.LastIndexOf("·"));  //保留省份
                                }
                                else
                                {
                                    var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
                                    if (disrs.Length >= 2)
                                    {
                                        x.code = disrs[disrs.Length - 1];
                                        x.name = disrs[disrs.Length - 2];   //不保留省份
                                                                            //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
                                    }
                                    else
                                    {
                                        x.code = region;
                                        x.name = region;
                                    }
                                }
                            }
                            else
                            {
                                x.name = x.id;
                                x.code = x.id;
                            }
                        });
                        List<RecRegionCnt> regionCnts = new();
                        ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
                            regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cHour });
                        });
                        saveCnts.regionCnts = regionCnts;
                        recCnts.Add(saveCnts);
                        //保存存至Blob文件
                        var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHH}.json");
                        urls.Add(url);
                    }
                    else if (timeType.Equals("Day"))
                    {
                        //天
                        List<RecApiCnt> tempApiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
                        dayApiCnt.AddRange(tempApiCnt);

                        //天
                        List<RecIpCnt> tempIpCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
                        dayIpCnt.AddRange(tempIpCnt);

                        dayCnts.Add(new MinuteCnt { minute = cHH, cnt = recInfo.Count });
                    }
                }

                if (timeType.Equals("Day"))
                {
                    RecCnt dayRecCnt = new();
                    dayRecCnt.apiCnt = dayApiCnt.GroupBy(g => g.api).Select(s => new RecApiCnt { api = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), ip = s.Select(i => i.ip).FirstOrDefault() }).ToList();

                    dayRecCnt.ipCnt = dayIpCnt.GroupBy(g => g.ip).Select(s => new RecIpCnt { ip = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), api = s.Select(i => i.api).FirstOrDefault() }).ToList();
                    dayRecCnt.minCnts = dayCnts;

                    var ipcounts = dayIpCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
                    ipcounts.ForEach(async x => {
                        string region = await _ipSearcher.SearchIpAsync(x.id);
                        if (!string.IsNullOrWhiteSpace(region))
                        {
                            string[] dis = region.Split("·");
                            if (dis.Length >= 2)
                            {
                                x.code = dis[dis.Length - 1];
                                x.name = dis[dis.Length - 2];  // 不保留省份
                                //x.name = region.Substring(0, region.LastIndexOf("·"));  //保留省份
                            }
                            else
                            {
                                var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
                                if (disrs.Length >= 2)
                                {
                                    x.code = disrs[disrs.Length - 1];
                                    x.name = disrs[disrs.Length - 2];   //不保留省份
                                    //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
                                }
                                else
                                {
                                    x.code = region;
                                    x.name = region;
                                }
                            }
                        }
                        else
                        {
                            x.name = x.id;
                            x.code = x.id;
                        }
                    });
                    List<RecRegionCnt> regionCnts = new();
                    ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
                        regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cDay });
                    });
                    dayRecCnt.regionCnts = regionCnts;
                    recCnts.Add(dayRecCnt);
                    //保存存至Blob文件
                    var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(dayRecCnt.ToJsonString(), $"visitCnt/{cDay}", $"days.json");
                    urls.Add(url);
                }
                
                var azureClient = _azureStorage.GetBlobContainerClient("0-public");//获取容器连接地址
                int expireTime = int.Parse(DateTimeOffset.UtcNow.AddDays(-180).ToString("yyyyMMdd"));
                await foreach (var blobItem in azureClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix: "visitCnt"))
                {
                    string[] sub_name = blobItem.Name.Split('/');
                    if (sub_name.Length > 2)
                    {
                        if (int.Parse(sub_name[1]) <= expireTime)
                        {
                            await azureClient.GetBlobBaseClient(blobItem.Name).DeleteIfExistsAsync();
                        }
                    }
                }

            } catch (Exception ex) {
               await _dingDing.SendBotMsg($"防火墙日志统计异常：{ex.Message}\n{ex.StackTrace}", GroupNames.成都开发測試群組);
            }

            return (recCnts, urls);
        }


    }
}