using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using TEAMModelOS.SDK.Context.BI; using TEAMModelOS.SDK.DI; using TEAMModelOS.SDK.Extension; using TEAMModelOS.SDK.Models.Cosmos.BI; namespace TEAMModelOS.SDK.Models.Service.BI { public static class BILogAnalyseService { /// /// 读取全部的防火墙日志文件并分析保存至 /// /// /// /// public static async Task<(List recCnts ,List saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null) { var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina); if ($"{site}".Equals(BIConst.Global)) { blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global); } List recCnts = new(); List urls = new(); //地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE"; await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName)) { StringBuilder visits = new("["); BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name); BlobDownloadInfo download = tempBlobClient.Download(); var content = download.Content; string text; using (var streamReader = new StreamReader(content)) { while ((text = streamReader.ReadLine()) != null) { if (streamReader.EndOfStream) visits.Append($"{text.ToString()}"); else visits.Append($"{text.ToString()},"); } visits.Append("]"); streamReader.Close(); } string input = visits.ToString(); List aGInfos = input.ToObject>(); DateTimeOffset dtime = DateTimeOffset.UtcNow; string cHour = dtime.ToString("yyyyMMddHH"); string cDay = dtime.ToString("yyyyMMdd"); if (aGInfos.Count > 0) { cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First(); cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First(); } RecCnt saveCnts = new(); List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost }).ToList(); List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList(); saveCnts.apiCnt = apiCnt; List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList(); saveCnts.ipCnt = ipCnt; recCnts.Add(saveCnts); ////保存存至Blob文件 var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json"); urls.Add(url); } return (recCnts, urls); } /// /// 通过路径获取日志文件并分析结果 /// /// /// 防火墙路径 /// 连接字串 /// public static async Task<(List recCnts, List saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage, DingDing _dingDing, string path, string connectName, string timeType = "Hour") { List recCnts = new(); List urls = new(); TimeZoneInfo localTimezone = TimeZoneInfo.Local; var Hours = localTimezone.BaseUtcOffset.Hours; DateTimeOffset dtime = DateTimeOffset.UtcNow; if (Hours!=0) { //有时差 dtime = DateTimeOffset.UtcNow.AddHours(8-Hours); } string cDay = dtime.ToString("yyyyMMdd"); //天api List dayApiCnt = new(); //天ip List dayIpCnt = new(); //天 List dayCnts = new(); try { var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-appservicehttplogs", name: connectName); await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path)) { StringBuilder visits = new("["); //BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name); //BlobDownloadInfo download = tempBlobClient.Download(); BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download(); var content = download.Content; string text; using (var streamReader = new StreamReader(content)) { while ((text = streamReader.ReadLine()) != null) { if (streamReader.EndOfStream) visits.Append($"{text.ToString()}"); else visits.Append($"{text.ToString()},"); } visits.Append("]"); streamReader.Close(); } string input = visits.ToString(); List tempAinfos = input.ToObject>(); List tempsert = new List(); List aGInfos = new List(); tempAinfos.FindAll(x=>x.properties.CsMethod.Equals("POST"))?.ForEach(item => { string requestUri = item.properties.CsUriStem; if (!string.IsNullOrWhiteSpace(requestUri)) { var isType = StaticValue.suffixName.Where(k => requestUri.Contains(k)).ToList(); if (isType.Count == 0) aGInfos.Add(item); } }); //foreach (var item in tempAinfos) //{ // string requestUri = item.properties.requestUri; // var isType = type.Where(k => requestUri.Contains(k)).ToList(); // if (isType.Count == 0) // aGInfos.Add(item); //} string cHour = dtime.ToString("yyyyMMddHH"); string cHH = dtime.ToString("HH"); if (aGInfos.Count > 0) { cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First(); cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First(); cHH = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("HH")).First(); } RecCnt saveCnts = new(); List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost,minute = DateTimeOffset.Parse(s.time).ToString("mm")}).ToList(); if (timeType.Equals("Hour")) { //小时 List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList(); saveCnts.apiCnt = apiCnt; List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList(); saveCnts.ipCnt = ipCnt; List minCnts = recInfo.GroupBy(a => a.minute).Select(s => new MinuteCnt { minute = s.Key, cnt = s.Count() }).ToList(); saveCnts.minCnts = minCnts; var ipcounts = saveCnts.ipCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList(); ipcounts.ForEach(async x => { //string region = await _ipSearcher.SearchIpAsync(x.id); //if (!string.IsNullOrWhiteSpace(region)) //{ // string[] dis = region.Split("·"); // if (dis.Length >= 2) // { // x.code = dis[dis.Length - 1]; // x.name = dis[dis.Length - 2]; // 不保留省份 // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份 // } // else // { // var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+"); // if (disrs.Length >= 2) // { // x.code = disrs[disrs.Length - 1]; // x.name = disrs[disrs.Length - 2]; //不保留省份 // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份 // } // else // { // x.code = region; // x.name = region; // } // } //} //else //{ // x.name = x.id; // x.code = x.id; //} }); List regionCnts = new(); ipcounts.GroupBy(x => x.name).ToList().ForEach(z => { regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cHour }); }); saveCnts.regionCnts = regionCnts; recCnts.Add(saveCnts); //保存存至Blob文件 var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHH}.json"); urls.Add(url); } else if (timeType.Equals("Day")) { //天 List tempApiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList(); dayApiCnt.AddRange(tempApiCnt); //天 List tempIpCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList(); dayIpCnt.AddRange(tempIpCnt); dayCnts.Add(new MinuteCnt { minute = cHH, cnt = recInfo.Count }); } } if (timeType.Equals("Day")) { RecCnt dayRecCnt = new(); dayRecCnt.apiCnt = dayApiCnt.GroupBy(g => g.api).Select(s => new RecApiCnt { api = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), ip = s.Select(i => i.ip).FirstOrDefault() }).ToList(); dayRecCnt.ipCnt = dayIpCnt.GroupBy(g => g.ip).Select(s => new RecIpCnt { ip = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), api = s.Select(i => i.api).FirstOrDefault() }).ToList(); dayRecCnt.minCnts = dayCnts; var ipcounts = dayIpCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList(); ipcounts.ForEach(async x => { //string region = await _ipSearcher.SearchIpAsync(x.id); //if (!string.IsNullOrWhiteSpace(region)) //{ // string[] dis = region.Split("·"); // if (dis.Length >= 2) // { // x.code = dis[dis.Length - 1]; // x.name = dis[dis.Length - 2]; // 不保留省份 // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份 // } // else // { // var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+"); // if (disrs.Length >= 2) // { // x.code = disrs[disrs.Length - 1]; // x.name = disrs[disrs.Length - 2]; //不保留省份 // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份 // } // else // { // x.code = region; // x.name = region; // } // } //} //else //{ // x.name = x.id; // x.code = x.id; //} }); List regionCnts = new(); ipcounts.GroupBy(x => x.name).ToList().ForEach(z => { regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cDay }); }); dayRecCnt.regionCnts = regionCnts; recCnts.Add(dayRecCnt); //保存存至Blob文件 var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(dayRecCnt.ToJsonString(), $"visitCnt/{cDay}", $"days.json"); urls.Add(url); } var azureClient = _azureStorage.GetBlobContainerClient("0-public");//获取容器连接地址 int expireTime = int.Parse(DateTimeOffset.UtcNow.AddDays(-180).ToString("yyyyMMdd")); await foreach (var blobItem in azureClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix: "visitCnt")) { string[] sub_name = blobItem.Name.Split('/'); if (sub_name.Length > 2) { if (int.Parse(sub_name[1]) <= expireTime) { await azureClient.GetBlobBaseClient(blobItem.Name).DeleteIfExistsAsync(); } } } } catch (Exception ex) { await _dingDing.SendBotMsg($"防火墙日志统计异常:{ex.Message}\n{ex.StackTrace}", GroupNames.成都开发測試群組); } return (recCnts, urls); } } }