using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using TEAMModelOS.SDK.Context.BI; using TEAMModelOS.SDK.DI; using TEAMModelOS.SDK.Extension; using TEAMModelOS.SDK.Models.Cosmos.BI; namespace TEAMModelOS.SDK.Models.Service.BI { public static class BILogAnalyseService { /// /// 读取全部的防火墙日志文件并分析保存至 /// /// /// /// public static async Task<(List recCnts ,List saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null) { var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina); if ($"{site}".Equals(BIConst.Global)) { blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global); } List recCnts = new(); List urls = new(); //地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE"; await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName)) { StringBuilder visits = new("["); BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name); BlobDownloadInfo download = tempBlobClient.Download(); var content = download.Content; string text; using (var streamReader = new StreamReader(content)) { while ((text = streamReader.ReadLine()) != null) { if (streamReader.EndOfStream) visits.Append($"{text.ToString()}"); else visits.Append($"{text.ToString()},"); } visits.Append("]"); streamReader.Close(); } string input = visits.ToString(); List aGInfos = input.ToObject>(); DateTimeOffset dtime = DateTimeOffset.UtcNow; string cHour = dtime.ToString("yyyyMMddHH"); string cDay = dtime.ToString("yyyyMMdd"); if (aGInfos.Count > 0) { cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First(); cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First(); } RecCnt saveCnts = new(); List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList(); List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList(); saveCnts.apiCnt = apiCnt; List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList(); saveCnts.ipCnt = ipCnt; recCnts.Add(saveCnts); ////保存存至Blob文件 var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json"); urls.Add(url); } return (recCnts, urls); } /// /// 通过路径获取日志文件并分析结果 /// /// /// /// /// public static async Task<(List recCnts, List saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage,string path, string site = null) { var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina); if ($"{site}".Equals(BIConst.Global)) { blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogGlobal); } List recCnts = new(); List urls = new(); string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE"; await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path)) { StringBuilder visits = new("["); //BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name); //BlobDownloadInfo download = tempBlobClient.Download(); BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download(); var content = download.Content; string text; using (var streamReader = new StreamReader(content)) { while ((text = streamReader.ReadLine()) != null) { if (streamReader.EndOfStream) visits.Append($"{text.ToString()}"); else visits.Append($"{text.ToString()},"); } visits.Append("]"); streamReader.Close(); } string input = visits.ToString(); List aGInfos = input.ToObject>(); DateTimeOffset dtime = DateTimeOffset.UtcNow; string cHour = dtime.ToString("yyyyMMddHH"); string cDay = dtime.ToString("yyyyMMdd"); if (aGInfos.Count > 0) { cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First(); cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First(); } RecCnt saveCnts = new(); List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList(); List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList(); saveCnts.apiCnt = apiCnt; List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList(); saveCnts.ipCnt = ipCnt; recCnts.Add(saveCnts); ////保存存至Blob文件 var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json"); urls.Add(url); } return (recCnts, urls); } } }