using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Text.Json; using System.Threading.Tasks; using TEAMModelOS.SDK.Context.BI; using TEAMModelOS.SDK.DI; using TEAMModelOS.SDK.Extension; using TEAMModelOS.SDK.Models.Cosmos.BI; namespace TEAMModelBI.Controllers.BIHome { [Route("analyse")] [ApiController] public class AnalyseFileController : ControllerBase { private readonly IWebHostEnvironment _environment; //读取文件流 private readonly AzureStorageFactory _azureStorage; public AnalyseFileController(IWebHostEnvironment environment, AzureStorageFactory azureStorage) { _environment = environment; _azureStorage = azureStorage; } /// /// 本地Json /// /// /// [HttpPost("get-visitjson")] public async Task GetVisitJson(JsonElement jsonElement) { jsonElement.TryGetProperty("path", out JsonElement _path); jsonElement.TryGetProperty("time", out JsonElement _time); var path = $"{_environment.ContentRootPath}/JsonFile/TempFile/PT1H.json"; StreamReader streamReader = new StreamReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), Encoding.UTF8); List objs = new(); StringBuilder visits = new StringBuilder("["); string text; while ((text = streamReader.ReadLine()) != null) { if (streamReader.EndOfStream) visits.Append($"{text.ToString()}"); else visits.Append($"{text.ToString()},"); } visits.Append("]"); streamReader.Close(); string input = visits.ToString(); List aGInfos = input.ToObject>(); DateTimeOffset dtime = DateTimeOffset.UtcNow; string cHour = dtime.ToString("yyyyMMddHH"); string cDay = dtime.ToString("yyyyMMdd"); if (aGInfos.Count > 0) { cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First(); cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First(); } RecCnt saveCnts = new(); //var ipGroup = aGInfos.GroupBy(g => g.properties.clientIp).ToDictionary(k => k.Key, k => k.Count()).ToList(); List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList(); List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList(); saveCnts.apiCnt= apiCnt; List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList(); saveCnts.ipCnt = ipCnt; ////保存存至Blob文件 var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json"); var blob = _azureStorage.GetBlobContainerClient($"0-public"); List recCnts = new(); await foreach (BlobItem blobItem in blob.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{cDay}")) { BlobClient blobClient = blob.GetBlobClient(blobItem.Name); if (await blobClient.ExistsAsync()) { using (var meomoryStream = new MemoryStream()) { var response = blob.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream); //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream); var temps = meomoryStream.ToString(); var temp1 = Encoding.UTF8.GetString(meomoryStream.ToArray()); var temp = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToObject(); RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject(); recCnts.Add(recCnt); } } } return Ok(new { state = 200, recCnts, apiCon = apiCnt.Count(), apiSum = apiCnt.Select(ap => ap.count).Sum(), ipCount = ipCnt.Count, ipSum = ipCnt.Select(ip => ip.count).Sum(), ipCnt, apiCnt }); ; } /// /// 日志访问秘钥 /// /// /// [HttpPost("get-loginfos")] public async Task GetOSLog(JsonElement jsonElement) { jsonElement.TryGetProperty("site", out JsonElement site); var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog",name:BIConst.LogChina); if ($"{site}".Equals(BIConst.Global)) { blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogGlobal); } //List contents = new(); List recCnts = new(); List urls = new(); //地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE"; await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName)) { StringBuilder visits = new("["); BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name); BlobDownloadInfo download = tempBlobClient.Download(); var content = download.Content; string text; using (var streamReader = new StreamReader(content)) { while ((text = streamReader.ReadLine()) != null) { if (streamReader.EndOfStream) visits.Append($"{text.ToString()}"); else visits.Append($"{text.ToString()},"); } visits.Append("]"); streamReader.Close(); } string input = visits.ToString(); List aGInfos = input.ToObject>(); DateTimeOffset dtime = DateTimeOffset.UtcNow; string cHour = dtime.ToString("yyyyMMddHH"); string cDay = dtime.ToString("yyyyMMdd"); if (aGInfos.Count > 0) { cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First(); cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First(); } RecCnt saveCnts = new(); List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList(); List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList(); saveCnts.apiCnt = apiCnt; List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList(); saveCnts.ipCnt = ipCnt; recCnts.Add(saveCnts); ////保存存至Blob文件 var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json"); urls.Add(url); } return Ok(new { state = 200, recCnts, urls }); } } }