AnalyseFileController.cs 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. using Azure.Storage.Blobs;
  2. using Azure.Storage.Blobs.Models;
  3. using Microsoft.AspNetCore.Hosting;
  4. using Microsoft.AspNetCore.Http;
  5. using Microsoft.AspNetCore.Mvc;
  6. using System;
  7. using System.Collections.Generic;
  8. using System.IO;
  9. using System.Linq;
  10. using System.Text;
  11. using System.Text.Json;
  12. using System.Text.RegularExpressions;
  13. using System.Threading.Tasks;
  14. using TEAMModelBI.Tool;
  15. using TEAMModelBI.Tool.CosmosBank;
  16. using TEAMModelOS.SDK.Context.BI;
  17. using TEAMModelOS.SDK.Context.Constant;
  18. using TEAMModelOS.SDK.DI;
  19. using TEAMModelOS.SDK.Extension;
  20. using TEAMModelOS.SDK.Models.Cosmos.BI;
  21. namespace TEAMModelBI.Controllers.BIBlob
  22. {
  23. [Route("analyse")]
  24. [ApiController]
  25. public class AnalyseFileController : ControllerBase
  26. {
  27. private readonly IWebHostEnvironment _environment; //读取文件流
  28. private readonly AzureStorageFactory _azureStorage;
  29. public AnalyseFileController(IWebHostEnvironment environment, AzureStorageFactory azureStorage)
  30. {
  31. _environment = environment;
  32. _azureStorage = azureStorage;
  33. }
  34. /// <summary>
  35. /// 查询一天的分析防火墙分析 已对接
  36. /// </summary>
  37. /// <param name="jsonElement"></param>
  38. /// <returns></returns>
  39. [ProducesDefaultResponseType]
  40. [HttpPost("get-dayapi")]
  41. public async Task<IActionResult> GetDayApi(JsonElement jsonElement)
  42. {
  43. jsonElement.TryGetProperty("dateTime", out JsonElement dateTime);
  44. jsonElement.TryGetProperty("site", out JsonElement site);
  45. DateTimeOffset dtOff = DateTimeOffset.UtcNow;
  46. var blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public");
  47. if ($"{site}".Equals(BIConst.Global))
  48. blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public", BIConst.Global);
  49. if (!string.IsNullOrEmpty($"{dateTime}"))
  50. dtOff = TimeHelper.GetDateTime(dateTime.GetInt64());
  51. string day = dtOff.ToString("yyyyMMdd");
  52. Dictionary<string, RecCnt> days = new();
  53. await foreach (BlobItem blobItem in blobCilent.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{day}"))
  54. {
  55. if (blobItem.Name.EndsWith(".json") && !blobItem.Name.EndsWith("/days.json"))
  56. {
  57. BlobClient tempClient = blobCilent.GetBlobClient(blobItem.Name);
  58. if (await tempClient.ExistsAsync())
  59. {
  60. using (var meomoryStream = new MemoryStream())
  61. {
  62. var response = blobCilent.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
  63. //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);
  64. RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
  65. string name = stringSuffix.MidStrEx(blobItem.Name, "/", ".");
  66. days.Add(name, recCnt);
  67. }
  68. }
  69. }
  70. }
  71. return Ok(new { state = RespondCode.Ok, days = days.ToList() });
  72. }
  73. /// <summary>
  74. /// 查询七天的数据
  75. /// </summary>
  76. /// <param name="jsonElement"></param>
  77. /// <returns></returns>
  78. [ProducesDefaultResponseType]
  79. [HttpPost("get-sevendayapi")]
  80. public async Task<IActionResult> GetSevenDayApi(JsonElement jsonElement)
  81. {
  82. jsonElement.TryGetProperty("site", out JsonElement site);
  83. var blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public");
  84. if ($"{site}".Equals(BIConst.Global))
  85. blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public", BIConst.Global);
  86. List<RecApiCnt> savenDayApis = new();
  87. List<string> day7 = TimeHelper.GetNearDay(DateTimeOffset.UtcNow, 7);
  88. foreach (var item in day7)
  89. {
  90. await foreach (BlobItem blobItem in blobCilent.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{item}"))
  91. {
  92. BlobClient tempClient = blobCilent.GetBlobClient(blobItem.Name);
  93. if (await tempClient.ExistsAsync())
  94. {
  95. using (var meomoryStream = new MemoryStream())
  96. {
  97. var response = blobCilent.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
  98. //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);
  99. RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
  100. savenDayApis.AddRange(recCnt.apiCnt);
  101. }
  102. }
  103. }
  104. }
  105. return Ok(new { state = RespondCode.Ok, savenDayApis });
  106. }
  107. /// <summary>
  108. /// 查询三十天的数据
  109. /// </summary>
  110. /// <param name="jsonElement"></param>
  111. /// <returns></returns>
  112. [ProducesDefaultResponseType]
  113. [HttpPost("get-thirtydayapi")]
  114. public async Task<IActionResult> GetThirtyDayApi(JsonElement jsonElement)
  115. {
  116. jsonElement.TryGetProperty("site", out JsonElement site);
  117. var blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public");
  118. if ($"{site}".Equals(BIConst.Global))
  119. blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public", BIConst.Global);
  120. List<RecApiCnt> thirtyDayApi = new();
  121. List<string> day30 = TimeHelper.GetNearDay(DateTimeOffset.UtcNow, 30);
  122. foreach (var item in day30)
  123. {
  124. await foreach (BlobItem blobItem in blobCilent.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{item}"))
  125. {
  126. BlobClient tempClient = blobCilent.GetBlobClient(blobItem.Name);
  127. if (await tempClient.ExistsAsync())
  128. {
  129. using (var meomoryStream = new MemoryStream())
  130. {
  131. var response = blobCilent.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
  132. //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);
  133. RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
  134. thirtyDayApi.AddRange(recCnt.apiCnt);
  135. }
  136. }
  137. }
  138. }
  139. return Ok(new { state = RespondCode.Ok, thirtyDayApi });
  140. }
  141. /// <summary>
  142. /// 本地Json
  143. /// </summary>
  144. /// <param name="jsonElement"></param>
  145. /// <returns></returns>
  146. [ProducesDefaultResponseType]
  147. [HttpPost("get-visitjson")]
  148. public async Task<IActionResult> GetVisitJson(JsonElement jsonElement)
  149. {
  150. jsonElement.TryGetProperty("path", out JsonElement _path);
  151. jsonElement.TryGetProperty("time", out JsonElement _time);
  152. var path = $"{_environment.ContentRootPath}/JsonFile/TempFile/PT1H.json";
  153. StreamReader streamReader = new StreamReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), Encoding.UTF8);
  154. List<object> objs = new();
  155. StringBuilder visits = new StringBuilder("[");
  156. string text;
  157. while ((text = streamReader.ReadLine()) != null)
  158. {
  159. if (streamReader.EndOfStream)
  160. visits.Append($"{text.ToString()}");
  161. else
  162. visits.Append($"{text.ToString()},");
  163. }
  164. visits.Append("]");
  165. streamReader.Close();
  166. string input = visits.ToString();
  167. List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
  168. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  169. string cHour = dtime.ToString("yyyyMMddHH");
  170. string cDay = dtime.ToString("yyyyMMdd");
  171. if (aGInfos.Count > 0)
  172. {
  173. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  174. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  175. }
  176. RecCnt saveCnts = new();
  177. //var ipGroup = aGInfos.GroupBy(g => g.properties.clientIp).ToDictionary(k => k.Key, k => k.Count()).ToList();
  178. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();
  179. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  180. saveCnts.apiCnt= apiCnt;
  181. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  182. saveCnts.ipCnt = ipCnt;
  183. ////保存存至Blob文件
  184. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
  185. var blob = _azureStorage.GetBlobContainerClient($"0-public");
  186. List<RecCnt> recCnts = new();
  187. await foreach (BlobItem blobItem in blob.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{cDay}"))
  188. {
  189. BlobClient blobClient = blob.GetBlobClient(blobItem.Name);
  190. if (await blobClient.ExistsAsync())
  191. {
  192. using (var meomoryStream = new MemoryStream())
  193. {
  194. var response = blob.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
  195. //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);
  196. RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
  197. recCnts.Add(recCnt);
  198. }
  199. }
  200. }
  201. return Ok(new { state = 200, recCnts, apiCon = apiCnt.Count(), apiSum = apiCnt.Select(ap => ap.count).Sum(), ipCount = ipCnt.Count, ipSum = ipCnt.Select(ip => ip.count).Sum(), ipCnt, apiCnt }); ;
  202. }
  203. /// <summary>
  204. /// 日志访问秘钥
  205. /// </summary>
  206. /// <param name="jsonElement"></param>
  207. /// <returns></returns>
  208. [ProducesDefaultResponseType]
  209. [HttpPost("get-loginfos")]
  210. public async Task<IActionResult> GetOSLog(JsonElement jsonElement)
  211. {
  212. jsonElement.TryGetProperty("site", out JsonElement site);
  213. var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog",name:BIConst.LogChina);
  214. if ($"{site}".Equals(BIConst.Global))
  215. {
  216. blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogGlobal);
  217. }
  218. //List<ReadContent> contents = new();
  219. List<RecCnt> recCnts = new();
  220. List<string> urls = new();
  221. //地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
  222. string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
  223. await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
  224. {
  225. StringBuilder visits = new("[");
  226. BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
  227. BlobDownloadInfo download = tempBlobClient.Download();
  228. var content = download.Content;
  229. string text;
  230. using (var streamReader = new StreamReader(content))
  231. {
  232. while ((text = streamReader.ReadLine()) != null)
  233. {
  234. if (streamReader.EndOfStream)
  235. visits.Append($"{text.ToString()}");
  236. else
  237. visits.Append($"{text.ToString()},");
  238. }
  239. visits.Append("]");
  240. streamReader.Close();
  241. }
  242. string input = visits.ToString();
  243. List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
  244. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  245. string cHour = dtime.ToString("yyyyMMddHH");
  246. string cDay = dtime.ToString("yyyyMMdd");
  247. if (aGInfos.Count > 0)
  248. {
  249. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  250. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  251. }
  252. RecCnt saveCnts = new();
  253. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();
  254. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  255. saveCnts.apiCnt = apiCnt;
  256. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  257. saveCnts.ipCnt = ipCnt;
  258. recCnts.Add(saveCnts);
  259. ////保存存至Blob文件
  260. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
  261. urls.Add(url);
  262. }
  263. return Ok(new { state = 200, recCnts, urls });
  264. }
  265. }
  266. }