BILogAnalyseService.cs 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
  1. using Azure.Storage.Blobs;
  2. using Azure.Storage.Blobs.Models;
  3. using Azure.Storage.Blobs.Specialized;
  4. using System;
  5. using System.Collections.Generic;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Text;
  9. using System.Threading.Tasks;
  10. using TEAMModelOS.SDK.Context.BI;
  11. using TEAMModelOS.SDK.DI;
  12. using TEAMModelOS.SDK.Extension;
  13. using TEAMModelOS.SDK.Models.Cosmos.BI;
  14. namespace TEAMModelOS.SDK.Models.Service.BI
  15. {
  16. public static class BILogAnalyseService
  17. {
  18. /// <summary>
  19. /// 读取全部的防火墙日志文件并分析保存至
  20. /// </summary>
  21. /// <param name="_azureStorage"></param>
  22. /// <param name="site"></param>
  23. /// <returns></returns>
  24. public static async Task<(List<RecCnt> recCnts ,List<string> saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null)
  25. {
  26. var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina);
  27. if ($"{site}".Equals(BIConst.Global))
  28. {
  29. blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global);
  30. }
  31. List<RecCnt> recCnts = new();
  32. List<string> urls = new();
  33. //地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
  34. string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
  35. await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
  36. {
  37. StringBuilder visits = new("[");
  38. BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
  39. BlobDownloadInfo download = tempBlobClient.Download();
  40. var content = download.Content;
  41. string text;
  42. using (var streamReader = new StreamReader(content))
  43. {
  44. while ((text = streamReader.ReadLine()) != null)
  45. {
  46. if (streamReader.EndOfStream)
  47. visits.Append($"{text.ToString()}");
  48. else
  49. visits.Append($"{text.ToString()},");
  50. }
  51. visits.Append("]");
  52. streamReader.Close();
  53. }
  54. string input = visits.ToString();
  55. List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
  56. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  57. string cHour = dtime.ToString("yyyyMMddHH");
  58. string cDay = dtime.ToString("yyyyMMdd");
  59. if (aGInfos.Count > 0)
  60. {
  61. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  62. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  63. }
  64. RecCnt saveCnts = new();
  65. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();
  66. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  67. saveCnts.apiCnt = apiCnt;
  68. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  69. saveCnts.ipCnt = ipCnt;
  70. recCnts.Add(saveCnts);
  71. ////保存存至Blob文件
  72. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
  73. urls.Add(url);
  74. }
  75. return (recCnts, urls);
  76. }
  77. /// <summary>
  78. /// 通过路径获取日志文件并分析结果
  79. /// </summary>
  80. /// <param name="_azureStorage"></param>
  81. /// <param name="path">防火墙路径</param>
  82. /// <param name="connectStr">连接字串</param>
  83. /// <returns></returns>
  84. public static async Task<(List<RecCnt> recCnts, List<string> saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage,string path, string connectName)
  85. {
  86. List<RecCnt> recCnts = new();
  87. List<string> urls = new();
  88. try {
  89. var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: connectName);
  90. await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path))
  91. {
  92. StringBuilder visits = new("[");
  93. //BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
  94. //BlobDownloadInfo download = tempBlobClient.Download();
  95. BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download();
  96. var content = download.Content;
  97. string text;
  98. using (var streamReader = new StreamReader(content))
  99. {
  100. while ((text = streamReader.ReadLine()) != null)
  101. {
  102. if (streamReader.EndOfStream)
  103. visits.Append($"{text.ToString()}");
  104. else
  105. visits.Append($"{text.ToString()},");
  106. }
  107. visits.Append("]");
  108. streamReader.Close();
  109. }
  110. string input = visits.ToString();
  111. List<AGInfo> tempAinfos = input.ToObject<List<AGInfo>>();
  112. List<AGInfo> tempsert = new List<AGInfo>();
  113. List<AGInfo> aGInfos = new List<AGInfo>();
  114. tempAinfos.ForEach(item =>
  115. {
  116. string requestUri = item.properties.requestUri;
  117. var isType = StaticValue.suffixName.Where(k => requestUri.Contains(k)).ToList();
  118. if (isType.Count == 0)
  119. aGInfos.Add(item);
  120. });
  121. //foreach (var item in tempAinfos)
  122. //{
  123. // string requestUri = item.properties.requestUri;
  124. // var isType = type.Where(k => requestUri.Contains(k)).ToList();
  125. // if (isType.Count == 0)
  126. // aGInfos.Add(item);
  127. //}
  128. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  129. string cHour = dtime.ToString("yyyyMMddHH");
  130. string cDay = dtime.ToString("yyyyMMdd");
  131. if (aGInfos.Count > 0)
  132. {
  133. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  134. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  135. }
  136. RecCnt saveCnts = new();
  137. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname,minute = DateTimeOffset.Parse(s.time).ToString("yyyyMMddHHmm") }).ToList();
  138. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  139. saveCnts.apiCnt = apiCnt;
  140. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  141. saveCnts.ipCnt = ipCnt;
  142. List<MinuteCnt> minCnts = recInfo.GroupBy(a => a.minute).Select(s => new MinuteCnt { minute = s.Key, cnt = s.Count() }).ToList();
  143. saveCnts.minCnts = minCnts;
  144. recCnts.Add(saveCnts);
  145. //保存存至Blob文件
  146. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
  147. urls.Add(url);
  148. }
  149. var azureClient = _azureStorage.GetBlobContainerClient("0-public");//获取容器连接地址
  150. int expireTime = int.Parse(DateTimeOffset.UtcNow.AddDays(-180).ToString("yyyyMMdd"));
  151. await foreach (var blobItem in azureClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix: "visitCnt"))
  152. {
  153. string[] sub_name = blobItem.Name.Split('/');
  154. if (sub_name.Length > 2)
  155. {
  156. if (int.Parse(sub_name[1]) <= expireTime)
  157. {
  158. await azureClient.GetBlobBaseClient(blobItem.Name).DeleteIfExistsAsync();
  159. }
  160. }
  161. }
  162. } catch (Exception ex) { }
  163. return (recCnts, urls);
  164. }
  165. }
  166. }