BILogAnalyseService.cs 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324
  1. using Azure.Storage.Blobs;
  2. using Azure.Storage.Blobs.Models;
  3. using Azure.Storage.Blobs.Specialized;
  4. using System;
  5. using System.Collections.Generic;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Text;
  9. using System.Text.RegularExpressions;
  10. using System.Threading.Tasks;
  11. using TEAMModelOS.SDK.Context.BI;
  12. using TEAMModelOS.SDK.DI;
  13. using TEAMModelOS.SDK.Extension;
  14. using TEAMModelOS.SDK.Models.Cosmos.BI;
  15. namespace TEAMModelOS.SDK.Models.Service.BI
  16. {
  17. public static class BILogAnalyseService
  18. {
  19. /// <summary>
  20. /// 读取全部的防火墙日志文件并分析保存至
  21. /// </summary>
  22. /// <param name="_azureStorage"></param>
  23. /// <param name="site"></param>
  24. /// <returns></returns>
  25. public static async Task<(List<RecCnt> recCnts ,List<string> saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null)
  26. {
  27. var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina);
  28. if ($"{site}".Equals(BIConst.Global))
  29. {
  30. blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global);
  31. }
  32. List<RecCnt> recCnts = new();
  33. List<string> urls = new();
  34. //地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
  35. string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
  36. await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
  37. {
  38. StringBuilder visits = new("[");
  39. BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
  40. BlobDownloadInfo download = tempBlobClient.Download();
  41. var content = download.Content;
  42. string text;
  43. using (var streamReader = new StreamReader(content))
  44. {
  45. while ((text = streamReader.ReadLine()) != null)
  46. {
  47. if (streamReader.EndOfStream)
  48. visits.Append($"{text.ToString()}");
  49. else
  50. visits.Append($"{text.ToString()},");
  51. }
  52. visits.Append("]");
  53. streamReader.Close();
  54. }
  55. string input = visits.ToString();
  56. List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
  57. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  58. string cHour = dtime.ToString("yyyyMMddHH");
  59. string cDay = dtime.ToString("yyyyMMdd");
  60. if (aGInfos.Count > 0)
  61. {
  62. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  63. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  64. }
  65. RecCnt saveCnts = new();
  66. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();
  67. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  68. saveCnts.apiCnt = apiCnt;
  69. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  70. saveCnts.ipCnt = ipCnt;
  71. recCnts.Add(saveCnts);
  72. ////保存存至Blob文件
  73. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
  74. urls.Add(url);
  75. }
  76. return (recCnts, urls);
  77. }
  78. /// <summary>
  79. /// 通过路径获取日志文件并分析结果
  80. /// </summary>
  81. /// <param name="_azureStorage"></param>
  82. /// <param name="path">防火墙路径</param>
  83. /// <param name="connectStr">连接字串</param>
  84. /// <returns></returns>
  85. public static async Task<(List<RecCnt> recCnts, List<string> saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage, IPSearcher _ipSearcher, DingDing _dingDing, string path, string connectName, string timeType = "Hour")
  86. {
  87. List<RecCnt> recCnts = new();
  88. List<string> urls = new();
  89. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  90. string cDay = dtime.ToString("yyyyMMdd");
  91. //天api
  92. List<RecApiCnt> dayApiCnt = new();
  93. //天ip
  94. List<RecIpCnt> dayIpCnt = new();
  95. //天
  96. List<MinuteCnt> dayCnts = new();
  97. try
  98. {
  99. var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: connectName);
  100. await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path))
  101. {
  102. StringBuilder visits = new("[");
  103. //BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
  104. //BlobDownloadInfo download = tempBlobClient.Download();
  105. BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download();
  106. var content = download.Content;
  107. string text;
  108. using (var streamReader = new StreamReader(content))
  109. {
  110. while ((text = streamReader.ReadLine()) != null)
  111. {
  112. if (streamReader.EndOfStream)
  113. visits.Append($"{text.ToString()}");
  114. else
  115. visits.Append($"{text.ToString()},");
  116. }
  117. visits.Append("]");
  118. streamReader.Close();
  119. }
  120. string input = visits.ToString();
  121. List<AGInfo> tempAinfos = input.ToObject<List<AGInfo>>();
  122. List<AGInfo> tempsert = new List<AGInfo>();
  123. List<AGInfo> aGInfos = new List<AGInfo>();
  124. tempAinfos.ForEach(item =>
  125. {
  126. string requestUri = item.properties.requestUri;
  127. var isType = StaticValue.suffixName.Where(k => requestUri.Contains(k)).ToList();
  128. if (isType.Count == 0)
  129. aGInfos.Add(item);
  130. });
  131. //foreach (var item in tempAinfos)
  132. //{
  133. // string requestUri = item.properties.requestUri;
  134. // var isType = type.Where(k => requestUri.Contains(k)).ToList();
  135. // if (isType.Count == 0)
  136. // aGInfos.Add(item);
  137. //}
  138. string cHour = dtime.ToString("yyyyMMddHH");
  139. string cHH = dtime.ToString("HH");
  140. if (aGInfos.Count > 0)
  141. {
  142. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  143. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  144. cHH = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("HH")).First();
  145. }
  146. RecCnt saveCnts = new();
  147. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname,minute = DateTimeOffset.Parse(s.time).ToString("mm")}).ToList();
  148. if (timeType.Equals("Hour"))
  149. {
  150. //小时
  151. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  152. saveCnts.apiCnt = apiCnt;
  153. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  154. saveCnts.ipCnt = ipCnt;
  155. List<MinuteCnt> minCnts = recInfo.GroupBy(a => a.minute).Select(s => new MinuteCnt { minute = s.Key, cnt = s.Count() }).ToList();
  156. saveCnts.minCnts = minCnts;
  157. var ipcounts = saveCnts.ipCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
  158. ipcounts.ForEach(async x => {
  159. string region = await _ipSearcher.SearchIpAsync(x.id);
  160. if (!string.IsNullOrWhiteSpace(region))
  161. {
  162. string[] dis = region.Split("·");
  163. if (dis.Length >= 2)
  164. {
  165. x.code = dis[dis.Length - 1];
  166. x.name = dis[dis.Length - 2]; // 不保留省份
  167. //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  168. }
  169. else
  170. {
  171. var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
  172. if (disrs.Length >= 2)
  173. {
  174. x.code = disrs[disrs.Length - 1];
  175. x.name = disrs[disrs.Length - 2]; //不保留省份
  176. //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  177. }
  178. else
  179. {
  180. x.code = region;
  181. x.name = region;
  182. }
  183. }
  184. }
  185. else
  186. {
  187. x.name = x.id;
  188. x.code = x.id;
  189. }
  190. });
  191. List<RecRegionCnt> regionCnts = new();
  192. ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
  193. regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cHour });
  194. });
  195. saveCnts.regionCnts = regionCnts;
  196. recCnts.Add(saveCnts);
  197. //保存存至Blob文件
  198. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHH}.json");
  199. urls.Add(url);
  200. }
  201. else if (timeType.Equals("Day"))
  202. {
  203. //天
  204. List<RecApiCnt> tempApiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  205. dayApiCnt.AddRange(tempApiCnt);
  206. //天
  207. List<RecIpCnt> tempIpCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  208. dayIpCnt.AddRange(tempIpCnt);
  209. dayCnts.Add(new MinuteCnt { minute = cHH, cnt = recInfo.Count });
  210. }
  211. }
  212. if (timeType.Equals("Day"))
  213. {
  214. RecCnt dayRecCnt = new();
  215. dayRecCnt.apiCnt = dayApiCnt.GroupBy(g => g.api).Select(s => new RecApiCnt { api = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), ip = s.Select(i => i.ip).FirstOrDefault() }).ToList();
  216. dayRecCnt.ipCnt = dayIpCnt.GroupBy(g => g.ip).Select(s => new RecIpCnt { ip = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), api = s.Select(i => i.api).FirstOrDefault() }).ToList();
  217. dayRecCnt.minCnts = dayCnts;
  218. var ipcounts = dayIpCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
  219. ipcounts.ForEach(async x => {
  220. string region = await _ipSearcher.SearchIpAsync(x.id);
  221. if (!string.IsNullOrWhiteSpace(region))
  222. {
  223. string[] dis = region.Split("·");
  224. if (dis.Length >= 2)
  225. {
  226. x.code = dis[dis.Length - 1];
  227. x.name = dis[dis.Length - 2]; // 不保留省份
  228. //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  229. }
  230. else
  231. {
  232. var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
  233. if (disrs.Length >= 2)
  234. {
  235. x.code = disrs[disrs.Length - 1];
  236. x.name = disrs[disrs.Length - 2]; //不保留省份
  237. //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  238. }
  239. else
  240. {
  241. x.code = region;
  242. x.name = region;
  243. }
  244. }
  245. }
  246. else
  247. {
  248. x.name = x.id;
  249. x.code = x.id;
  250. }
  251. });
  252. List<RecRegionCnt> regionCnts = new();
  253. ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
  254. regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cDay });
  255. });
  256. dayRecCnt.regionCnts = regionCnts;
  257. recCnts.Add(dayRecCnt);
  258. //保存存至Blob文件
  259. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(dayRecCnt.ToJsonString(), $"visitCnt/{cDay}", $"days.json");
  260. urls.Add(url);
  261. }
  262. var azureClient = _azureStorage.GetBlobContainerClient("0-public");//获取容器连接地址
  263. int expireTime = int.Parse(DateTimeOffset.UtcNow.AddDays(-180).ToString("yyyyMMdd"));
  264. await foreach (var blobItem in azureClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix: "visitCnt"))
  265. {
  266. string[] sub_name = blobItem.Name.Split('/');
  267. if (sub_name.Length > 2)
  268. {
  269. if (int.Parse(sub_name[1]) <= expireTime)
  270. {
  271. await azureClient.GetBlobBaseClient(blobItem.Name).DeleteIfExistsAsync();
  272. }
  273. }
  274. }
  275. } catch (Exception ex) {
  276. await _dingDing.SendBotMsg($"防火墙日志统计异常:{ex.Message}\n{ex.StackTrace}", GroupNames.成都开发測試群組);
  277. }
  278. return (recCnts, urls);
  279. }
  280. }
  281. }