BILogAnalyseService.cs 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. using Azure.Storage.Blobs;
  2. using Azure.Storage.Blobs.Models;
  3. using Azure.Storage.Blobs.Specialized;
  4. using System;
  5. using System.Collections.Generic;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Text;
  9. using System.Text.RegularExpressions;
  10. using System.Threading.Tasks;
  11. using TEAMModelOS.SDK.Context.BI;
  12. using TEAMModelOS.SDK.DI;
  13. using TEAMModelOS.SDK.Extension;
  14. using TEAMModelOS.SDK.Models.Cosmos.BI;
  15. namespace TEAMModelOS.SDK.Models.Service.BI
  16. {
  17. public static class BILogAnalyseService
  18. {
  19. /// <summary>
  20. /// 读取全部的防火墙日志文件并分析保存至
  21. /// </summary>
  22. /// <param name="_azureStorage"></param>
  23. /// <param name="site"></param>
  24. /// <returns></returns>
  25. public static async Task<(List<RecCnt> recCnts ,List<string> saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null)
  26. {
  27. var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina);
  28. if ($"{site}".Equals(BIConst.Global))
  29. {
  30. blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global);
  31. }
  32. List<RecCnt> recCnts = new();
  33. List<string> urls = new();
  34. //地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
  35. string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
  36. await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
  37. {
  38. StringBuilder visits = new("[");
  39. BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
  40. BlobDownloadInfo download = tempBlobClient.Download();
  41. var content = download.Content;
  42. string text;
  43. using (var streamReader = new StreamReader(content))
  44. {
  45. while ((text = streamReader.ReadLine()) != null)
  46. {
  47. if (streamReader.EndOfStream)
  48. visits.Append($"{text.ToString()}");
  49. else
  50. visits.Append($"{text.ToString()},");
  51. }
  52. visits.Append("]");
  53. streamReader.Close();
  54. }
  55. string input = visits.ToString();
  56. List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
  57. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  58. string cHour = dtime.ToString("yyyyMMddHH");
  59. string cDay = dtime.ToString("yyyyMMdd");
  60. if (aGInfos.Count > 0)
  61. {
  62. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  63. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  64. }
  65. RecCnt saveCnts = new();
  66. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost }).ToList();
  67. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  68. saveCnts.apiCnt = apiCnt;
  69. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  70. saveCnts.ipCnt = ipCnt;
  71. recCnts.Add(saveCnts);
  72. ////保存存至Blob文件
  73. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
  74. urls.Add(url);
  75. }
  76. return (recCnts, urls);
  77. }
  78. /// <summary>
  79. /// 通过路径获取日志文件并分析结果
  80. /// </summary>
  81. /// <param name="_azureStorage"></param>
  82. /// <param name="path">防火墙路径</param>
  83. /// <param name="connectStr">连接字串</param>
  84. /// <returns></returns>
  85. public static async Task<(List<RecCnt> recCnts, List<string> saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage, DingDing _dingDing, string path, string connectName, string timeType = "Hour")
  86. {
  87. List<RecCnt> recCnts = new();
  88. List<string> urls = new();
  89. TimeZoneInfo localTimezone = TimeZoneInfo.Local;
  90. var Hours = localTimezone.BaseUtcOffset.Hours;
  91. DateTimeOffset dtime = DateTimeOffset.UtcNow;
  92. if (Hours!=0)
  93. {
  94. //有时差
  95. dtime = DateTimeOffset.UtcNow.AddHours(8-Hours);
  96. }
  97. string cDay = dtime.ToString("yyyyMMdd");
  98. //天api
  99. List<RecApiCnt> dayApiCnt = new();
  100. //天ip
  101. List<RecIpCnt> dayIpCnt = new();
  102. //天
  103. List<MinuteCnt> dayCnts = new();
  104. try
  105. {
  106. var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-appservicehttplogs", name: connectName);
  107. await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path))
  108. {
  109. StringBuilder visits = new("[");
  110. //BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
  111. //BlobDownloadInfo download = tempBlobClient.Download();
  112. BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download();
  113. var content = download.Content;
  114. string text;
  115. using (var streamReader = new StreamReader(content))
  116. {
  117. while ((text = streamReader.ReadLine()) != null)
  118. {
  119. if (streamReader.EndOfStream)
  120. visits.Append($"{text.ToString()}");
  121. else
  122. visits.Append($"{text.ToString()},");
  123. }
  124. visits.Append("]");
  125. streamReader.Close();
  126. }
  127. string input = visits.ToString();
  128. List<AGInfo> tempAinfos = input.ToObject<List<AGInfo>>();
  129. List<AGInfo> tempsert = new List<AGInfo>();
  130. List<AGInfo> aGInfos = new List<AGInfo>();
  131. tempAinfos.FindAll(x=>x.properties.CsMethod.Equals("POST"))?.ForEach(item =>
  132. {
  133. string requestUri = item.properties.CsUriStem;
  134. if (!string.IsNullOrWhiteSpace(requestUri)) {
  135. var isType = StaticValue.suffixName.Where(k => requestUri.Contains(k)).ToList();
  136. if (isType.Count == 0)
  137. aGInfos.Add(item);
  138. }
  139. });
  140. //foreach (var item in tempAinfos)
  141. //{
  142. // string requestUri = item.properties.requestUri;
  143. // var isType = type.Where(k => requestUri.Contains(k)).ToList();
  144. // if (isType.Count == 0)
  145. // aGInfos.Add(item);
  146. //}
  147. string cHour = dtime.ToString("yyyyMMddHH");
  148. string cHH = dtime.ToString("HH");
  149. if (aGInfos.Count > 0)
  150. {
  151. cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
  152. cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
  153. cHH = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("HH")).First();
  154. }
  155. RecCnt saveCnts = new();
  156. List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost,minute = DateTimeOffset.Parse(s.time).ToString("mm")}).ToList();
  157. if (timeType.Equals("Hour"))
  158. {
  159. //小时
  160. List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  161. saveCnts.apiCnt = apiCnt;
  162. List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  163. saveCnts.ipCnt = ipCnt;
  164. List<MinuteCnt> minCnts = recInfo.GroupBy(a => a.minute).Select(s => new MinuteCnt { minute = s.Key, cnt = s.Count() }).ToList();
  165. saveCnts.minCnts = minCnts;
  166. var ipcounts = saveCnts.ipCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
  167. ipcounts.ForEach(async x => {
  168. //string region = await _ipSearcher.SearchIpAsync(x.id);
  169. //if (!string.IsNullOrWhiteSpace(region))
  170. //{
  171. // string[] dis = region.Split("·");
  172. // if (dis.Length >= 2)
  173. // {
  174. // x.code = dis[dis.Length - 1];
  175. // x.name = dis[dis.Length - 2]; // 不保留省份
  176. // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  177. // }
  178. // else
  179. // {
  180. // var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
  181. // if (disrs.Length >= 2)
  182. // {
  183. // x.code = disrs[disrs.Length - 1];
  184. // x.name = disrs[disrs.Length - 2]; //不保留省份
  185. // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  186. // }
  187. // else
  188. // {
  189. // x.code = region;
  190. // x.name = region;
  191. // }
  192. // }
  193. //}
  194. //else
  195. //{
  196. // x.name = x.id;
  197. // x.code = x.id;
  198. //}
  199. });
  200. List<RecRegionCnt> regionCnts = new();
  201. ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
  202. regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cHour });
  203. });
  204. saveCnts.regionCnts = regionCnts;
  205. recCnts.Add(saveCnts);
  206. //保存存至Blob文件
  207. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHH}.json");
  208. urls.Add(url);
  209. }
  210. else if (timeType.Equals("Day"))
  211. {
  212. //天
  213. List<RecApiCnt> tempApiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
  214. dayApiCnt.AddRange(tempApiCnt);
  215. //天
  216. List<RecIpCnt> tempIpCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
  217. dayIpCnt.AddRange(tempIpCnt);
  218. dayCnts.Add(new MinuteCnt { minute = cHH, cnt = recInfo.Count });
  219. }
  220. }
  221. if (timeType.Equals("Day"))
  222. {
  223. RecCnt dayRecCnt = new();
  224. dayRecCnt.apiCnt = dayApiCnt.GroupBy(g => g.api).Select(s => new RecApiCnt { api = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), ip = s.Select(i => i.ip).FirstOrDefault() }).ToList();
  225. dayRecCnt.ipCnt = dayIpCnt.GroupBy(g => g.ip).Select(s => new RecIpCnt { ip = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), api = s.Select(i => i.api).FirstOrDefault() }).ToList();
  226. dayRecCnt.minCnts = dayCnts;
  227. var ipcounts = dayIpCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
  228. ipcounts.ForEach(async x => {
  229. //string region = await _ipSearcher.SearchIpAsync(x.id);
  230. //if (!string.IsNullOrWhiteSpace(region))
  231. //{
  232. // string[] dis = region.Split("·");
  233. // if (dis.Length >= 2)
  234. // {
  235. // x.code = dis[dis.Length - 1];
  236. // x.name = dis[dis.Length - 2]; // 不保留省份
  237. // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  238. // }
  239. // else
  240. // {
  241. // var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
  242. // if (disrs.Length >= 2)
  243. // {
  244. // x.code = disrs[disrs.Length - 1];
  245. // x.name = disrs[disrs.Length - 2]; //不保留省份
  246. // //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
  247. // }
  248. // else
  249. // {
  250. // x.code = region;
  251. // x.name = region;
  252. // }
  253. // }
  254. //}
  255. //else
  256. //{
  257. // x.name = x.id;
  258. // x.code = x.id;
  259. //}
  260. });
  261. List<RecRegionCnt> regionCnts = new();
  262. ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
  263. regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cDay });
  264. });
  265. dayRecCnt.regionCnts = regionCnts;
  266. recCnts.Add(dayRecCnt);
  267. //保存存至Blob文件
  268. var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(dayRecCnt.ToJsonString(), $"visitCnt/{cDay}", $"days.json");
  269. urls.Add(url);
  270. }
  271. var azureClient = _azureStorage.GetBlobContainerClient("0-public");//获取容器连接地址
  272. int expireTime = int.Parse(DateTimeOffset.UtcNow.AddDays(-180).ToString("yyyyMMdd"));
  273. await foreach (var blobItem in azureClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix: "visitCnt"))
  274. {
  275. string[] sub_name = blobItem.Name.Split('/');
  276. if (sub_name.Length > 2)
  277. {
  278. if (int.Parse(sub_name[1]) <= expireTime)
  279. {
  280. await azureClient.GetBlobBaseClient(blobItem.Name).DeleteIfExistsAsync();
  281. }
  282. }
  283. }
  284. } catch (Exception ex) {
  285. await _dingDing.SendBotMsg($"防火墙日志统计异常:{ex.Message}\n{ex.StackTrace}", GroupNames.成都开发測試群組);
  286. }
  287. return (recCnts, urls);
  288. }
  289. }
  290. }