using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using TEAMModelOS.SDK.Context.BI;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Models.Cosmos.BI;
namespace TEAMModelOS.SDK.Models.Service.BI
{
public static class BILogAnalyseService
{
///
/// 读取全部的防火墙日志文件并分析保存至
///
///
///
///
public static async Task<(List recCnts ,List saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null)
{
var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina);
if ($"{site}".Equals(BIConst.Global))
{
blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global);
}
List recCnts = new();
List urls = new();
//地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
{
StringBuilder visits = new("[");
BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
BlobDownloadInfo download = tempBlobClient.Download();
var content = download.Content;
string text;
using (var streamReader = new StreamReader(content))
{
while ((text = streamReader.ReadLine()) != null)
{
if (streamReader.EndOfStream)
visits.Append($"{text.ToString()}");
else
visits.Append($"{text.ToString()},");
}
visits.Append("]");
streamReader.Close();
}
string input = visits.ToString();
List aGInfos = input.ToObject>();
DateTimeOffset dtime = DateTimeOffset.UtcNow;
string cHour = dtime.ToString("yyyyMMddHH");
string cDay = dtime.ToString("yyyyMMdd");
if (aGInfos.Count > 0)
{
cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
}
RecCnt saveCnts = new();
List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost }).ToList();
List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
saveCnts.apiCnt = apiCnt;
List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
saveCnts.ipCnt = ipCnt;
recCnts.Add(saveCnts);
////保存存至Blob文件
var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
urls.Add(url);
}
return (recCnts, urls);
}
///
/// 通过路径获取日志文件并分析结果
///
///
/// 防火墙路径
/// 连接字串
///
public static async Task<(List recCnts, List saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage, DingDing _dingDing, string path, string connectName, string timeType = "Hour")
{
List recCnts = new();
List urls = new();
TimeZoneInfo localTimezone = TimeZoneInfo.Local;
var Hours = localTimezone.BaseUtcOffset.Hours;
DateTimeOffset dtime = DateTimeOffset.UtcNow;
if (Hours!=0)
{
//有时差
dtime = DateTimeOffset.UtcNow.AddHours(8-Hours);
}
string cDay = dtime.ToString("yyyyMMdd");
//天api
List dayApiCnt = new();
//天ip
List dayIpCnt = new();
//天
List dayCnts = new();
try
{
var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-appservicehttplogs", name: connectName);
await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path))
{
StringBuilder visits = new("[");
//BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
//BlobDownloadInfo download = tempBlobClient.Download();
BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download();
var content = download.Content;
string text;
using (var streamReader = new StreamReader(content))
{
while ((text = streamReader.ReadLine()) != null)
{
if (streamReader.EndOfStream)
visits.Append($"{text.ToString()}");
else
visits.Append($"{text.ToString()},");
}
visits.Append("]");
streamReader.Close();
}
string input = visits.ToString();
List tempAinfos = input.ToObject>();
List tempsert = new List();
List aGInfos = new List();
tempAinfos.FindAll(x=>x.properties.CsMethod.Equals("POST"))?.ForEach(item =>
{
string requestUri = item.properties.CsUriStem;
if (!string.IsNullOrWhiteSpace(requestUri)) {
var isType = StaticValue.suffixName.Where(k => requestUri.Contains(k)).ToList();
if (isType.Count == 0)
aGInfos.Add(item);
}
});
//foreach (var item in tempAinfos)
//{
// string requestUri = item.properties.requestUri;
// var isType = type.Where(k => requestUri.Contains(k)).ToList();
// if (isType.Count == 0)
// aGInfos.Add(item);
//}
string cHour = dtime.ToString("yyyyMMddHH");
string cHH = dtime.ToString("HH");
if (aGInfos.Count > 0)
{
cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
cHH = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("HH")).First();
}
RecCnt saveCnts = new();
List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.CIp, api = s.properties.CsUriStem.Split("?").ToList().Count() > 1 ? s.properties.CsUriStem.Split("?").ToList()[0] : s.properties.CsUriStem, hostName = s.properties.CsHost,minute = DateTimeOffset.Parse(s.time).ToString("mm")}).ToList();
if (timeType.Equals("Hour"))
{
//小时
List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
saveCnts.apiCnt = apiCnt;
List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
saveCnts.ipCnt = ipCnt;
List minCnts = recInfo.GroupBy(a => a.minute).Select(s => new MinuteCnt { minute = s.Key, cnt = s.Count() }).ToList();
saveCnts.minCnts = minCnts;
var ipcounts = saveCnts.ipCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
ipcounts.ForEach(async x => {
//string region = await _ipSearcher.SearchIpAsync(x.id);
//if (!string.IsNullOrWhiteSpace(region))
//{
// string[] dis = region.Split("·");
// if (dis.Length >= 2)
// {
// x.code = dis[dis.Length - 1];
// x.name = dis[dis.Length - 2]; // 不保留省份
// //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
// }
// else
// {
// var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
// if (disrs.Length >= 2)
// {
// x.code = disrs[disrs.Length - 1];
// x.name = disrs[disrs.Length - 2]; //不保留省份
// //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
// }
// else
// {
// x.code = region;
// x.name = region;
// }
// }
//}
//else
//{
// x.name = x.id;
// x.code = x.id;
//}
});
List regionCnts = new();
ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cHour });
});
saveCnts.regionCnts = regionCnts;
recCnts.Add(saveCnts);
//保存存至Blob文件
var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHH}.json");
urls.Add(url);
}
else if (timeType.Equals("Day"))
{
//天
List tempApiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
dayApiCnt.AddRange(tempApiCnt);
//天
List tempIpCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cDay, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
dayIpCnt.AddRange(tempIpCnt);
dayCnts.Add(new MinuteCnt { minute = cHH, cnt = recInfo.Count });
}
}
if (timeType.Equals("Day"))
{
RecCnt dayRecCnt = new();
dayRecCnt.apiCnt = dayApiCnt.GroupBy(g => g.api).Select(s => new RecApiCnt { api = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), ip = s.Select(i => i.ip).FirstOrDefault() }).ToList();
dayRecCnt.ipCnt = dayIpCnt.GroupBy(g => g.ip).Select(s => new RecIpCnt { ip = s.Key, count = s.Sum(gsc => gsc.count), hour = cDay, hostName = s.Select(hn => hn.hostName).FirstOrDefault(), api = s.Select(i => i.api).FirstOrDefault() }).ToList();
dayRecCnt.minCnts = dayCnts;
var ipcounts = dayIpCnt.Select(z => new IdCodeCount { id = z.ip, count = z.count }).ToList();
ipcounts.ForEach(async x => {
//string region = await _ipSearcher.SearchIpAsync(x.id);
//if (!string.IsNullOrWhiteSpace(region))
//{
// string[] dis = region.Split("·");
// if (dis.Length >= 2)
// {
// x.code = dis[dis.Length - 1];
// x.name = dis[dis.Length - 2]; // 不保留省份
// //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
// }
// else
// {
// var disrs = Regex.Split(region.TrimStart().TrimEnd(), @"\s+");
// if (disrs.Length >= 2)
// {
// x.code = disrs[disrs.Length - 1];
// x.name = disrs[disrs.Length - 2]; //不保留省份
// //x.name = region.Substring(0, region.LastIndexOf("·")); //保留省份
// }
// else
// {
// x.code = region;
// x.name = region;
// }
// }
//}
//else
//{
// x.name = x.id;
// x.code = x.id;
//}
});
List regionCnts = new();
ipcounts.GroupBy(x => x.name).ToList().ForEach(z => {
regionCnts.Add(new RecRegionCnt { region = z.Key, count = z.ToList().Sum(y => y.count), hour = cDay });
});
dayRecCnt.regionCnts = regionCnts;
recCnts.Add(dayRecCnt);
//保存存至Blob文件
var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(dayRecCnt.ToJsonString(), $"visitCnt/{cDay}", $"days.json");
urls.Add(url);
}
var azureClient = _azureStorage.GetBlobContainerClient("0-public");//获取容器连接地址
int expireTime = int.Parse(DateTimeOffset.UtcNow.AddDays(-180).ToString("yyyyMMdd"));
await foreach (var blobItem in azureClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix: "visitCnt"))
{
string[] sub_name = blobItem.Name.Split('/');
if (sub_name.Length > 2)
{
if (int.Parse(sub_name[1]) <= expireTime)
{
await azureClient.GetBlobBaseClient(blobItem.Name).DeleteIfExistsAsync();
}
}
}
} catch (Exception ex) {
await _dingDing.SendBotMsg($"防火墙日志统计异常:{ex.Message}\n{ex.StackTrace}", GroupNames.成都开发測試群組);
}
return (recCnts, urls);
}
}
}