using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using TEAMModelOS.SDK.Context.BI;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Models.Cosmos.BI;
namespace TEAMModelOS.SDK.Models.Service.BI
{
public static class BILogAnalyseService
{
///
/// 读取全部的防火墙日志文件并分析保存至
///
///
///
///
public static async Task<(List recCnts ,List saveUrls)> GetAllLogAnalyse(AzureStorageFactory _azureStorage,string site = null)
{
var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina);
if ($"{site}".Equals(BIConst.Global))
{
blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.Global);
}
List recCnts = new();
List urls = new();
//地址: y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
{
StringBuilder visits = new("[");
BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
BlobDownloadInfo download = tempBlobClient.Download();
var content = download.Content;
string text;
using (var streamReader = new StreamReader(content))
{
while ((text = streamReader.ReadLine()) != null)
{
if (streamReader.EndOfStream)
visits.Append($"{text.ToString()}");
else
visits.Append($"{text.ToString()},");
}
visits.Append("]");
streamReader.Close();
}
string input = visits.ToString();
List aGInfos = input.ToObject>();
DateTimeOffset dtime = DateTimeOffset.UtcNow;
string cHour = dtime.ToString("yyyyMMddHH");
string cDay = dtime.ToString("yyyyMMdd");
if (aGInfos.Count > 0)
{
cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
}
RecCnt saveCnts = new();
List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();
List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
saveCnts.apiCnt = apiCnt;
List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
saveCnts.ipCnt = ipCnt;
recCnts.Add(saveCnts);
////保存存至Blob文件
var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
urls.Add(url);
}
return (recCnts, urls);
}
///
/// 通过路径获取日志文件并分析结果
///
///
///
///
///
public static async Task<(List recCnts, List saveUrls)> GetPathAnalyse(AzureStorageFactory _azureStorage,string path, string site = null)
{
var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogChina);
if ($"{site}".Equals(BIConst.Global))
{
blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogGlobal);
}
List recCnts = new();
List urls = new();
string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, path))
{
StringBuilder visits = new("[");
//BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
//BlobDownloadInfo download = tempBlobClient.Download();
BlobDownloadInfo download = blobClient.GetBlobClient(blobItem.Name).Download();
var content = download.Content;
string text;
using (var streamReader = new StreamReader(content))
{
while ((text = streamReader.ReadLine()) != null)
{
if (streamReader.EndOfStream)
visits.Append($"{text.ToString()}");
else
visits.Append($"{text.ToString()},");
}
visits.Append("]");
streamReader.Close();
}
string input = visits.ToString();
List aGInfos = input.ToObject>();
DateTimeOffset dtime = DateTimeOffset.UtcNow;
string cHour = dtime.ToString("yyyyMMddHH");
string cDay = dtime.ToString("yyyyMMdd");
if (aGInfos.Count > 0)
{
cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
}
RecCnt saveCnts = new();
List recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();
List apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
saveCnts.apiCnt = apiCnt;
List ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
saveCnts.ipCnt = ipCnt;
recCnts.Add(saveCnts);
////保存存至Blob文件
var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
urls.Add(url);
}
return (recCnts, urls);
}
}
}