TEAMModelOS/TEAMModelBI/Controllers/BIBlob/AnalyseFileController.cs

using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using TEAMModelBI.Tool;
using TEAMModelBI.Tool.CosmosBank;
using TEAMModelOS.SDK.Context.BI;
using TEAMModelOS.SDK.Context.Constant;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Models.Cosmos.BI;
using TEAMModelOS.SDK.Models.Service.BI;

namespace TEAMModelBI.Controllers.BIBlob
{
    [Route("analyse")]
    [ApiController]
    public class AnalyseFileController : ControllerBase
    {
        private readonly IWebHostEnvironment _environment; //读取文件流
        private readonly AzureStorageFactory _azureStorage;

        public AnalyseFileController(IWebHostEnvironment environment, AzureStorageFactory azureStorage) 
        {
            _environment = environment;
            _azureStorage = azureStorage;
        }

        /// <summary>
        /// 查询一天的分析防火墙分析  已对接
        /// </summary>
        /// <param name="jsonElement"></param>
        /// <returns></returns>
        [ProducesDefaultResponseType]
        [HttpPost("get-dayapi")]
        public async Task<IActionResult> GetDayApi(JsonElement jsonElement)
        {
            jsonElement.TryGetProperty("dateTime", out JsonElement dateTime);
            //jsonElement.TryGetProperty("site", out JsonElement site); //分开部署，就不需要，一站多用时，取消注释

            DateTimeOffset dtOff = DateTimeOffset.UtcNow;

            var blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public");
            ////分开部署，就不需要，一站多用时，取消注释
            //if ($"{site}".Equals(BIConst.Global))
            //    blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public", BIConst.Global);

            if (!string.IsNullOrEmpty($"{dateTime}"))
                dtOff = TimeHelper.GetDateTime(dateTime.GetInt64());

            string day = dtOff.ToString("yyyyMMdd");
            Dictionary<string, RecCnt> days = new();

            await foreach (BlobItem blobItem in blobCilent.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{day}"))
            {
                if (blobItem.Name.EndsWith(".json") && !blobItem.Name.EndsWith("/days.json"))
                {
                    BlobClient tempClient = blobCilent.GetBlobClient(blobItem.Name);
                    if (await tempClient.ExistsAsync())
                    {
                        using (var meomoryStream = new MemoryStream())
                        {
                            var response = blobCilent.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
                            //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);

                            RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
                            string name = stringSuffix.MidStrEx(blobItem.Name, "/", ".");
                            days.Add(name, recCnt);
                        }
                    }
                }
            }

            return Ok(new { state = RespondCode.Ok, days = days.ToList()  });
        }


        /// <summary>
        /// 查询七天的数据
        /// </summary>
        /// <param name="jsonElement"></param>
        /// <returns></returns>
        [ProducesDefaultResponseType]
        [HttpPost("get-sevendayapi")]
        public async Task<IActionResult> GetSevenDayApi(JsonElement jsonElement)
        {

            //jsonElement.TryGetProperty("site", out JsonElement site); //分开部署，就不需要，一站多用时，取消注释
            var blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public");
            ////分开部署，就不需要，一站多用时，取消注释
            //if ($"{site}".Equals(BIConst.Global))
            //    blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public", BIConst.Global);

            List<RecApiCnt> savenDayApis = new();
            List<string> day7 = TimeHelper.GetNearDay(DateTimeOffset.UtcNow, 7);
            foreach (var item in day7)
            {
                await foreach (BlobItem blobItem in blobCilent.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{item}"))
                {
                    BlobClient tempClient = blobCilent.GetBlobClient(blobItem.Name);
                    if (await tempClient.ExistsAsync())
                    {
                        using (var meomoryStream = new MemoryStream())
                        {
                            var response = blobCilent.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
                            //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);

                            RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
                            savenDayApis.AddRange(recCnt.apiCnt);
                        }
                    }
                }
            }

            return Ok(new { state = RespondCode.Ok, savenDayApis });
        }

        /// <summary>
        /// 查询三十天的数据
        /// </summary>
        /// <param name="jsonElement"></param>
        /// <returns></returns>
        [ProducesDefaultResponseType]
        [HttpPost("get-thirtydayapi")]
        public async Task<IActionResult> GetThirtyDayApi(JsonElement jsonElement) 
        {

            //jsonElement.TryGetProperty("site", out JsonElement site);   //分开部署，就不需要，一站多用时，取消注释
            var blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public");
            ////分开部署，就不需要，一站多用时，取消注释
            //if ($"{site}".Equals(BIConst.Global))
            //    blobCilent = _azureStorage.GetBlobContainerClient(containerName: "0-public", BIConst.Global);

            List<RecApiCnt> thirtyDayApi = new();
            List<string> day30 = TimeHelper.GetNearDay(DateTimeOffset.UtcNow, 30);

            foreach (var item in day30)
            {
                await foreach (BlobItem blobItem in blobCilent.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{item}"))
                {
                    BlobClient tempClient = blobCilent.GetBlobClient(blobItem.Name);
                    if (await tempClient.ExistsAsync())
                    {
                        using (var meomoryStream = new MemoryStream())
                        {
                            var response = blobCilent.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
                            //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);

                            RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
                            thirtyDayApi.AddRange(recCnt.apiCnt);

                        }
                    }
                }
            }

            return Ok(new { state = RespondCode.Ok, thirtyDayApi });
            
        }

        /// <summary>
        /// 本地Json
        /// </summary>
        /// <param name="jsonElement"></param>
        /// <returns></returns>
        [ProducesDefaultResponseType]
        [HttpPost("get-visitjson")]
        public async Task<IActionResult> GetVisitJson(JsonElement jsonElement) 
        {
            jsonElement.TryGetProperty("path", out JsonElement _path);
            jsonElement.TryGetProperty("time", out JsonElement _time);
            var path = $"{_environment.ContentRootPath}/JsonFile/TempFile/PT1H.json";
            StreamReader streamReader = new StreamReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), Encoding.UTF8);
            List<object> objs = new(); 
            StringBuilder visits = new StringBuilder("[");
            string text;
            while ((text = streamReader.ReadLine()) != null) 
            {
                if (streamReader.EndOfStream)
                    visits.Append($"{text.ToString()}");
                else
                    visits.Append($"{text.ToString()},");                
            }
            visits.Append("]");
            streamReader.Close();
            string input = visits.ToString();
            List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
            DateTimeOffset dtime = DateTimeOffset.UtcNow;
            string cHour = dtime.ToString("yyyyMMddHH");
            string cDay = dtime.ToString("yyyyMMdd");
            if (aGInfos.Count > 0) 
            {
                cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
                cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
            }

            RecCnt saveCnts = new();
            //var ipGroup = aGInfos.GroupBy(g => g.properties.clientIp).ToDictionary(k => k.Key, k => k.Count()).ToList();

            List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();

            List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
            saveCnts.apiCnt= apiCnt;

            List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
            saveCnts.ipCnt = ipCnt;

            ////保存存至Blob文件
            var url = await  _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");
            var blob = _azureStorage.GetBlobContainerClient($"0-public");

            List<RecCnt> recCnts = new();
            await foreach (BlobItem blobItem in blob.GetBlobsAsync(BlobTraits.None, BlobStates.None, $"visitCnt/{cDay}"))
            {

                BlobClient blobClient = blob.GetBlobClient(blobItem.Name);
                if (await blobClient.ExistsAsync()) 
                {
                    using (var meomoryStream = new MemoryStream())
                    {
                        var response = blob.GetBlobClient($"{blobItem.Name}").DownloadTo(meomoryStream);
                        //var response = await blob.GetBlobClient($"{blobItem.Name}").DownloadToAsync(meomoryStream);

                        RecCnt recCnt = Encoding.UTF8.GetString(meomoryStream.ToArray()).ToString().ToObject<RecCnt>();
                        recCnts.Add(recCnt);
                    }
                }
            }
            return Ok(new { state = 200, recCnts,  apiCon = apiCnt.Count(), apiSum = apiCnt.Select(ap => ap.count).Sum(), ipCount = ipCnt.Count, ipSum = ipCnt.Select(ip => ip.count).Sum(), ipCnt, apiCnt }); ;
        }

        /// <summary>
        /// 日志访问秘钥
        /// </summary>
        /// <param name="jsonElement"></param>
        /// <returns></returns>
        [ProducesDefaultResponseType]
        [HttpPost("get-loginfos")]
        public async Task<IActionResult> GetOSLog(JsonElement jsonElement) 
        {
            //jsonElement.TryGetProperty("site", out JsonElement site); //分开部署，就不需要，一站多用时，取消注释

            var blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog",name:BIConst.LogChina);
            ////分开部署，就不需要，一站多用时，取消注释
            //if ($"{site}".Equals(BIConst.Global)) 
            //    blobClient = _azureStorage.GetBlobContainerClient($"insights-logs-applicationgatewayfirewalllog", name: BIConst.LogGlobal);
            
            //List<ReadContent> contents = new();
            List<RecCnt> recCnts = new();
            List<string> urls = new();

            //地址:    y={year}/m={month}/d={day}/h={hour}/m=00/PT1H.json
            string logName = "resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE";
            await foreach (BlobItem blobItem in blobClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, logName))
            {

                StringBuilder visits = new("[");

                BlobClient tempBlobClient = blobClient.GetBlobClient(blobItem.Name);
                BlobDownloadInfo download = tempBlobClient.Download();
                var content = download.Content;
                string text;
                using (var streamReader = new StreamReader(content))
                {
                    while ((text = streamReader.ReadLine()) != null)
                    {
                        if (streamReader.EndOfStream)
                            visits.Append($"{text.ToString()}");
                        else
                            visits.Append($"{text.ToString()},");
                    }

                    visits.Append("]");
                    streamReader.Close();
                }

                string input = visits.ToString();
                List<AGInfo> aGInfos = input.ToObject<List<AGInfo>>();
                DateTimeOffset dtime = DateTimeOffset.UtcNow;
                string cHour = dtime.ToString("yyyyMMddHH");
                string cDay = dtime.ToString("yyyyMMdd");
                if (aGInfos.Count > 0)
                {
                    cHour = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMddHH")).First();
                    cDay = aGInfos.Select(s => DateTimeOffset.Parse(s.time).ToString("yyyyMMdd")).First();
                }

                RecCnt saveCnts = new();

                List<RecAppGWInfo> recInfo = aGInfos.Select(s => new RecAppGWInfo { hour = cHour, ip = s.properties.clientIp, api = s.properties.requestUri.Split("?").ToList().Count() > 1 ? s.properties.requestUri.Split("?").ToList()[0] : s.properties.requestUri, hostName = s.properties.hostname }).ToList();

                List<RecApiCnt> apiCnt = recInfo.GroupBy(a => a.api).Select(g => new RecApiCnt { api = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), ip = g.Select(i => i.ip).Distinct().ToList() }).ToList();
                saveCnts.apiCnt = apiCnt;

                List<RecIpCnt> ipCnt = recInfo.GroupBy(a => a.ip).Select(g => new RecIpCnt { ip = g.Key, count = g.Count(), hour = cHour, hostName = g.Select(h => h.hostName).Distinct().ToList(), api = g.Select(i => i.api).Distinct().ToList() }).ToList();
                saveCnts.ipCnt = ipCnt;
                recCnts.Add(saveCnts);

                ////保存存至Blob文件
                var url = await _azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(saveCnts.ToJsonString(), $"visitCnt/{cDay}", $"{cHour}.json");

                urls.Add(url);
            }

            return Ok(new { state = 200, recCnts, urls });
        }


    }
}