TEAMModelOS/TEAMModelOS.SDK/DI/AzureStorage/AzureStorageBlobExtensions.cs

using System.Threading.Tasks;
using Azure.Storage;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models; 
using System;
using System.IO;
using Azure.Storage.Blobs.Specialized;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Azure.Core;
using Azure;
using TEAMModelOS.SDK;
using TEAMModelOS.SDK.Extension;
using System.Text.Encodings.Web;
using TEAMModelOS.SDK.Models.Table;
using Microsoft.AspNetCore.Http;
using TEAMModelOS.Models;
using Microsoft.Table;

using static TEAMModelOS.SDK.CoreAPIHttpService;

namespace TEAMModelOS.SDK.DI
{
    public static class AzureStorageBlobExtensions
    {
        /// <summary>
        /// 取得指定前置詞的 Blob 名稱的總大小(Bytes)，例如指定目錄名稱為前置詞
        /// </summary>      
        /// <param name="prefix">篩選開頭名稱，Null代表容器總大小</param>
        /// <returns>總大小(Bytes)，如果為Null代表查無前置詞或者發生錯誤</returns>
        public static async Task<long?> GetBlobsSize(this BlobContainerClient client, string prefix = null)
        {
            long? size = 0;
            try
            {
                await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix))
                {

                    if (item.Name.StartsWith("res/", StringComparison.OrdinalIgnoreCase) && item.Name.EndsWith(".htex", StringComparison.OrdinalIgnoreCase))
                    { continue; }
                    if (!prefix.Equals(item.Name))
                    {
                        //避免操作(1111) /1111/1111.json  /1111111/11111.json
                        if (!prefix.EndsWith("/"))
                        {
                            if (item.Name.StartsWith(prefix + "/"))
                            {
                                size += item.Properties.ContentLength;
                            }
                        }
                    }
                    else
                    {
                        size += item.Properties.ContentLength;
                    }
                };
                return size;
            }
            catch
            {
                return size;
            }
        }

        public static async Task<List<string>> List(this BlobContainerClient client, string prefix = null) {
            try
            {
                List<string> items = new List<string>();
                await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix)) {
                    items.Add(item.Name);
                }
                return items;
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// 取得指定前置詞的 Blob 名稱的總大小(Bytes)，例如指定目錄名稱為前置詞
        /// </summary>      
        /// <param name="prefix">篩選開頭名稱，Null代表容器總大小</param>
        /// <returns>總大小(Bytes)，如果為Null代表查無前置詞或者發生錯誤</returns>
        public static async Task<(long?, Dictionary<string, double?>)> GetBlobsCatalogSize(this BlobContainerClient client, string prefix = null)
        {
            long? size = 0;
            Dictionary<string, double?> dict = new Dictionary<string, double?>();
            try
            {
                List<KeyValuePair<string, double?>> foderSize = new List<KeyValuePair<string, double?>>();
                await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix))
                {
                    var len = item.Properties.ContentLength;
                    foderSize.Add(new KeyValuePair<string, double?>(item.Name.Split("/")[0], len));
                    size += item.Properties.ContentLength;
                };
                foderSize.Select(x => new { x.Key, x.Value }).GroupBy(y=>y.Key).ToList().ForEach(g=> {
                    var gpsize = g.Select(m => m.Value).Sum();
                    dict[g.Key] = gpsize;
                });
                return (size, dict);
            }
            catch
            {
                return (size, dict);
            }
        }

        public class OptUrl
        {
            public string url { get; set; }
            public long size { get; set; }
        }
        /// <summary>
        /// 取得指定前置詞的 Blob 名稱的總大小(Bytes)，例如指定目錄名稱為前置詞
        /// </summary>      
        /// <param name="urls">多个文件的连接/param>
        /// <returns>總大小(Bytes)，如果為Null代表查無前置詞或者發生錯誤</returns>
        public static async Task<List<OptUrl>> GetBlobsSize(this BlobContainerClient client, List<string> urls)
        {
            List<OptUrl> optUrls = new List<OptUrl>();
            try
            {
                if (urls != null) {
                    foreach (var url in urls)
                    {
                        OptUrl optUrl = new OptUrl { url = url, size = 0 };
                        var eurl = System.Web.HttpUtility.UrlDecode(url, Encoding.UTF8);
                        var blob = client.GetBlobClient(eurl);
                        if (blob.Exists())
                        {
                            var props = await blob.GetPropertiesAsync();
                            var size = props.Value.ContentLength;
                            optUrl.size = size;
                        }
                        optUrls.Add(optUrl);
                    }
                }
                
                return optUrls;
            }
            catch
            {
                return optUrls;
            }
        }
        /// <summary>        
        ///prefixs多个文件或文件夹路径删除
        ///prefixs 或者按前缀文件夹删除
        /// 
        /// </summary>      
        /// <param name="prefix">篩選開頭名稱，Null代表容器</param>        
        public static async Task<bool> DeleteBlobs(this BlobServiceClient client,DingDing _dingDing, string blobContainerName, List<string> prefixs )
        {
            if (!prefixs.IsNotEmpty()) return false;
            try
            {
                BlobContainerClient bcc = client.GetBlobContainerClient(blobContainerName.ToLower());
                BlobBatchClient bbc = client.GetBlobBatchClient();
                List<Uri> blobs = new List<Uri>();
                List<Task<ResponseMessage<bool>>> list = new List<Task<Response<bool>>>();
                foreach (var prefix in prefixs) {
                    string px = prefix;
                    if (prefix.StartsWith("/")) {
                        px= prefix.Substring(1);
                    }
                    //目录必须有两层以上，避免删除根目录所有的。
                    var pxlen= px.Split("/");
                    if (pxlen.Length >=2) {
                        var items = bcc.GetBlobsAsync(BlobTraits.None, BlobStates.None, px);
                        await foreach (var item in items)
                        {
                            var urib = new UriBuilder(bcc.Uri);

                            if (!prefix.Equals(item.Name))
                            {
                                //避免操作(1111) /1111/1111.json  /1111111/11111.json
                                if (!prefix.EndsWith("/"))
                                {
                                    if (item.Name.StartsWith(prefix + "/"))
                                    {
                                        string path = $"{urib.Uri.AbsoluteUri}/{item.Name}";
                                        list.Add(bcc.GetBlobClient(item.Name).DeleteIfExistsAsync());
                                        if (item.Name.StartsWith("res/") && item.Name.EndsWith("/index.json"))
                                        {
                                            list.Add(bcc.GetBlobClient($"{prefix}.htex").DeleteIfExistsAsync());
                                            list.Add(bcc.GetBlobClient($"{prefix}.HTEX").DeleteIfExistsAsync());
                                        }
                                        blobs.Add(new Uri(path));
                                    }
                                }
                            }
                            else
                            {
                                string path = $"{urib.Uri.AbsoluteUri}/{item.Name}";
                                list.Add(bcc.GetBlobClient(item.Name).DeleteIfExistsAsync());
                                blobs.Add(new Uri(path));
                            }
                        };
                    }
                    
                }
                if (list.Count > 0) {
                    if (list.Count <= 256)
                    {
                        await Task.WhenAll(list);
                    }
                    else
                    {
                        int pages = (list.Count + 255) / 256; //256是批量操作最大值，pages = (total + max -1) / max;
                        for (int i = 0; i < pages; i++)
                        {
                            List<Task<ResponseMessage<bool>>> lists = list.Skip((i) * 256).Take(256).ToList();
                            await Task.WhenAll(lists);
                        }
                    }
                }
                return true;
                /*
                if (blobs.Count <= 256)
                {
                    if (blobs.Count > 0) {
                        try {
                            ResponseMessage[] ass = await bbc.DeleteBlobsAsync(blobs); 
                        }
                        catch (AggregateException ex)
                        {
                            //删除多个时会，如果其中一个文件不存在 或者其中一个删除失败会引发该异常
                            return true;
                        }
                        catch (RequestFailedException ex)
                        {
                            await _dingDing.SendBotMsg($"/{ex.Message}\n{ex.StackTrace}", GroupNames.醍摩豆服務運維群組);
                            return true;
                        }
                        catch (Exception ex)
                        {
                            return true;
                        }
                    }
                    return true;
                }
                else
                {
                    int pages = (blobs.Count + 255) / 256; //256是批量操作最大值，pages = (total + max -1) / max;
                    for (int i = 0; i < pages; i++)
                    {
                        List<Uri> lists = blobs.Skip((i) * 256).Take(256).ToList();
                        try { ResponseMessage[] ass = await bbc.DeleteBlobsAsync(lists); }
                        catch (AggregateException ex)
                        {
                            //删除多个时会，如果其中一个文件不存在 或者其中一个删除失败会引发该异常
                            return true;
                        }
                        catch (RequestFailedException ex)
                        {
                            return true;
                        }
                        catch (Exception ex)
                        {
                            return true;
                        }
                    }
                    return true;
                }
                */
            }
            catch(Exception ex )
            {
                await _dingDing.SendBotMsg($"文件删除异常{ex.Message}\n{ex.StackTrace}{prefixs.ToJsonString()}", GroupNames.醍摩豆服務運維群組);
                return false;
            }
        }

        /// <summary>
        /// 保存操作记录
        /// </summary>
        /// <param name="azureStorage"></param>
        /// <param name="type"></param>
        /// <param name="msg"></param>
        /// <param name="dingDing"></param>
        /// <param name="scope"></param>
        /// <param name="bizId"></param>
        /// <param name="option"></param>
        /// <param name="httpContext"></param>
        /// <returns></returns>
        public static async Task SaveLog(this AzureStorageFactory azureStorage, string type, string msg,DingDing dingDing, string scope = null, string bizId = null, Option option = null, HttpContext httpContext = null)
        {
            var table = azureStorage.GetCloudTableClient().GetTableReference("IESOptLog");
            OptLog log = new() { RowKey = Guid.NewGuid().ToString() };
            try
            {
                object id = null, school = null, name = null, website = null ;
                httpContext?.Items.TryGetValue("ID", out id);
                httpContext?.Items.TryGetValue("School", out school);
                httpContext?.Items.TryGetValue("Name", out name);
                httpContext?.Items.TryGetValue("Website", out website);
                log.tmdId = id != null ? $"{id}" : log.tmdId;
                log.name = name != null ? $"{name}" : log.name;
                string host = httpContext?.Request?.Host.Value;
                log.school = school != null ? $"{school}" : log.school;
                log.PartitionKey = type != null ? $"Log-{type}" : "Log-Default";
                log.RowKey = bizId != null ? bizId : log.RowKey;
                log.platform = website!=null? $"{website}" : "Default";
                log.msg = msg;
                log.type = type;
                log.scope = scope;
                host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default";
                log.url =$"{host}{httpContext?.Request.Path}" ;
                if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 256)
                {
                    log.saveMod = 1;
                    log.jsonfile = $"/0-public/optlog/{log.RowKey}-{log.PartitionKey}.json";
                    await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(log.ToJsonString(), "optlog", $"{log.RowKey}-{log.PartitionKey}.json");
                    log.msg = null;
                    await table.SaveOrUpdate<OptLog>(log);
                }
                else {
                    await table.SaveOrUpdate<OptLog>(log);
                }
            }
            catch (Exception ex)
            {
                _ = dingDing.SendBotMsg($"日志保存失败：{ex.Message}\n{ex.StackTrace},,{log.ToJsonString()}", GroupNames.醍摩豆服務運維群組);
            }
        }

        /// <summary>
        /// 保存日志文件
        /// </summary>
        /// <param name="azureStorage"></param>
        /// <param name="type"></param>
        /// <param name="rootPath"></param>
        /// <param name="replaceData"></param>
        /// <param name="dingDing"></param>
        /// <param name="scope"></param>
        /// <param name="bizId"></param>
        /// <param name="option"></param>
        /// <param name="httpContext"></param>
        /// <returns></returns>
        public static async Task SaveLogLang(this AzureStorageFactory azureStorage, string type,string rootPath,  Dictionary<string, object> replaceData, DingDing dingDing, string scope = null, string bizId = null, Option option = null, HttpContext httpContext = null)
        {
            string msg = FileWay.FileValue(rootPath, "zh-cn", type, replaceData);
            var table = azureStorage.GetCloudTableClient().GetTableReference("IESOptLog");
            OptLog log = new() { RowKey = Guid.NewGuid().ToString() };
            try
            {
                object id = null, school = null, name = null, website = null;
                httpContext?.Items.TryGetValue("ID", out id);
                httpContext?.Items.TryGetValue("School", out school);
                httpContext?.Items.TryGetValue("Name", out name);
                httpContext?.Items.TryGetValue("Website", out website);
                log.tmdId = id != null ? $"{id}" : log.tmdId;
                log.name = name != null ? $"{name}" : log.name;
                string host = httpContext?.Request?.Host.Value;
                log.school = school != null ? $"{school}" : log.school;
                log.PartitionKey = type != null ? $"Log-{type}" : "Log-Default";
                log.RowKey = bizId != null ? bizId : log.RowKey;
                log.platform = website != null ? $"{website}" : "Default";
                log.msg = msg;
                log.tmsg = FileWay.FileValue(rootPath, "zh-tw", type, replaceData);
                log.emsg = FileWay.FileValue(rootPath, "en-us", type, replaceData);
                log.type = type;
                log.scope = scope;
                host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default";
                log.url = $"{host}{httpContext?.Request.Path}";
                if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 256)
                {
                    log.saveMod = 1;
                    log.jsonfile = $"/0-public/optlog/{log.RowKey}-{log.PartitionKey}.json";
                    await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(log.ToJsonString(), "optlog", $"{log.RowKey}-{log.PartitionKey}.json");
                    log.msg = null;
                    await table.SaveOrUpdate<OptLog>(log);
                }
                else
                {
                    await table.SaveOrUpdate<OptLog>(log);
                }
            }
            catch (Exception ex)
            {
                _ = dingDing.SendBotMsg($"日志保存失败：{ex.Message}\n{ex.StackTrace},,{log.ToJsonString()}", GroupNames.醍摩豆服務運維群組);
            }
        }
        /// <summary>
        /// 系统管理员 资源，题目关联，htex关联，学习活动学生上传文件关联，基本信息关联，教室平面图关联，评测冷数据关联
        /// "system": [ "res", "item", "htex", "task", "info", "room", "exam" ],
        /// 资源，题目关联，htex关联，学习活动学生上传文件关联，基本信息关联，教室平面图关联，评测冷数据关联
        /// "school": [ "res", "item", "htex", "task", "info", "room", "exam" ],
        /// 资源，题目关联，htex关联，学习活动关联，教师基本信息关联
        /// "teacher": [ "res", "item", "htex", "task", "info" ],
        /// 答案及学习活动上传的文件，学生基本信息关联
        ///"student": [ "stu/{studentId}/ans", "stu/{studentId}/task" ]
        /// </summary>
        /// <param name="name">容器名称</param>
        /// <param name="json">文件内容的流</param>
        /// <param name="folder">业务文件夹</param>
        /// <param name="fileName">文件名</param>
        /// <param name="contentTypeDefault">是否存放文件后缀对应的contentType</param>
        /// <returns></returns>

        public static async Task<string> UploadFileByContainer(this BlobContainerClient blobContainer,   string json, string root, string blobpath, bool contentTypeDefault = true)
        {
            // string groupName =folder;
            //BlobContainerClient blobContainer = azureStorage.GetBlobContainerClient(name.ToLower().Replace("#", "")); //blobClient.GetContainerReference(groupName);            
            var blockBlob = blobContainer.GetBlobClient($"{root}/{blobpath}");
            string content_type = "application/octet-stream";
            if (!contentTypeDefault)
            {
                string fileext = blobpath.Substring(blobpath.LastIndexOf(".") > 0 ? blobpath.LastIndexOf(".") : 0);
                ContentTypeDict.dict.TryGetValue(fileext, out string contenttype);
                if (!string.IsNullOrEmpty(contenttype))
                {
                    content_type = contenttype;
                }
            }
            byte[] bytes = System.Text.Encoding.Default.GetBytes(json);
            Stream streamBlob = new MemoryStream(bytes);
            await blockBlob.UploadAsync(streamBlob, true);
            blockBlob.SetHttpHeaders(new BlobHttpHeaders { ContentType = content_type });
            return blockBlob.Uri.ToString();
        }

        /// <summary>
        /// 系统管理员 资源，题目关联，htex关联，学习活动学生上传文件关联，基本信息关联，教室平面图关联，评测冷数据关联
        /// "system": [ "res", "item", "htex", "task", "info", "room", "exam" ],
        /// 资源，题目关联，htex关联，学习活动学生上传文件关联，基本信息关联，教室平面图关联，评测冷数据关联
        /// "school": [ "res", "item", "htex", "task", "info", "room", "exam" ],
        /// 资源，题目关联，htex关联，学习活动关联，教师基本信息关联
        /// "teacher": [ "res", "item", "htex", "task", "info" ],
        /// 答案及学习活动上传的文件，学生基本信息关联
        ///"student": [ "stu/{studentId}/ans", "stu/{studentId}/task" ]
        /// </summary>
        /// <param name="name">容器名称</param>
        /// <param name="stream">文件内容的流</param>
        /// <param name="folder">业务文件夹</param>
        /// <param name="fileName">文件名</param>
        /// <param name="contentTypeDefault">是否存放文件后缀对应的contentType</param>
        /// <returns></returns>
        public static async Task<string> UploadFileByContainer(this BlobContainerClient blobContainer, Stream stream, string root, string blobpath, bool contentTypeDefault = true)
        {
            //BlobContainerClient blobContainer = azureStorage.GetBlobContainerClient(name.ToLower().Replace("#", "")); //blobClient.GetContainerReference(groupName); 
            Uri url = blobContainer.Uri;
            var blockBlob = blobContainer.GetBlobClient($"{root}/{blobpath}");
            string content_type = "application/octet-stream";
            if (!contentTypeDefault)
            {
                string fileext = blobpath.Substring(blobpath.LastIndexOf(".") > 0 ? blobpath.LastIndexOf(".") : 0);
                ContentTypeDict.dict.TryGetValue(fileext, out string contenttype);
                if (!string.IsNullOrEmpty(contenttype))
                {
                    content_type = contenttype;
                }
            }
            await blockBlob.UploadAsync(stream, true);
            blockBlob.SetHttpHeaders(new BlobHttpHeaders { ContentType = content_type });
            return blockBlob.Uri.ToString();
        }

        
        /// <summary>
        /// BI保存操作记录
        /// </summary>
        /// <param name="azureStorage"></param>
        /// <param name="type"></param>
        /// <param name="msg"></param>
        /// <param name="dingDing"></param>
        /// <param name="scope"></param>
        /// <param name="option"></param>
        /// <param name="httpContext"></param>
        /// <returns></returns>
        public static async Task SaveBILog(BlobContainerClient blobContainer, CloudTableClient tableClient, string type, string msg, DingDing dingDing, string tid = null, string tname = null, string twebsite = null, string scope = null, Option option = null, HttpContext httpContext = null)
        {
            var table = tableClient.GetTableReference("BIOptLog");
            BIOptLog biLog = new() { RowKey = Guid.NewGuid().ToString() };
            try
            {
                object id = null, name = null, ddid = null, ddname = null, website = null;
                httpContext?.Items.TryGetValue("ID", out id);
                httpContext?.Items.TryGetValue("Name", out name);
                httpContext?.Items.TryGetValue("DDId", out ddid);
                httpContext?.Items.TryGetValue("DDName", out ddname);
                httpContext?.Items.TryGetValue("Website", out website);
                string site = twebsite != null ? twebsite : $"{website}";
                biLog.tmdId = id != null ? $"{id}" : tid;
                biLog.name = name != null ? $"{name}" : tname;
                biLog.PartitionKey = type != null ? $"{site}-Log-{type}" : $"{site}-Log-Default";
                biLog.platform = site != null ? site : "Default";
                biLog.msg = msg;
                biLog.type = type;
                biLog.scope = scope;
                string host = httpContext?.Request?.Host.Value;
                host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default";
                biLog.url = $"{host}{httpContext?.Request.Path}";
                if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 255)
                {
                    biLog.saveMod = 1;
                    biLog.jsonfile = $"/0-public/BIOptLog/{biLog.PartitionKey}-{biLog.RowKey}.json";
                    await UploadFileByContainer(blobContainer, biLog.ToJsonString(), "BIOptLog", $"{biLog.PartitionKey}-{biLog.RowKey}.json");
                    biLog.msg = null;
                    await table.SaveOrUpdate<BIOptLog>(biLog);
                }
                else await table.SaveOrUpdate<BIOptLog>(biLog);
            }
            catch (Exception ex)
            {
                _ = dingDing.SendBotMsg($"BI日志保存失败：{ex.Message}\n{ex.StackTrace},,{biLog.ToJsonString()}", GroupNames.成都开发測試群組);
            }
        }
    }
}