using System.Threading.Tasks; using Azure.Storage; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using System; using System.IO; using Azure.Storage.Blobs.Specialized; using System.Collections.Generic; using System.Linq; using System.Text; using Azure.Core; using Azure; using TEAMModelOS.SDK; using TEAMModelOS.SDK.Extension; using System.Text.Encodings.Web; using TEAMModelOS.SDK.Models.Table; using Microsoft.AspNetCore.Http; using TEAMModelOS.Models; using Microsoft.Table; using static TEAMModelOS.SDK.CoreAPIHttpService; namespace TEAMModelOS.SDK.DI { public static class AzureStorageBlobExtensions { /// /// 取得指定前置詞的 Blob 名稱的總大小(Bytes),例如指定目錄名稱為前置詞 /// /// 篩選開頭名稱,Null代表容器總大小 /// 總大小(Bytes),如果為Null代表查無前置詞或者發生錯誤 public static async Task GetBlobsSize(this BlobContainerClient client, string prefix = null) { long? size = 0; try { await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix)) { if (item.Name.StartsWith("res/", StringComparison.OrdinalIgnoreCase) && item.Name.EndsWith(".htex", StringComparison.OrdinalIgnoreCase)) { continue; } if (!prefix.Equals(item.Name)) { //避免操作(1111) /1111/1111.json /1111111/11111.json if (!prefix.EndsWith("/")) { if (item.Name.StartsWith(prefix + "/")) { size += item.Properties.ContentLength; } } } else { size += item.Properties.ContentLength; } }; return size; } catch { return size; } } public static async Task> List(this BlobContainerClient client, string prefix = null) { try { List items = new List(); await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix)) { items.Add(item.Name); } return items; } catch { return null; } } /// /// 取得指定前置詞的 Blob 名稱的總大小(Bytes),例如指定目錄名稱為前置詞 /// /// 篩選開頭名稱,Null代表容器總大小 /// 總大小(Bytes),如果為Null代表查無前置詞或者發生錯誤 public static async Task<(long?, Dictionary)> GetBlobsCatalogSize(this BlobContainerClient client, string prefix = null) { long? size = 0; Dictionary dict = new Dictionary(); try { List> foderSize = new List>(); await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix)) { var len = item.Properties.ContentLength; foderSize.Add(new KeyValuePair(item.Name.Split("/")[0], len)); size += item.Properties.ContentLength; }; foderSize.Select(x => new { x.Key, x.Value }).GroupBy(y=>y.Key).ToList().ForEach(g=> { var gpsize = g.Select(m => m.Value).Sum(); dict[g.Key] = gpsize; }); return (size, dict); } catch { return (size, dict); } } public class OptUrl { public string url { get; set; } public long size { get; set; } } /// /// 取得指定前置詞的 Blob 名稱的總大小(Bytes),例如指定目錄名稱為前置詞 /// /// 多个文件的连接/param> /// 總大小(Bytes),如果為Null代表查無前置詞或者發生錯誤 public static async Task> GetBlobsSize(this BlobContainerClient client, List urls) { List optUrls = new List(); try { if (urls != null) { foreach (var url in urls) { OptUrl optUrl = new OptUrl { url = url, size = 0 }; var eurl = System.Web.HttpUtility.UrlDecode(url, Encoding.UTF8); var blob = client.GetBlobClient(eurl); if (blob.Exists()) { var props = await blob.GetPropertiesAsync(); var size = props.Value.ContentLength; optUrl.size = size; } optUrls.Add(optUrl); } } return optUrls; } catch { return optUrls; } } /// ///prefixs多个文件或文件夹路径删除 ///prefixs 或者按前缀文件夹删除 /// /// /// 篩選開頭名稱,Null代表容器 public static async Task DeleteBlobs(this BlobServiceClient client,DingDing _dingDing, string blobContainerName, List prefixs ) { if (!prefixs.IsNotEmpty()) return false; try { BlobContainerClient bcc = client.GetBlobContainerClient(blobContainerName.ToLower()); BlobBatchClient bbc = client.GetBlobBatchClient(); List blobs = new List(); List>> list = new List>>(); foreach (var prefix in prefixs) { string px = prefix; if (prefix.StartsWith("/")) { px= prefix.Substring(1); } //目录必须有两层以上,避免删除根目录所有的。 var pxlen= px.Split("/"); if (pxlen.Length >=2) { var items = bcc.GetBlobsAsync(BlobTraits.None, BlobStates.None, px); await foreach (var item in items) { var urib = new UriBuilder(bcc.Uri); if (!prefix.Equals(item.Name)) { //避免操作(1111) /1111/1111.json /1111111/11111.json if (!prefix.EndsWith("/")) { if (item.Name.StartsWith(prefix + "/")) { string path = $"{urib.Uri.AbsoluteUri}/{item.Name}"; list.Add(bcc.GetBlobClient(item.Name).DeleteIfExistsAsync()); if (item.Name.StartsWith("res/") && item.Name.EndsWith("/index.json")) { list.Add(bcc.GetBlobClient($"{prefix}.htex").DeleteIfExistsAsync()); list.Add(bcc.GetBlobClient($"{prefix}.HTEX").DeleteIfExistsAsync()); } blobs.Add(new Uri(path)); } } } else { string path = $"{urib.Uri.AbsoluteUri}/{item.Name}"; list.Add(bcc.GetBlobClient(item.Name).DeleteIfExistsAsync()); blobs.Add(new Uri(path)); } }; } } if (list.Count > 0) { if (list.Count <= 256) { await Task.WhenAll(list); } else { int pages = (list.Count + 255) / 256; //256是批量操作最大值,pages = (total + max -1) / max; for (int i = 0; i < pages; i++) { List>> lists = list.Skip((i) * 256).Take(256).ToList(); await Task.WhenAll(lists); } } } return true; /* if (blobs.Count <= 256) { if (blobs.Count > 0) { try { ResponseMessage[] ass = await bbc.DeleteBlobsAsync(blobs); } catch (AggregateException ex) { //删除多个时会,如果其中一个文件不存在 或者其中一个删除失败会引发该异常 return true; } catch (RequestFailedException ex) { await _dingDing.SendBotMsg($"/{ex.Message}\n{ex.StackTrace}", GroupNames.醍摩豆服務運維群組); return true; } catch (Exception ex) { return true; } } return true; } else { int pages = (blobs.Count + 255) / 256; //256是批量操作最大值,pages = (total + max -1) / max; for (int i = 0; i < pages; i++) { List lists = blobs.Skip((i) * 256).Take(256).ToList(); try { ResponseMessage[] ass = await bbc.DeleteBlobsAsync(lists); } catch (AggregateException ex) { //删除多个时会,如果其中一个文件不存在 或者其中一个删除失败会引发该异常 return true; } catch (RequestFailedException ex) { return true; } catch (Exception ex) { return true; } } return true; } */ } catch(Exception ex ) { await _dingDing.SendBotMsg($"文件删除异常{ex.Message}\n{ex.StackTrace}{prefixs.ToJsonString()}", GroupNames.醍摩豆服務運維群組); return false; } } /// /// 保存操作记录 /// /// /// /// /// /// /// /// /// /// public static async Task SaveLog(this AzureStorageFactory azureStorage, string type, string msg,DingDing dingDing, string scope = null, string bizId = null, Option option = null, HttpContext httpContext = null) { var table = azureStorage.GetCloudTableClient().GetTableReference("IESOptLog"); OptLog log = new() { RowKey = Guid.NewGuid().ToString() }; try { object id = null, school = null, name = null, website = null ; httpContext?.Items.TryGetValue("ID", out id); httpContext?.Items.TryGetValue("School", out school); httpContext?.Items.TryGetValue("Name", out name); httpContext?.Items.TryGetValue("Website", out website); log.tmdId = id != null ? $"{id}" : log.tmdId; log.name = name != null ? $"{name}" : log.name; string host = httpContext?.Request?.Host.Value; log.school = school != null ? $"{school}" : log.school; log.PartitionKey = type != null ? $"Log-{type}" : "Log-Default"; log.RowKey = bizId != null ? bizId : log.RowKey; log.platform = website!=null? $"{website}" : "Default"; log.msg = msg; log.type = type; log.scope = scope; host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default"; log.url =$"{host}{httpContext?.Request.Path}" ; if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 256) { log.saveMod = 1; log.jsonfile = $"/0-public/optlog/{log.RowKey}-{log.PartitionKey}.json"; await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(log.ToJsonString(), "optlog", $"{log.RowKey}-{log.PartitionKey}.json"); log.msg = null; await table.SaveOrUpdate(log); } else { await table.SaveOrUpdate(log); } } catch (Exception ex) { _ = dingDing.SendBotMsg($"日志保存失败:{ex.Message}\n{ex.StackTrace},,{log.ToJsonString()}", GroupNames.醍摩豆服務運維群組); } } /// /// 保存日志文件 /// /// /// /// /// /// /// /// /// /// /// public static async Task SaveLogLang(this AzureStorageFactory azureStorage, string type,string rootPath, Dictionary replaceData, DingDing dingDing, string scope = null, string bizId = null, Option option = null, HttpContext httpContext = null) { string msg = FileWay.FileValue(rootPath, "zh-cn", type, replaceData); var table = azureStorage.GetCloudTableClient().GetTableReference("IESOptLog"); OptLog log = new() { RowKey = Guid.NewGuid().ToString() }; try { object id = null, school = null, name = null, website = null; httpContext?.Items.TryGetValue("ID", out id); httpContext?.Items.TryGetValue("School", out school); httpContext?.Items.TryGetValue("Name", out name); httpContext?.Items.TryGetValue("Website", out website); log.tmdId = id != null ? $"{id}" : log.tmdId; log.name = name != null ? $"{name}" : log.name; string host = httpContext?.Request?.Host.Value; log.school = school != null ? $"{school}" : log.school; log.PartitionKey = type != null ? $"Log-{type}" : "Log-Default"; log.RowKey = bizId != null ? bizId : log.RowKey; log.platform = website != null ? $"{website}" : "Default"; log.msg = msg; log.tmsg = FileWay.FileValue(rootPath, "zh-tw", type, replaceData); log.emsg = FileWay.FileValue(rootPath, "en-us", type, replaceData); log.type = type; log.scope = scope; host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default"; log.url = $"{host}{httpContext?.Request.Path}"; if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 256) { log.saveMod = 1; log.jsonfile = $"/0-public/optlog/{log.RowKey}-{log.PartitionKey}.json"; await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(log.ToJsonString(), "optlog", $"{log.RowKey}-{log.PartitionKey}.json"); log.msg = null; await table.SaveOrUpdate(log); } else { await table.SaveOrUpdate(log); } } catch (Exception ex) { _ = dingDing.SendBotMsg($"日志保存失败:{ex.Message}\n{ex.StackTrace},,{log.ToJsonString()}", GroupNames.醍摩豆服務運維群組); } } /// /// 系统管理员 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联 /// "system": [ "res", "item", "htex", "task", "info", "room", "exam" ], /// 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联 /// "school": [ "res", "item", "htex", "task", "info", "room", "exam" ], /// 资源,题目关联,htex关联,学习活动关联,教师基本信息关联 /// "teacher": [ "res", "item", "htex", "task", "info" ], /// 答案及学习活动上传的文件,学生基本信息关联 ///"student": [ "stu/{studentId}/ans", "stu/{studentId}/task" ] /// /// 容器名称 /// 文件内容的流 /// 业务文件夹 /// 文件名 /// 是否存放文件后缀对应的contentType /// public static async Task UploadFileByContainer(this BlobContainerClient blobContainer, string json, string root, string blobpath, bool contentTypeDefault = true) { // string groupName =folder; //BlobContainerClient blobContainer = azureStorage.GetBlobContainerClient(name.ToLower().Replace("#", "")); //blobClient.GetContainerReference(groupName); var blockBlob = blobContainer.GetBlobClient($"{root}/{blobpath}"); string content_type = "application/octet-stream"; if (!contentTypeDefault) { string fileext = blobpath.Substring(blobpath.LastIndexOf(".") > 0 ? blobpath.LastIndexOf(".") : 0); ContentTypeDict.dict.TryGetValue(fileext, out string contenttype); if (!string.IsNullOrEmpty(contenttype)) { content_type = contenttype; } } byte[] bytes = System.Text.Encoding.Default.GetBytes(json); Stream streamBlob = new MemoryStream(bytes); await blockBlob.UploadAsync(streamBlob, true); blockBlob.SetHttpHeaders(new BlobHttpHeaders { ContentType = content_type }); return blockBlob.Uri.ToString(); } /// /// 系统管理员 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联 /// "system": [ "res", "item", "htex", "task", "info", "room", "exam" ], /// 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联 /// "school": [ "res", "item", "htex", "task", "info", "room", "exam" ], /// 资源,题目关联,htex关联,学习活动关联,教师基本信息关联 /// "teacher": [ "res", "item", "htex", "task", "info" ], /// 答案及学习活动上传的文件,学生基本信息关联 ///"student": [ "stu/{studentId}/ans", "stu/{studentId}/task" ] /// /// 容器名称 /// 文件内容的流 /// 业务文件夹 /// 文件名 /// 是否存放文件后缀对应的contentType /// public static async Task UploadFileByContainer(this BlobContainerClient blobContainer, Stream stream, string root, string blobpath, bool contentTypeDefault = true) { //BlobContainerClient blobContainer = azureStorage.GetBlobContainerClient(name.ToLower().Replace("#", "")); //blobClient.GetContainerReference(groupName); Uri url = blobContainer.Uri; var blockBlob = blobContainer.GetBlobClient($"{root}/{blobpath}"); string content_type = "application/octet-stream"; if (!contentTypeDefault) { string fileext = blobpath.Substring(blobpath.LastIndexOf(".") > 0 ? blobpath.LastIndexOf(".") : 0); ContentTypeDict.dict.TryGetValue(fileext, out string contenttype); if (!string.IsNullOrEmpty(contenttype)) { content_type = contenttype; } } await blockBlob.UploadAsync(stream, true); blockBlob.SetHttpHeaders(new BlobHttpHeaders { ContentType = content_type }); return blockBlob.Uri.ToString(); } /// /// BI保存操作记录 /// /// /// /// /// /// /// /// /// public static async Task SaveBILog(BlobContainerClient blobContainer, CloudTableClient tableClient, string type, string msg, DingDing dingDing, string tid = null, string tname = null, string twebsite = null, string scope = null, Option option = null, HttpContext httpContext = null) { var table = tableClient.GetTableReference("BIOptLog"); BIOptLog biLog = new() { RowKey = Guid.NewGuid().ToString() }; try { object id = null, name = null, ddid = null, ddname = null, website = null; httpContext?.Items.TryGetValue("ID", out id); httpContext?.Items.TryGetValue("Name", out name); httpContext?.Items.TryGetValue("DDId", out ddid); httpContext?.Items.TryGetValue("DDName", out ddname); httpContext?.Items.TryGetValue("Website", out website); string site = twebsite != null ? twebsite : $"{website}"; biLog.tmdId = id != null ? $"{id}" : tid; biLog.name = name != null ? $"{name}" : tname; biLog.PartitionKey = type != null ? $"{site}-Log-{type}" : $"{site}-Log-Default"; biLog.platform = site != null ? site : "Default"; biLog.msg = msg; biLog.type = type; biLog.scope = scope; string host = httpContext?.Request?.Host.Value; host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default"; biLog.url = $"{host}{httpContext?.Request.Path}"; if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 255) { biLog.saveMod = 1; biLog.jsonfile = $"/0-public/BIOptLog/{biLog.PartitionKey}-{biLog.RowKey}.json"; await UploadFileByContainer(blobContainer, biLog.ToJsonString(), "BIOptLog", $"{biLog.PartitionKey}-{biLog.RowKey}.json"); biLog.msg = null; await table.SaveOrUpdate(biLog); } else await table.SaveOrUpdate(biLog); } catch (Exception ex) { _ = dingDing.SendBotMsg($"BI日志保存失败:{ex.Message}\n{ex.StackTrace},,{biLog.ToJsonString()}", GroupNames.成都开发測試群組); } } } }