using System.Threading.Tasks;
using Azure.Storage;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System;
using System.IO;
using Azure.Storage.Blobs.Specialized;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Azure.Core;
using Azure;
using TEAMModelOS.SDK;
using TEAMModelOS.SDK.Extension;
using System.Text.Encodings.Web;
using TEAMModelOS.SDK.Models.Table;
using Microsoft.AspNetCore.Http;
using TEAMModelOS.Models;
using Microsoft.Table;
using static TEAMModelOS.SDK.CoreAPIHttpService;
namespace TEAMModelOS.SDK.DI
{
public static class AzureStorageBlobExtensions
{
///
/// 取得指定前置詞的 Blob 名稱的總大小(Bytes),例如指定目錄名稱為前置詞
///
/// 篩選開頭名稱,Null代表容器總大小
/// 總大小(Bytes),如果為Null代表查無前置詞或者發生錯誤
public static async Task GetBlobsSize(this BlobContainerClient client, string prefix = null)
{
long? size = 0;
try
{
await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix))
{
if (item.Name.StartsWith("res/", StringComparison.OrdinalIgnoreCase) && item.Name.EndsWith(".htex", StringComparison.OrdinalIgnoreCase))
{ continue; }
if (!prefix.Equals(item.Name))
{
//避免操作(1111) /1111/1111.json /1111111/11111.json
if (!prefix.EndsWith("/"))
{
if (item.Name.StartsWith(prefix + "/"))
{
size += item.Properties.ContentLength;
}
}
}
else
{
size += item.Properties.ContentLength;
}
};
return size;
}
catch
{
return size;
}
}
public static async Task> List(this BlobContainerClient client, string prefix = null) {
try
{
List items = new List();
await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix)) {
items.Add(item.Name);
}
return items;
}
catch
{
return null;
}
}
///
/// 取得指定前置詞的 Blob 名稱的總大小(Bytes),例如指定目錄名稱為前置詞
///
/// 篩選開頭名稱,Null代表容器總大小
/// 總大小(Bytes),如果為Null代表查無前置詞或者發生錯誤
public static async Task<(long?, Dictionary)> GetBlobsCatalogSize(this BlobContainerClient client, string prefix = null)
{
long? size = 0;
Dictionary dict = new Dictionary();
try
{
List> foderSize = new List>();
await foreach (BlobItem item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, prefix))
{
var len = item.Properties.ContentLength;
foderSize.Add(new KeyValuePair(item.Name.Split("/")[0], len));
size += item.Properties.ContentLength;
};
foderSize.Select(x => new { x.Key, x.Value }).GroupBy(y=>y.Key).ToList().ForEach(g=> {
var gpsize = g.Select(m => m.Value).Sum();
dict[g.Key] = gpsize;
});
return (size, dict);
}
catch
{
return (size, dict);
}
}
public class OptUrl
{
public string url { get; set; }
public long size { get; set; }
}
///
/// 取得指定前置詞的 Blob 名稱的總大小(Bytes),例如指定目錄名稱為前置詞
///
/// 多个文件的连接/param>
/// 總大小(Bytes),如果為Null代表查無前置詞或者發生錯誤
public static async Task> GetBlobsSize(this BlobContainerClient client, List urls)
{
List optUrls = new List();
try
{
if (urls != null) {
foreach (var url in urls)
{
OptUrl optUrl = new OptUrl { url = url, size = 0 };
var eurl = System.Web.HttpUtility.UrlDecode(url, Encoding.UTF8);
var blob = client.GetBlobClient(eurl);
if (blob.Exists())
{
var props = await blob.GetPropertiesAsync();
var size = props.Value.ContentLength;
optUrl.size = size;
}
optUrls.Add(optUrl);
}
}
return optUrls;
}
catch
{
return optUrls;
}
}
///
///prefixs多个文件或文件夹路径删除
///prefixs 或者按前缀文件夹删除
///
///
/// 篩選開頭名稱,Null代表容器
public static async Task DeleteBlobs(this BlobServiceClient client,DingDing _dingDing, string blobContainerName, List prefixs )
{
if (!prefixs.IsNotEmpty()) return false;
try
{
BlobContainerClient bcc = client.GetBlobContainerClient(blobContainerName.ToLower());
BlobBatchClient bbc = client.GetBlobBatchClient();
List blobs = new List();
List>> list = new List>>();
foreach (var prefix in prefixs) {
string px = prefix;
if (prefix.StartsWith("/")) {
px= prefix.Substring(1);
}
//目录必须有两层以上,避免删除根目录所有的。
var pxlen= px.Split("/");
if (pxlen.Length >=2) {
var items = bcc.GetBlobsAsync(BlobTraits.None, BlobStates.None, px);
await foreach (var item in items)
{
var urib = new UriBuilder(bcc.Uri);
if (!prefix.Equals(item.Name))
{
//避免操作(1111) /1111/1111.json /1111111/11111.json
if (!prefix.EndsWith("/"))
{
if (item.Name.StartsWith(prefix + "/"))
{
string path = $"{urib.Uri.AbsoluteUri}/{item.Name}";
list.Add(bcc.GetBlobClient(item.Name).DeleteIfExistsAsync());
if (item.Name.StartsWith("res/") && item.Name.EndsWith("/index.json"))
{
list.Add(bcc.GetBlobClient($"{prefix}.htex").DeleteIfExistsAsync());
list.Add(bcc.GetBlobClient($"{prefix}.HTEX").DeleteIfExistsAsync());
}
blobs.Add(new Uri(path));
}
}
}
else
{
string path = $"{urib.Uri.AbsoluteUri}/{item.Name}";
list.Add(bcc.GetBlobClient(item.Name).DeleteIfExistsAsync());
blobs.Add(new Uri(path));
}
};
}
}
if (list.Count > 0) {
if (list.Count <= 256)
{
await Task.WhenAll(list);
}
else
{
int pages = (list.Count + 255) / 256; //256是批量操作最大值,pages = (total + max -1) / max;
for (int i = 0; i < pages; i++)
{
List>> lists = list.Skip((i) * 256).Take(256).ToList();
await Task.WhenAll(lists);
}
}
}
return true;
/*
if (blobs.Count <= 256)
{
if (blobs.Count > 0) {
try {
ResponseMessage[] ass = await bbc.DeleteBlobsAsync(blobs);
}
catch (AggregateException ex)
{
//删除多个时会,如果其中一个文件不存在 或者其中一个删除失败会引发该异常
return true;
}
catch (RequestFailedException ex)
{
await _dingDing.SendBotMsg($"/{ex.Message}\n{ex.StackTrace}", GroupNames.醍摩豆服務運維群組);
return true;
}
catch (Exception ex)
{
return true;
}
}
return true;
}
else
{
int pages = (blobs.Count + 255) / 256; //256是批量操作最大值,pages = (total + max -1) / max;
for (int i = 0; i < pages; i++)
{
List lists = blobs.Skip((i) * 256).Take(256).ToList();
try { ResponseMessage[] ass = await bbc.DeleteBlobsAsync(lists); }
catch (AggregateException ex)
{
//删除多个时会,如果其中一个文件不存在 或者其中一个删除失败会引发该异常
return true;
}
catch (RequestFailedException ex)
{
return true;
}
catch (Exception ex)
{
return true;
}
}
return true;
}
*/
}
catch(Exception ex )
{
await _dingDing.SendBotMsg($"文件删除异常{ex.Message}\n{ex.StackTrace}{prefixs.ToJsonString()}", GroupNames.醍摩豆服務運維群組);
return false;
}
}
///
/// 保存操作记录
///
///
///
///
///
///
///
///
///
///
public static async Task SaveLog(this AzureStorageFactory azureStorage, string type, string msg,DingDing dingDing, string scope = null, string bizId = null, Option option = null, HttpContext httpContext = null)
{
var table = azureStorage.GetCloudTableClient().GetTableReference("IESOptLog");
OptLog log = new() { RowKey = Guid.NewGuid().ToString() };
try
{
object id = null, school = null, name = null, website = null ;
httpContext?.Items.TryGetValue("ID", out id);
httpContext?.Items.TryGetValue("School", out school);
httpContext?.Items.TryGetValue("Name", out name);
httpContext?.Items.TryGetValue("Website", out website);
log.tmdId = id != null ? $"{id}" : log.tmdId;
log.name = name != null ? $"{name}" : log.name;
string host = httpContext?.Request?.Host.Value;
log.school = school != null ? $"{school}" : log.school;
log.PartitionKey = type != null ? $"Log-{type}" : "Log-Default";
log.RowKey = bizId != null ? bizId : log.RowKey;
log.platform = website!=null? $"{website}" : "Default";
log.msg = msg;
log.type = type;
log.scope = scope;
host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default";
log.url =$"{host}{httpContext?.Request.Path}" ;
if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 256)
{
log.saveMod = 1;
log.jsonfile = $"/0-public/optlog/{log.RowKey}-{log.PartitionKey}.json";
await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(log.ToJsonString(), "optlog", $"{log.RowKey}-{log.PartitionKey}.json");
log.msg = null;
await table.SaveOrUpdate(log);
}
else {
await table.SaveOrUpdate(log);
}
}
catch (Exception ex)
{
_ = dingDing.SendBotMsg($"日志保存失败:{ex.Message}\n{ex.StackTrace},,{log.ToJsonString()}", GroupNames.醍摩豆服務運維群組);
}
}
///
/// 保存日志文件
///
///
///
///
///
///
///
///
///
///
///
public static async Task SaveLogLang(this AzureStorageFactory azureStorage, string type,string rootPath, Dictionary replaceData, DingDing dingDing, string scope = null, string bizId = null, Option option = null, HttpContext httpContext = null)
{
string msg = FileWay.FileValue(rootPath, "zh-cn", type, replaceData);
var table = azureStorage.GetCloudTableClient().GetTableReference("IESOptLog");
OptLog log = new() { RowKey = Guid.NewGuid().ToString() };
try
{
object id = null, school = null, name = null, website = null;
httpContext?.Items.TryGetValue("ID", out id);
httpContext?.Items.TryGetValue("School", out school);
httpContext?.Items.TryGetValue("Name", out name);
httpContext?.Items.TryGetValue("Website", out website);
log.tmdId = id != null ? $"{id}" : log.tmdId;
log.name = name != null ? $"{name}" : log.name;
string host = httpContext?.Request?.Host.Value;
log.school = school != null ? $"{school}" : log.school;
log.PartitionKey = type != null ? $"Log-{type}" : "Log-Default";
log.RowKey = bizId != null ? bizId : log.RowKey;
log.platform = website != null ? $"{website}" : "Default";
log.msg = msg;
log.tmsg = FileWay.FileValue(rootPath, "zh-tw", type, replaceData);
log.emsg = FileWay.FileValue(rootPath, "en-us", type, replaceData);
log.type = type;
log.scope = scope;
host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default";
log.url = $"{host}{httpContext?.Request.Path}";
if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 256)
{
log.saveMod = 1;
log.jsonfile = $"/0-public/optlog/{log.RowKey}-{log.PartitionKey}.json";
await azureStorage.GetBlobContainerClient("0-public").UploadFileByContainer(log.ToJsonString(), "optlog", $"{log.RowKey}-{log.PartitionKey}.json");
log.msg = null;
await table.SaveOrUpdate(log);
}
else
{
await table.SaveOrUpdate(log);
}
}
catch (Exception ex)
{
_ = dingDing.SendBotMsg($"日志保存失败:{ex.Message}\n{ex.StackTrace},,{log.ToJsonString()}", GroupNames.醍摩豆服務運維群組);
}
}
///
/// 系统管理员 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联
/// "system": [ "res", "item", "htex", "task", "info", "room", "exam" ],
/// 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联
/// "school": [ "res", "item", "htex", "task", "info", "room", "exam" ],
/// 资源,题目关联,htex关联,学习活动关联,教师基本信息关联
/// "teacher": [ "res", "item", "htex", "task", "info" ],
/// 答案及学习活动上传的文件,学生基本信息关联
///"student": [ "stu/{studentId}/ans", "stu/{studentId}/task" ]
///
/// 容器名称
/// 文件内容的流
/// 业务文件夹
/// 文件名
/// 是否存放文件后缀对应的contentType
///
public static async Task UploadFileByContainer(this BlobContainerClient blobContainer, string json, string root, string blobpath, bool contentTypeDefault = true)
{
// string groupName =folder;
//BlobContainerClient blobContainer = azureStorage.GetBlobContainerClient(name.ToLower().Replace("#", "")); //blobClient.GetContainerReference(groupName);
var blockBlob = blobContainer.GetBlobClient($"{root}/{blobpath}");
string content_type = "application/octet-stream";
if (!contentTypeDefault)
{
string fileext = blobpath.Substring(blobpath.LastIndexOf(".") > 0 ? blobpath.LastIndexOf(".") : 0);
ContentTypeDict.dict.TryGetValue(fileext, out string contenttype);
if (!string.IsNullOrEmpty(contenttype))
{
content_type = contenttype;
}
}
byte[] bytes = System.Text.Encoding.Default.GetBytes(json);
Stream streamBlob = new MemoryStream(bytes);
await blockBlob.UploadAsync(streamBlob, true);
blockBlob.SetHttpHeaders(new BlobHttpHeaders { ContentType = content_type });
return blockBlob.Uri.ToString();
}
///
/// 系统管理员 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联
/// "system": [ "res", "item", "htex", "task", "info", "room", "exam" ],
/// 资源,题目关联,htex关联,学习活动学生上传文件关联,基本信息关联,教室平面图关联,评测冷数据关联
/// "school": [ "res", "item", "htex", "task", "info", "room", "exam" ],
/// 资源,题目关联,htex关联,学习活动关联,教师基本信息关联
/// "teacher": [ "res", "item", "htex", "task", "info" ],
/// 答案及学习活动上传的文件,学生基本信息关联
///"student": [ "stu/{studentId}/ans", "stu/{studentId}/task" ]
///
/// 容器名称
/// 文件内容的流
/// 业务文件夹
/// 文件名
/// 是否存放文件后缀对应的contentType
///
public static async Task UploadFileByContainer(this BlobContainerClient blobContainer, Stream stream, string root, string blobpath, bool contentTypeDefault = true)
{
//BlobContainerClient blobContainer = azureStorage.GetBlobContainerClient(name.ToLower().Replace("#", "")); //blobClient.GetContainerReference(groupName);
Uri url = blobContainer.Uri;
var blockBlob = blobContainer.GetBlobClient($"{root}/{blobpath}");
string content_type = "application/octet-stream";
if (!contentTypeDefault)
{
string fileext = blobpath.Substring(blobpath.LastIndexOf(".") > 0 ? blobpath.LastIndexOf(".") : 0);
ContentTypeDict.dict.TryGetValue(fileext, out string contenttype);
if (!string.IsNullOrEmpty(contenttype))
{
content_type = contenttype;
}
}
await blockBlob.UploadAsync(stream, true);
blockBlob.SetHttpHeaders(new BlobHttpHeaders { ContentType = content_type });
return blockBlob.Uri.ToString();
}
///
/// BI保存操作记录
///
///
///
///
///
///
///
///
///
public static async Task SaveBILog(BlobContainerClient blobContainer, CloudTableClient tableClient, string type, string msg, DingDing dingDing, string tid = null, string tname = null, string twebsite = null, string scope = null, Option option = null, HttpContext httpContext = null)
{
var table = tableClient.GetTableReference("BIOptLog");
BIOptLog biLog = new() { RowKey = Guid.NewGuid().ToString() };
try
{
object id = null, name = null, ddid = null, ddname = null, website = null;
httpContext?.Items.TryGetValue("ID", out id);
httpContext?.Items.TryGetValue("Name", out name);
httpContext?.Items.TryGetValue("DDId", out ddid);
httpContext?.Items.TryGetValue("DDName", out ddname);
httpContext?.Items.TryGetValue("Website", out website);
string site = twebsite != null ? twebsite : $"{website}";
biLog.tmdId = id != null ? $"{id}" : tid;
biLog.name = name != null ? $"{name}" : tname;
biLog.PartitionKey = type != null ? $"{site}-Log-{type}" : $"{site}-Log-Default";
biLog.platform = site != null ? site : "Default";
biLog.msg = msg;
biLog.type = type;
biLog.scope = scope;
string host = httpContext?.Request?.Host.Value;
host = !string.IsNullOrWhiteSpace($"{host}") ? $"{host}" : option?.Location != null ? $"{host}" : "Default";
biLog.url = $"{host}{httpContext?.Request.Path}";
if (!string.IsNullOrWhiteSpace(msg) && msg.Length > 255)
{
biLog.saveMod = 1;
biLog.jsonfile = $"/0-public/BIOptLog/{biLog.PartitionKey}-{biLog.RowKey}.json";
await UploadFileByContainer(blobContainer, biLog.ToJsonString(), "BIOptLog", $"{biLog.PartitionKey}-{biLog.RowKey}.json");
biLog.msg = null;
await table.SaveOrUpdate(biLog);
}
else await table.SaveOrUpdate(biLog);
}
catch (Exception ex)
{
_ = dingDing.SendBotMsg($"BI日志保存失败:{ex.Message}\n{ex.StackTrace},,{biLog.ToJsonString()}", GroupNames.成都开发測試群組);
}
}
}
}