TEAMModelOS/TEAMModelOS.SDK/Models/Service/Common/BlobService.cs

using Azure;
using Azure.Cosmos;
using Azure.Messaging.ServiceBus;
using Azure.Storage.Blobs;
using DocumentFormat.OpenXml.Spreadsheet;
using DocumentFormat.OpenXml.Wordprocessing;
using HTEXLib.COMM.Helpers;
using Microsoft.Azure.Cosmos.Table;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using StackExchange.Redis;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using TEAMModelOS.Models;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Models;
using TEAMModelOS.SDK.Models.Service;
using static Google.Protobuf.Reflection.SourceCodeInfo.Types;

namespace TEAMModelOS.SDK.Services
{
    public static class BlobService
    {

        public static async Task RefreshBlobRoot(BlobRefreshMessage message, AzureServiceBusFactory _serviceBus,IConfiguration _configuration,AzureRedisFactory _azureRedis) {
            if (!string.IsNullOrWhiteSpace(message.root) && !string.IsNullOrWhiteSpace(message.name)) {
                string changeKey = $"Blob:Change:{message.name}";


                string lockKey = $"Blob:Lock:{message.name}:{message.root}";
                bool exist = await _azureRedis.GetRedisClient(8).KeyExistsAsync(lockKey);
                //不存在Blob:Lock:hbcn:video 文件夹在队列中 则加入队列
                if (!exist)
                {
                    //保持一天的时间
                    TimeSpan timeSpan = new TimeSpan(1,0,0);
                    //加入队列的时间
                    long action = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
                    await _azureRedis.GetRedisClient(8).StringSetAsync(lockKey, action, expiry: timeSpan);
                    var messageBlob = new ServiceBusMessage(message.ToJsonString());
                    messageBlob.ApplicationProperties.Add("name", "BlobRoot");
                    var ActiveTask = _configuration.GetValue<string>("Azure:ServiceBus:ActiveTask");
                    await _serviceBus.GetServiceBusClient().SendMessageAsync(ActiveTask, messageBlob);
                }
                //如果已经存在，则忽略，不加入队列。
            }
        }
        /// <summary>
        /// 获取学校或者个人的剩余空间
        /// </summary>
        /// <param name="name">学校id 或者醍摩豆id</param>
        /// <param name="scope"></param>
        /// <param name="_azureCosmos"></param>
        /// <param name="_azureRedis"></param>
        public static async Task<(long usedSize, long teach, long total, long surplus, Dictionary<string, double?> catalog)> 
            GetSurplusSpace(string name, string scope,string location, AzureCosmosFactory _azureCosmos, AzureRedisFactory _azureRedis, AzureStorageFactory _azureStorage,DingDing _dingDing, HttpTrigger _httpTrigger)
        {
            //已经存储的空间
            long usedSize = 0;
            //分配给教师的
            long teach = 0;
            //总空间
            long total = 0;
            // 剩余的
            long surplus = 0;
            Teacher teacher = null;
            Dictionary<string, double?> catalog = new Dictionary<string, double?>();
            try
            {
                if (!string.IsNullOrWhiteSpace(name))
                {

                    if ("school".Equals(scope, StringComparison.OrdinalIgnoreCase))
                    {

                        School school = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemAsync<School>(name, new PartitionKey("Base"));
                        total = school.size;
                        teach = school.tsize;
                        //用tsize代替
                        //await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryStreamIterator(queryText: $"SELECT sum(c.size) as size FROM c ",
                        //    requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Teacher-{name}") }))
                        //{
                        //    var json = await JsonDocument.ParseAsync(item.ContentStream);
                        //    foreach (var elmt in json.RootElement.GetProperty("Documents").EnumerateArray())
                        //    {
                        //        if (elmt.TryGetProperty("size", out JsonElement _size) && _size.ValueKind.Equals(JsonValueKind.Number))
                        //        {
                        //            teach = _size.GetInt32();
                        //            break;
                        //        }
                        //    }
                        //}
                    }
                    else
                    {
                         teacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).ReadItemAsync<Teacher>(name, new PartitionKey("Base"));
                        total = teacher.size;
                        foreach (var school in teacher.schools)
                        {
                            try
                            {
                                SchoolTeacher schoolTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemAsync<SchoolTeacher>(name, new PartitionKey($"Teacher-{school.schoolId}"));
                                total += schoolTeacher.size;
                            }
                            catch (Exception ex) { }
                        }
                    }
                    long blobsize = 0;
                    RedisValue value = default;
                    value = _azureRedis.GetRedisClient(8).HashGet($"Blob:Record", name);
                    if (value != default && !value.IsNullOrEmpty)
                    {
                        JsonElement record = value.ToString().ToObject<JsonElement>();
                        if (record.TryGetInt64(out blobsize))
                        {
                        }
                    }
                    else
                    {
                        var client = _azureStorage.GetBlobContainerClient(name);
                        var size = await client.GetBlobsCatalogSize();
                        if (size.Item1 > 0)
                        {
                            await _azureRedis.GetRedisClient(8).HashSetAsync($"Blob:Record", name, size.Item1);
                        }

                        foreach (var key in size.Item2.Keys)
                        {
                            await _azureRedis.GetRedisClient(8).SortedSetRemoveAsync($"Blob:Catalog:{name}", key);
                            await _azureRedis.GetRedisClient(8).SortedSetIncrementAsync($"Blob:Catalog:{name}", key, size.Item2[key].HasValue ? size.Item2[key].Value : 0);
                        }
                        usedSize = size.Item1.Value;
                        //1024 * 1024 * 1024=1073741824  =1G
                        surplus = (total - teach) * 1073741824 - usedSize;
                        catalog=size.Item2;
                        //return (usedSize, teach, total, surplus, catalog = size.Item2);
                    }

                    SortedSetEntry[] Scores = _azureRedis.GetRedisClient(8).SortedSetRangeByScoreWithScores($"Blob:Catalog:{name}");
                    if (Scores != null)
                    {
                        foreach (var score in Scores)
                        {
                            double val = score.Score;
                            string key = score.Element.ToString();
                            if (!string.IsNullOrWhiteSpace(key))
                            {
                                catalog[key] = val;
                            }
                        }
                        usedSize = blobsize;
                        //1024 * 1024 * 1024=1073741824  =1G
                        surplus = (total - teach) * 1073741824 - usedSize;
                        //return (usedSize, teach, total, surplus, catalog);
                    }
                    else
                    {

                        await  _dingDing.SendBotMsg($"{location}Blob空间计算没有缓存{name}", GroupNames.成都开发測試群組);
                        //没有缓存
                        var client = _azureStorage.GetBlobContainerClient(name);
                        var size = await client.GetBlobsCatalogSize();
                        if (size.Item1 > 0)
                        {
                            await _azureRedis.GetRedisClient(8).HashSetAsync($"Blob:Record", name, size.Item1);
                        }
                        foreach (var key in size.Item2.Keys)
                        {
                            await _azureRedis.GetRedisClient(8).SortedSetRemoveAsync($"Blob:Catalog:{name}", key);
                            await _azureRedis.GetRedisClient(8).SortedSetIncrementAsync($"Blob:Catalog:{name}", key, size.Item2[key].HasValue ? size.Item2[key].Value : 0);
                        }
                        usedSize = size.Item1.Value;
                        //1024 * 1024 * 1024=1073741824  =1G
                        //surplus 可能为负数
                        surplus = (total - teach) * 1073741824 - usedSize;
                        catalog = size.Item2;
                        //return (usedSize, teach, total, surplus, catalog = size.Item2);
                    }
                }
            }
            catch (Exception ex)
            {
                await _dingDing.SendBotMsg($"IES5，{location}，GetSurplusSpace \n{ex.Message}\n{ex.StackTrace}\n{name},\n{scope}", GroupNames.醍摩豆服務運維群組);
                return (usedSize, teach, total, surplus, catalog);
            }
            if (usedSize > 0 && total > 0) {

                double  percent = (surplus * 1.0 * 100 / 1073741824 / total );
                if (percent < 10) {
                    _ = _httpTrigger.RequestHttpTrigger(new { name, scope, percent }, location, "surplus-space-notify");
                }
            }
            return (usedSize, teach, total, surplus, catalog);
        }
        /// <summary>
        /// 记录在redis,使用ttl 进行过期管理
        /// </summary>
        public class BlobSpaceNotify
        {
            /// <summary>
            /// id=$"{tmdid}-{tag}"
            /// </summary>
            public string? id { get; set; }
            public int tag { get; set; }
            public string? containerName { get; set; }
            public string? scope { get; set; }
            public string? notifyIndex { get; set; }
        }

        /// <param name="scope">school or private</param>
        /// <param name="containerName">school code or tmid</param>
        /// <param name="type">doc, res, item... Empty if no need.</param>
        /// <param name="periodId">Only for school</param>
        public static async Task<List<BlobCount>> BloblogCount(CosmosClient clientc, string scope, string containerName, string type, string periodId)
        {
            List<BlobCount> bloblogcnt = new List<BlobCount>();
            try
            {
                //必須項檢查
                if(scope.Equals("school") && string.IsNullOrWhiteSpace(periodId))
                {
                    return bloblogcnt;
                }
                //資料取得
                var queryslt = new StringBuilder();
                queryslt.Append($"SELECT COUNT(1) AS count, c.type FROM c ");
                if (scope.Equals("school"))
                {
                    queryslt.Append($"JOIN A1 IN c.periodId WHERE A1 IN ('{periodId}') ");
                    if(!string.IsNullOrWhiteSpace(type)) queryslt.Append($"AND c.type='{type}'");
                    queryslt.Append("GROUP BY c.type");
                   await foreach (var item in clientc.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator<BlobCount>(queryText: queryslt.ToString(), requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Bloblog-{containerName}") }))
                    {
                        bloblogcnt.Add(item);
                    }
                }
                else if (scope.Equals("private"))
                {
                    if (!string.IsNullOrWhiteSpace(type)) queryslt.Append($"AND c.type='{type}'");
                    queryslt.Append("GROUP BY c.type");
                    await foreach (var item in clientc.GetContainer(Constant.TEAMModelOS, "Teacher").GetItemQueryIterator<BlobCount>(queryText: queryslt.ToString(), requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Bloblog-{containerName}") }))
                    {
                        bloblogcnt.Add(item);
                    }
                }

                return bloblogcnt;
            }
            catch (Exception ex)
            {
                return bloblogcnt;
            }
        }

        public class UsedBlob
        {
            public long teach { get; set; }
            public long? size { get; set; }
            public Dictionary<string, double?> catalog { get; set; }

        }

        public class BlobCount
        {
            public string type { get; set; }
            public int count { get; set; }
        }
    }
    public class BlobRefreshMessage{ 
        public string id { get; set; }= Guid.NewGuid().ToString();
        public string name { get; set; }
        public string progress { get; set; }
        public string root { get; set; }
    }
}