BlobService.cs 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295
  1. using Azure;
  2. using Azure.Cosmos;
  3. using Azure.Messaging.ServiceBus;
  4. using Azure.Storage.Blobs;
  5. using DocumentFormat.OpenXml.Wordprocessing;
  6. using HTEXLib.COMM.Helpers;
  7. using Microsoft.Azure.Cosmos.Table;
  8. using Microsoft.Extensions.Configuration;
  9. using StackExchange.Redis;
  10. using System;
  11. using System.Collections.Generic;
  12. using System.Linq;
  13. using System.Text;
  14. using System.Text.Json;
  15. using System.Threading.Tasks;
  16. using TEAMModelOS.Models;
  17. using TEAMModelOS.SDK.DI;
  18. using TEAMModelOS.SDK.Extension;
  19. using TEAMModelOS.SDK.Models;
  20. namespace TEAMModelOS.SDK.Services
  21. {
  22. public static class BlobService
  23. {
  24. public static async Task RefreshBlobRoot(BlobRefreshMessage message, AzureServiceBusFactory _serviceBus,IConfiguration _configuration,AzureRedisFactory _azureRedis) {
  25. if (!string.IsNullOrWhiteSpace(message.root) && !string.IsNullOrWhiteSpace(message.name)) {
  26. string lockKey = $"Blob:Lock:{message.name}:{message.root}";
  27. bool exist = await _azureRedis.GetRedisClient(8).KeyExistsAsync(lockKey);
  28. //不存在Blob:Lock:hbcn:video 文件夹在队列中 则加入队列
  29. if (!exist)
  30. {
  31. //保持一天的时间
  32. TimeSpan timeSpan = new TimeSpan(1,0,0);
  33. //加入队列的时间
  34. long action = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
  35. await _azureRedis.GetRedisClient(8).StringSetAsync(lockKey, action, expiry: timeSpan);
  36. var messageBlob = new ServiceBusMessage(message.ToJsonString());
  37. messageBlob.ApplicationProperties.Add("name", "BlobRoot");
  38. var ActiveTask = _configuration.GetValue<string>("Azure:ServiceBus:ActiveTask");
  39. await _serviceBus.GetServiceBusClient().SendMessageAsync(ActiveTask, messageBlob);
  40. }
  41. //如果已经存在,则忽略,不加入队列。
  42. }
  43. }
  44. /// <summary>
  45. /// 获取学校或者个人的剩余空间
  46. /// </summary>
  47. /// <param name="name">学校id 或者醍摩豆id</param>
  48. /// <param name="scope"></param>
  49. /// <param name="_azureCosmos"></param>
  50. /// <param name="_azureRedis"></param>
  51. public static async Task<(long usedSize, long teach, long total, long surplus, Dictionary<string, double?> catalog)> GetSurplusSpace(string name, string scope,Option _option, AzureCosmosFactory _azureCosmos, AzureRedisFactory _azureRedis, AzureStorageFactory _azureStorage,DingDing _dingDing )
  52. {
  53. //已经存储的空间
  54. long usedSize = 0;
  55. //分配给教师的
  56. long teach = 0;
  57. //总空间
  58. long total = 0;
  59. // 剩余的
  60. long surplus = 0;
  61. Dictionary<string, double?> catalog = new Dictionary<string, double?>();
  62. try
  63. {
  64. if (!string.IsNullOrWhiteSpace(name))
  65. {
  66. if ("school".Equals(scope, StringComparison.OrdinalIgnoreCase))
  67. {
  68. School school = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemAsync<School>(name, new PartitionKey("Base"));
  69. total = school.size;
  70. teach = school.tsize;
  71. //用tsize代替
  72. //await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryStreamIterator(queryText: $"SELECT sum(c.size) as size FROM c ",
  73. // requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Teacher-{name}") }))
  74. //{
  75. // var json = await JsonDocument.ParseAsync(item.ContentStream);
  76. // foreach (var elmt in json.RootElement.GetProperty("Documents").EnumerateArray())
  77. // {
  78. // if (elmt.TryGetProperty("size", out JsonElement _size) && _size.ValueKind.Equals(JsonValueKind.Number))
  79. // {
  80. // teach = _size.GetInt32();
  81. // break;
  82. // }
  83. // }
  84. //}
  85. }
  86. else
  87. {
  88. Teacher teacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).ReadItemAsync<Teacher>(name, new PartitionKey("Base"));
  89. total = teacher.size;
  90. foreach (var school in teacher.schools)
  91. {
  92. try
  93. {
  94. SchoolTeacher schoolTeacher = await _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).ReadItemAsync<SchoolTeacher>(name, new PartitionKey($"Teacher-{school.schoolId}"));
  95. total += schoolTeacher.size;
  96. }
  97. catch (Exception ex) { }
  98. }
  99. }
  100. long blobsize = 0;
  101. RedisValue value = default;
  102. value = _azureRedis.GetRedisClient(8).HashGet($"Blob:Record", name);
  103. if (value != default && !value.IsNullOrEmpty)
  104. {
  105. JsonElement record = value.ToString().ToObject<JsonElement>();
  106. if (record.TryGetInt64(out blobsize))
  107. {
  108. }
  109. }
  110. else
  111. {
  112. var client = _azureStorage.GetBlobContainerClient(name);
  113. var size = await client.GetBlobsCatalogSize();
  114. if (size.Item1 > 0)
  115. {
  116. await _azureRedis.GetRedisClient(8).HashSetAsync($"Blob:Record", name, size.Item1);
  117. }
  118. foreach (var key in size.Item2.Keys)
  119. {
  120. await _azureRedis.GetRedisClient(8).SortedSetRemoveAsync($"Blob:Catalog:{name}", key);
  121. await _azureRedis.GetRedisClient(8).SortedSetIncrementAsync($"Blob:Catalog:{name}", key, size.Item2[key].HasValue ? size.Item2[key].Value : 0);
  122. }
  123. usedSize = size.Item1.Value;
  124. //1024 * 1024 * 1024=1073741824 =1G
  125. surplus = (total - teach) * 1073741824 - usedSize;
  126. catalog=size.Item2;
  127. //return (usedSize, teach, total, surplus, catalog = size.Item2);
  128. }
  129. SortedSetEntry[] Scores = _azureRedis.GetRedisClient(8).SortedSetRangeByScoreWithScores($"Blob:Catalog:{name}");
  130. if (Scores != null)
  131. {
  132. foreach (var score in Scores)
  133. {
  134. double val = score.Score;
  135. string key = score.Element.ToString();
  136. catalog.Add(key, val);
  137. }
  138. usedSize = blobsize;
  139. //1024 * 1024 * 1024=1073741824 =1G
  140. surplus = (total - teach) * 1073741824 - usedSize;
  141. //return (usedSize, teach, total, surplus, catalog);
  142. }
  143. else
  144. {
  145. //没有缓存
  146. var client = _azureStorage.GetBlobContainerClient(name);
  147. var size = await client.GetBlobsCatalogSize();
  148. if (size.Item1 > 0)
  149. {
  150. await _azureRedis.GetRedisClient(8).HashSetAsync($"Blob:Record", name, size.Item1);
  151. }
  152. foreach (var key in size.Item2.Keys)
  153. {
  154. await _azureRedis.GetRedisClient(8).SortedSetRemoveAsync($"Blob:Catalog:{name}", key);
  155. await _azureRedis.GetRedisClient(8).SortedSetIncrementAsync($"Blob:Catalog:{name}", key, size.Item2[key].HasValue ? size.Item2[key].Value : 0);
  156. }
  157. usedSize = size.Item1.Value;
  158. //1024 * 1024 * 1024=1073741824 =1G
  159. //surplus 可能为负数
  160. surplus = (total - teach) * 1073741824 - usedSize;
  161. catalog = size.Item2;
  162. //return (usedSize, teach, total, surplus, catalog = size.Item2);
  163. }
  164. }
  165. }
  166. catch (Exception ex)
  167. {
  168. await _dingDing.SendBotMsg($"IES5,{_option.Location},blob/used-space()\n{ex.Message}\n{ex.StackTrace}", GroupNames.醍摩豆服務運維群組);
  169. return (usedSize, teach, total, surplus, catalog);
  170. }
  171. if (usedSize > 0 && total > 0) {
  172. double percent= surplus / total * 1073741824 *100;
  173. int tag =11;
  174. if (percent <= 10 && percent > 5)
  175. {
  176. tag = 10;
  177. }
  178. else if (percent <= 5 && percent > 0)
  179. {
  180. tag = 5;
  181. }
  182. else if (percent <= 0)
  183. {
  184. tag = 0;
  185. }
  186. //如果已经扩容请忽略此通知!
  187. string key = scope.Equals("school",StringComparison.OrdinalIgnoreCase)? $"Blob:Space:School:Notify:{name}" : "Blob:Space:Private:Notify";
  188. //string filed=
  189. if (tag < 11)
  190. {
  191. BlobSpaceNotify? blobSpaceNotify = null;
  192. RedisValue value = _azureRedis.GetRedisClient(8).HashGet(key, name);
  193. if (value != default && !value.IsNullOrEmpty)
  194. {
  195. JsonElement record = value.ToString().ToObject<JsonElement>();
  196. }
  197. blobSpaceNotify = new BlobSpaceNotify { };
  198. await _azureRedis.GetRedisClient(8).HashSetAsync(key, name, blobSpaceNotify.ToJsonString());
  199. }
  200. else {
  201. }
  202. }
  203. return (usedSize, teach, total, surplus, catalog);
  204. }
  205. /// <summary>
  206. /// 记录在redis,使用ttl 进行过期管理
  207. /// </summary>
  208. public class BlobSpaceNotify
  209. {
  210. /// <summary>
  211. /// id=$"{tmdid}-{tag}"
  212. /// </summary>
  213. public string? id { get; set; }
  214. public int tag { get; set; }
  215. public string? containerName { get; set; }
  216. public string? scope { get; set; }
  217. public string? notifyIndex { get; set; }
  218. }
  219. /// <param name="scope">school or private</param>
  220. /// <param name="containerName">school code or tmid</param>
  221. /// <param name="type">doc, res, item... Empty if no need.</param>
  222. /// <param name="periodId">Only for school</param>
  223. public static async Task<List<BlobCount>> BloblogCount(CosmosClient clientc, string scope, string containerName, string type, string periodId)
  224. {
  225. List<BlobCount> bloblogcnt = new List<BlobCount>();
  226. try
  227. {
  228. //必須項檢查
  229. if(scope.Equals("school") && string.IsNullOrWhiteSpace(periodId))
  230. {
  231. return bloblogcnt;
  232. }
  233. //資料取得
  234. var queryslt = new StringBuilder();
  235. queryslt.Append($"SELECT COUNT(1) AS count, c.type FROM c ");
  236. if (scope.Equals("school"))
  237. {
  238. queryslt.Append($"JOIN A1 IN c.periodId WHERE A1 IN ('{periodId}') ");
  239. if(!string.IsNullOrWhiteSpace(type)) queryslt.Append($"AND c.type='{type}'");
  240. queryslt.Append("GROUP BY c.type");
  241. await foreach (var item in clientc.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator<BlobCount>(queryText: queryslt.ToString(), requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Bloblog-{containerName}") }))
  242. {
  243. bloblogcnt.Add(item);
  244. }
  245. }
  246. else if (scope.Equals("private"))
  247. {
  248. if (!string.IsNullOrWhiteSpace(type)) queryslt.Append($"AND c.type='{type}'");
  249. queryslt.Append("GROUP BY c.type");
  250. await foreach (var item in clientc.GetContainer(Constant.TEAMModelOS, "Teacher").GetItemQueryIterator<BlobCount>(queryText: queryslt.ToString(), requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Bloblog-{containerName}") }))
  251. {
  252. bloblogcnt.Add(item);
  253. }
  254. }
  255. return bloblogcnt;
  256. }
  257. catch (Exception ex)
  258. {
  259. return bloblogcnt;
  260. }
  261. }
  262. public class UsedBlob
  263. {
  264. public long teach { get; set; }
  265. public long? size { get; set; }
  266. public Dictionary<string, double?> catalog { get; set; }
  267. }
  268. public class BlobCount
  269. {
  270. public string type { get; set; }
  271. public int count { get; set; }
  272. }
  273. }
  274. public class BlobRefreshMessage{
  275. public string id { get; set; }= Guid.NewGuid().ToString();
  276. public string name { get; set; }
  277. public string progress { get; set; }
  278. public string root { get; set; }
  279. }
  280. }