IESTimerTrigger.cs 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339
  1. using Azure.Cosmos;
  2. using Azure.Storage.Blobs.Models;
  3. using DinkToPdf;
  4. using DinkToPdf.Contracts;
  5. using HTEXLib.COMM.Helpers;
  6. using Microsoft.Azure.Cosmos.Table;
  7. using Microsoft.Azure.Functions.Worker;
  8. using Microsoft.Azure.Functions.Worker.Http;
  9. using Microsoft.Extensions.Logging;
  10. using StackExchange.Redis;
  11. using System;
  12. using System.Collections.Generic;
  13. using System.IO;
  14. using System.Linq;
  15. using System.Net;
  16. using System.Reflection;
  17. using System.Text;
  18. using System.Text.Json;
  19. using System.Threading.Tasks;
  20. using TEAMModelOS.SDK;
  21. using TEAMModelOS.SDK.DI;
  22. using TEAMModelOS.SDK.Extension;
  23. using TEAMModelOS.SDK.Models;
  24. using TEAMModelOS.SDK.Models.Cosmos.Teacher;
  25. using TEAMModelOS.SDK.Models.Service.BI;
  26. using TEAMModelOS.SDK.Models.Table;
  27. using static TEAMModelOS.SDK.Models.Teacher;
  28. namespace TEAMModelOS.FunctionV4.TimeTrigger
  29. {
  30. public class IESTimerTrigger
  31. {
  32. /// <summary>
  33. /// 文档。https://docs.microsoft.com/zh-cn/azure/azure-functions/functions-bindings-timer?tabs=in-process&pivots=programming-language-csharp
  34. /// Timer 在线测试 https://ncrontab.swimburger.net/
  35. /// </summary>
  36. private readonly AzureCosmosFactory _azureCosmos;
  37. private readonly DingDing _dingDing;
  38. private readonly AzureStorageFactory _azureStorage;
  39. private readonly AzureRedisFactory _azureRedis;
  40. private readonly IConverter _converter;
  41. private readonly SnowflakeId _snowflakeId;
  42. public IESTimerTrigger(SnowflakeId snowflakeId,IConverter converter, AzureCosmosFactory azureCosmos, DingDing dingDing, AzureStorageFactory azureStorage, AzureRedisFactory azureRedis)
  43. {
  44. _azureCosmos = azureCosmos;
  45. _dingDing = dingDing;
  46. _azureStorage = azureStorage;
  47. _azureRedis = azureRedis;
  48. _converter = converter;
  49. _snowflakeId=snowflakeId;
  50. }
  51. /// <summary>
  52. /// 防火墙日志记录文件
  53. /// </summary>
  54. /// <param name="req"></param>
  55. /// <param name="log"></param>
  56. /// <returns></returns>
  57. [Function("FireWallFileLog")]
  58. //https://docs.azure.cn/zh-cn/azure-functions/functions-bindings-timer?tabs=in-process&pivots=programming-language-csharp
  59. //0 1 * * * * 一天中每小时的第 1 分钟
  60. //0 */1 * * * * 每五分钟一次
  61. public async Task FireWallFileLog([TimerTrigger("0 1 * * * *")] TimerInfo myTimer, ILogger log)
  62. {
  63. try
  64. {
  65. string location = Environment.GetEnvironmentVariable("Option:Location");
  66. var datetime = DateTimeOffset.UtcNow.AddHours(-1);
  67. var y = datetime.Year;
  68. var m = datetime.Month >= 10 ? $"{datetime.Month}" : $"0{datetime.Month}";
  69. var d = datetime.Day >= 10 ? $"{datetime.Day}" : $"0{datetime.Day}";
  70. var h = datetime.Hour >= 10 ? $"{datetime.Hour}" : $"0{datetime.Hour}";
  71. if (location.Equals("China"))
  72. {
  73. string path = $"resourceId=/SUBSCRIPTIONS/73B7F9EF-D8B7-4444-9E8D-D80B43BF3CD4/RESOURCEGROUPS/TEAMMODELCHENGDU/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/OSFIREWARE/y={y}/m={m}/d={d}/h={h}/m=00/PT1H.json";
  74. var retn = await BILogAnalyseService.GetPathAnalyse(_azureStorage, path, "LogStorage");
  75. if (retn.recCnts.IsNotEmpty())
  76. {
  77. var topApi = retn.recCnts.SelectMany(x => x.apiCnt).OrderByDescending(o => o.count).Take(5).Select(x => new { x.api, x.ip, x.count });
  78. var topIp = retn.recCnts.SelectMany(x => x.ipCnt).OrderByDescending(o => o.count).Take(5).Select(x => new { x.api, x.ip, x.count });
  79. //https://teammodelos.blob.core.chinacloudapi.cn/0-public/pie-borderRadius.html
  80. await _dingDing.SendBotMarkdown("防火墙日志记录", $"#### 防火墙日志记录\n> 记录时间:{datetime.AddHours(8).ToString("yyyy-MM-dd HH")}\n> ![screenshot]()\n> ###### 发布时间:{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")} [地址]({retn.saveUrls.First()}) \n", GroupNames.成都开发測試群組);
  81. }
  82. }
  83. else if (location.Contains("Global"))
  84. {
  85. }
  86. }
  87. catch (Exception ex)
  88. {
  89. // await _dingDing.SendBotMsg($"FireWallFileLog 防火墙日志记录: {DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}\n{ex.Message}\n{ex.StackTrace}", GroupNames.成都开发測試群組);
  90. }
  91. }
  92. /// <summary>
  93. /// 清理HiTeach教研类型的课例文件上传在Blob空间的文件。 0 */10 22-23 * * *
  94. /// 该代码执行之后 需要删除或者取消Timer任务。
  95. /// </summary>
  96. /// <param name="myTimer"></param>
  97. /// <param name="log"></param>
  98. /// <returns></returns>
  99. [Function("CleanUnusedLessonRecord")]
  100. //5分钟一次
  101. public async Task CleanUnusedLessonRecord([TimerTrigger("0 */10 15-23 * * *")] TimerInfo myTimer, ILogger log) {
  102. string location = Environment.GetEnvironmentVariable("Option:Location");
  103. try {
  104. if (location.Equals("China") || location.Equals("Global"))
  105. {
  106. bool lockKey = await _azureRedis.GetRedisClient(8).KeyExistsAsync($"LessonRecord:Unused:Lock:{location}");
  107. //bool keyLock = await _azureRedis.GetRedisClient(8).KeyExistsAsync($"LessonRecord:Unused:Lock:Lock");
  108. //if (keyLock) {
  109. // return;
  110. //}
  111. //key不存在的时候 。
  112. if (!lockKey)
  113. {
  114. var schoolKeys = new List<IdCodeCount>();
  115. var teacherKeys = new List<IdCodeCount>();
  116. string sqlT = "select c.id,c.name, 'Teacher' as code from c where c.code='Base'";
  117. string sqlS = "select c.id,c.name, 'School' as code from c where c.code='Base'";
  118. await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).
  119. GetItemQueryIterator<IdCodeCount>(queryText: sqlT, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey("Base") }))
  120. {
  121. teacherKeys.Add(item);
  122. }
  123. await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).
  124. GetItemQueryIterator<IdCodeCount>(queryText: sqlS, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey("Base") }))
  125. {
  126. schoolKeys.Add(item);
  127. }
  128. string sqlcount = "select value count(1) from c where c.pk='LessonRecord'";
  129. foreach (var key in schoolKeys)
  130. {
  131. await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).
  132. GetItemQueryIterator<int>(queryText: sqlcount, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey($"LessonRecord-{key.id}") }))
  133. {
  134. key.count = item;
  135. break;
  136. }
  137. }
  138. foreach (var key in teacherKeys)
  139. {
  140. await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).
  141. GetItemQueryIterator<int>(queryText: $"select value count(1) from c where c.pk='LessonRecord' and c.tmdid='{key.id}'",
  142. requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey($"LessonRecord") }))
  143. {
  144. key.count = item;
  145. break;
  146. }
  147. }
  148. schoolKeys.AddRange(teacherKeys);
  149. //以下代码作用,用于根据当前拥有的课例数判断区分学校和个人使用课例频率越多的,
  150. //并均分每次需要处理的学校和个人容器个数,避免每次处理批次花费时间相差太大。
  151. //将每个批次处理数量级控制在500-1000左右。
  152. List<IEnumerable<IdCodeCount>> counts = new List<IEnumerable<IdCodeCount>>();
  153. //501-∞
  154. var count501_ = schoolKeys.Where(x => x.count >= 501); //至少500条
  155. counts.AddRange(count501_.Page(1));//1个学校或个人
  156. //201-500
  157. var count201_500 = schoolKeys.Where(x => x.count >= 201 && x.count <= 500); //至少603条-1500条
  158. counts.AddRange(count201_500.Page(3));//2个学校或个人
  159. //100-200
  160. var count100_200 = schoolKeys.Where(x => x.count >= 100 && x.count <= 200);//至少500条-1000条
  161. counts.AddRange(count100_200.Page(10));//5个学校或个人
  162. //51-99
  163. var count51_99 = schoolKeys.Where(x => x.count >= 51 && x.count <= 99);//至少500条-990条
  164. counts.AddRange(count51_99.Page(10));//10个学校或个人
  165. //21-50
  166. var count21_50 = schoolKeys.Where(x => x.count >= 21 && x.count <= 50);//至少410条-1000条
  167. counts.AddRange(count21_50.Page(20));//20个学校或个人
  168. //10-20
  169. var count10_20 = schoolKeys.Where(x => x.count >= 10 && x.count <= 20);//至少500条-1000条
  170. counts.AddRange(count10_20.Page(50));//50个学校或个人
  171. //5-9
  172. var count0_9 = schoolKeys.Where(x => x.count >= 5 && x.count <= 9);//至少500条-900条
  173. counts.AddRange(count0_9.Page(100));//100个学校或个人
  174. //2-4
  175. var count2_4 = schoolKeys.Where(x => x.count >= 2 && x.count <= 4);//至少600条-1200条
  176. counts.AddRange(count2_4.Page(300));
  177. //0-1,已经处理过一次,不用再处理
  178. var count0_1 = schoolKeys.Where(x => x.count ==1);//至少500,就算没有课例也算一次。
  179. counts.AddRange(count0_1.Page(1000));
  180. int field = 1;
  181. foreach (var item in counts)
  182. {
  183. if (item.Any())
  184. {
  185. bool stuallstatus = await _azureRedis.GetRedisClient(8).HashSetAsync($"LessonRecord:Unused:Lock:{location}", field,
  186. new UnusedLock { field = field, status = 0, item = item }.ToJsonString());
  187. field += 1;
  188. }
  189. }
  190. await _dingDing.SendBotMsg($"{location},获取到:{schoolKeys.Count} 个学校和个人的容器\n" +
  191. $"大概要处理:{schoolKeys.Sum(x => x.count) + schoolKeys.Where(x => x.count >= 0).Count()} 条数据\n" +
  192. $"将数据分为:{field - 1} 次处理,每次处理500-1000条数据。", GroupNames.醍摩豆服務運維群組);
  193. }
  194. //key存在的时候 开始进行数据处理
  195. var records = await _azureRedis.GetRedisClient(8).HashGetAllAsync($"LessonRecord:Unused:Lock:{location}");
  196. List<UnusedLock> unuseds = new List<UnusedLock>();
  197. foreach (var rcd in records)
  198. {
  199. var value = rcd.Value.ToString().ToObject<UnusedLock>();
  200. unuseds.Add(value);
  201. }
  202. int max = unuseds.Max(x => x.field);
  203. int index = max + 1;
  204. var unusedLock = unuseds.Where(s => s.status == 0)?.OrderBy(x => x.field)?.First();
  205. if (unusedLock != null)
  206. {
  207. var stime = DateTimeOffset.UtcNow;
  208. long s = stime.ToUnixTimeMilliseconds();
  209. string sdata = stime.ToString("yyyy-MM-dd HH:mm:ss");
  210. await _dingDing.SendBotMsg($"{location}-开始第{unusedLock.field}轮清理冗余的课例记录文件...\n开始时间:{sdata}\n清理容器有{unusedLock.item.Select(x => $"{x.name}-{x.id}").ToJsonString()}", GroupNames.醍摩豆服務運維群組);
  211. //将当前的标记为1锁定。
  212. bool stuallstatus = await _azureRedis.GetRedisClient(8).HashSetAsync($"LessonRecord:Unused:Lock:{location}", unusedLock.field,
  213. new UnusedLock { field = unusedLock.field, status = 1, item = unusedLock.item }.ToJsonString());
  214. List<KeyValuePair<string, List<string>>> deleteUrls = new List<KeyValuePair<string, List<string>>>();
  215. foreach (IdCodeCount idCode in unusedLock.item)
  216. {
  217. try
  218. {
  219. List<string> urls = new List<string>();
  220. var ContainerClient = _azureStorage.GetBlobContainerClient(idCode.id);
  221. List<string> items = await ContainerClient.List("records");
  222. if (items.IsNotEmpty())
  223. {
  224. HashSet<string> set = new HashSet<string>();
  225. items.ForEach(z => {
  226. var uri = z.Split("/");
  227. if (uri.Length > 1)
  228. {
  229. set.Add(uri[1]);
  230. }
  231. });
  232. if (set.Any())
  233. {
  234. string tbname = idCode.code.Equals("Teacher") ? Constant.Teacher : Constant.School;
  235. string pk = idCode.code.Equals("Teacher") ? "LessonRecord" : $"LessonRecord-{idCode.id}";
  236. string sql = $"select value c.id from c where c.pk='LessonRecord' and c.id in ({string.Join(",", set.Select(m => $"'{m}'"))})";
  237. List<string> ids = new List<string>();
  238. await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, tbname)
  239. .GetItemQueryIterator<string>(queryText: sql, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey(pk) }))
  240. {
  241. ids.Add(item);
  242. }
  243. var notin = set.Except(ids);
  244. if (notin.Any())
  245. {
  246. foreach (var not in notin)
  247. {
  248. string url = $"records/{not}";
  249. long id = -1;
  250. long.TryParse(not,out id);
  251. if (id > -1) {
  252. id = _snowflakeId.ParseIdToTimeStamp(id);
  253. }
  254. if (id > -1)
  255. {
  256. //72小时之外的数据。需要被删除 72 * 60 * 60 *1000= 259200000
  257. //当前时间-课例产生的时间戳
  258. if (s - id> 259200000) {
  259. urls.Add(url);
  260. await _azureStorage.GetBlobServiceClient().DeleteBlobs(_dingDing, idCode.id, new List<string> { url });
  261. }
  262. }
  263. else
  264. {
  265. urls.Add(url);
  266. await _azureStorage.GetBlobServiceClient().DeleteBlobs(_dingDing, idCode.id, new List<string> { url });
  267. }
  268. }
  269. }
  270. }
  271. }
  272. if (urls.IsNotEmpty()) {
  273. deleteUrls.Add(new KeyValuePair<string, List<string>>($"{idCode.name}-{idCode.id}", urls));
  274. }
  275. }
  276. catch (Exception ex)
  277. {
  278. //异常的学校,需要将数据放回redis 重新处理。
  279. await _azureRedis.GetRedisClient(8).HashSetAsync($"LessonRecord:Unused:Lock:{location}", index,
  280. new UnusedLock { field = index, status = 0, item = new List<IdCodeCount>() { idCode } }.ToJsonString());
  281. index += 1;
  282. await _dingDing.SendBotMsg($"{location}-第{unusedLock.field}轮清理时容器:{idCode.id},类型:{idCode.code}出现异常。\n将{idCode.id}重新加入队列,队列编号:{index}\n" +
  283. $"异常信息:{ex.Message},{ex.StackTrace}", GroupNames.醍摩豆服務運維群組);
  284. }
  285. }
  286. await _azureRedis.GetRedisClient(8).HashDeleteAsync($"LessonRecord:Unused:Lock:{location}", unusedLock.field);
  287. var etime = DateTimeOffset.UtcNow;
  288. long e = etime.ToUnixTimeMilliseconds();
  289. string edata = etime.ToString("yyyy-MM-dd HH:mm:ss");
  290. long d = (e - s) / 1000;
  291. string timeStr = d >= 60 ? Math.Round(d / 60.0) + "分" : d + "秒";
  292. List<string> content = new List<string>();
  293. foreach (var url in deleteUrls)
  294. {
  295. if (url.Value.IsNotEmpty()) {
  296. content.Add($"{url.Key}\n {string.Join(" \t\n", url.Value)}");
  297. }
  298. }
  299. if (content.IsNotEmpty())
  300. {
  301. string str = string.Join("\n", content);
  302. await _dingDing.SendBotMsg($"{location}-结束第{unusedLock.field}轮清理冗余的课例记录文件...\n结束时间:{edata}\n耗时:{timeStr}\n清理内容:\n{str}", GroupNames.醍摩豆服務運維群組);
  303. }
  304. else {
  305. await _dingDing.SendBotMsg($"{location}-结束第{unusedLock.field}轮清理冗余的课例记录文件...\n结束时间:{edata}\n耗时:{timeStr}\n清理内容:暂无", GroupNames.醍摩豆服務運維群組);
  306. }
  307. }
  308. }
  309. } catch (Exception ex) {
  310. await _dingDing.SendBotMsg($"{location} 清理时容器出现异常。\n异常信息:{ex.Message},{ex.StackTrace}", GroupNames.醍摩豆服務運維群組);
  311. }
  312. }
  313. public class UnusedLock {
  314. public int field { get; set; }
  315. /// <summary>
  316. /// 0默认未被执行,1 正在执行中
  317. /// </summary>
  318. public int status { get; set; }
  319. public IEnumerable<IdCodeCount> item { get; set; } = new List<IdCodeCount>();
  320. }
  321. public class IdCodeCount {
  322. public string id { get; set; }
  323. public string name { get; set; }
  324. public string code { get; set; }
  325. public int count { get; set; }
  326. }
  327. }
  328. }