|
@@ -90,134 +90,202 @@ namespace TEAMModelOS.FunctionV4.TimeTrigger
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
/// <summary>
|
|
- ///
|
|
|
|
|
|
+ /// 该代码执行之后 需要删除或者取消Timer任务。
|
|
/// </summary>
|
|
/// </summary>
|
|
/// <param name="myTimer"></param>
|
|
/// <param name="myTimer"></param>
|
|
/// <param name="log"></param>
|
|
/// <param name="log"></param>
|
|
/// <returns></returns>
|
|
/// <returns></returns>
|
|
[Function("CleanUnusedLessonRecord")]
|
|
[Function("CleanUnusedLessonRecord")]
|
|
- public async Task CleanUnusedLessonRecord([TimerTrigger("0 1 * * * *")] TimerInfo myTimer, ILogger log) {
|
|
|
|
|
|
+ //5分钟一次
|
|
|
|
+ public async Task CleanUnusedLessonRecord([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, ILogger log) {
|
|
string location = Environment.GetEnvironmentVariable("Option:Location");
|
|
string location = Environment.GetEnvironmentVariable("Option:Location");
|
|
-
|
|
|
|
- if (location.Equals("China")|| location.Equals("Global"))
|
|
|
|
- {
|
|
|
|
- bool lockKey = await _azureRedis.GetRedisClient(8).KeyExistsAsync($"LessonRecord:Unused:Lock:{location}");
|
|
|
|
- //key不存在的时候 。
|
|
|
|
- if (!lockKey)
|
|
|
|
|
|
+ try {
|
|
|
|
+ if (location.Equals("China") || location.Equals("Global"))
|
|
{
|
|
{
|
|
- var schoolKeys = new List<IdCodeCount>();
|
|
|
|
- var teacherKeys = new List<IdCodeCount>();
|
|
|
|
- string sql = "select value c.id from c where c.code='Base'";
|
|
|
|
- await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).
|
|
|
|
- GetItemQueryIterator<string>(queryText: sql, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey("Base") }))
|
|
|
|
- {
|
|
|
|
- teacherKeys.Add(new IdCodeCount { id = item, code = "Teacher" });
|
|
|
|
- }
|
|
|
|
- await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).
|
|
|
|
- GetItemQueryIterator<string>(queryText: sql, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey("Base") }))
|
|
|
|
- {
|
|
|
|
- schoolKeys.Add(new IdCodeCount { id = item, code = "School" });
|
|
|
|
|
|
+ bool lockKey = await _azureRedis.GetRedisClient(8).KeyExistsAsync($"LessonRecord:Unused:Lock:{location}");
|
|
|
|
+ bool keyLock = await _azureRedis.GetRedisClient(8).KeyExistsAsync($"LessonRecord:Unused:Lock:Lock");
|
|
|
|
+ if (keyLock) {
|
|
|
|
+ return;
|
|
}
|
|
}
|
|
- string sqlcount = "select value count(1) from c where c.pk='LessonRecord'";
|
|
|
|
- foreach (var key in schoolKeys)
|
|
|
|
|
|
+ //key不存在的时候 。
|
|
|
|
+ if (!lockKey)
|
|
{
|
|
{
|
|
|
|
+ var schoolKeys = new List<IdCodeCount>();
|
|
|
|
+ var teacherKeys = new List<IdCodeCount>();
|
|
|
|
+ string sqlT = "select c.id,c.name, 'Teacher' as code from c where c.code='Base'";
|
|
|
|
+ string sqlS = "select c.id,c.name, 'School' as code from c where c.code='Base'";
|
|
|
|
+ await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).
|
|
|
|
+ GetItemQueryIterator<IdCodeCount>(queryText: sqlT, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey("Base") }))
|
|
|
|
+ {
|
|
|
|
+ teacherKeys.Add(item);
|
|
|
|
+ }
|
|
await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).
|
|
await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).
|
|
- GetItemQueryIterator<int>(queryText: sqlcount, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey($"LessonRecord-{key.id}") }))
|
|
|
|
|
|
+ GetItemQueryIterator<IdCodeCount>(queryText: sqlS, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey("Base") }))
|
|
{
|
|
{
|
|
- key.count = item;
|
|
|
|
- break;
|
|
|
|
|
|
+ schoolKeys.Add(item);
|
|
}
|
|
}
|
|
- }
|
|
|
|
- foreach (var key in teacherKeys)
|
|
|
|
- {
|
|
|
|
- await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).
|
|
|
|
- GetItemQueryIterator<int>(queryText: $"select value count(1) from c where c.pk='LessonRecord' and c.tmdid='{key.id}'",
|
|
|
|
- requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey($"LessonRecord") }))
|
|
|
|
|
|
+ string sqlcount = "select value count(1) from c where c.pk='LessonRecord'";
|
|
|
|
+ foreach (var key in schoolKeys)
|
|
{
|
|
{
|
|
- key.count = item;
|
|
|
|
- break;
|
|
|
|
|
|
+ await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.School).
|
|
|
|
+ GetItemQueryIterator<int>(queryText: sqlcount, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey($"LessonRecord-{key.id}") }))
|
|
|
|
+ {
|
|
|
|
+ key.count = item;
|
|
|
|
+ break;
|
|
|
|
+ }
|
|
}
|
|
}
|
|
- }
|
|
|
|
- schoolKeys.AddRange(teacherKeys);
|
|
|
|
- //以下代码作用,用于根据当前拥有的课例数判断区分学校和个人使用课例频率越多的,
|
|
|
|
- //并均分每次需要处理的学校和个人容器个数,避免每次处理批次花费时间相差太大。
|
|
|
|
- //将每个批次处理数量级控制在500-1000左右。
|
|
|
|
- List<IEnumerable<IdCodeCount>> counts = new List<IEnumerable<IdCodeCount>>();
|
|
|
|
- //501-∞
|
|
|
|
- var count501_ = schoolKeys.Where(x => x.count >= 501); //至少500条
|
|
|
|
- counts.AddRange(count501_.Page(1));//1个学校或个人
|
|
|
|
- //201-500
|
|
|
|
- var count201_500 = schoolKeys.Where(x => x.count >= 201 && x.count <= 500); //至少402条-1000条
|
|
|
|
- counts.AddRange(count201_500.Page(2));//2个学校或个人
|
|
|
|
- //100-200
|
|
|
|
- var count100_200 = schoolKeys.Where(x => x.count >= 100 && x.count <= 200);//至少500条-1000条
|
|
|
|
- counts.AddRange(count100_200.Page(5));//5个学校或个人
|
|
|
|
- //51-99
|
|
|
|
- var count51_99 = schoolKeys.Where(x => x.count >= 51 && x.count <= 99);//至少500条-990条
|
|
|
|
- counts.AddRange(count51_99.Page(10));//10个学校或个人
|
|
|
|
- //21-50
|
|
|
|
- var count21_50 = schoolKeys.Where(x => x.count >= 21 && x.count <= 50);//至少410条-1000条
|
|
|
|
- counts.AddRange(count21_50.Page(20));//20个学校或个人
|
|
|
|
- //10-20
|
|
|
|
- var count10_20 = schoolKeys.Where(x => x.count >= 10 && x.count <= 20);//至少500条-1000条
|
|
|
|
- counts.AddRange(count10_20.Page(50));//50个学校或个人
|
|
|
|
- //5-9
|
|
|
|
- var count0_9 = schoolKeys.Where(x => x.count >= 5 && x.count <= 9);//至少500条-900条
|
|
|
|
- counts.AddRange(count0_9.Page(100));//100个学校或个人
|
|
|
|
- //2-4
|
|
|
|
- var count2_4 = schoolKeys.Where(x => x.count >= 2 && x.count <= 4);//至少500条-1000条
|
|
|
|
- counts.AddRange(count2_4.Page(250));
|
|
|
|
- //0-1
|
|
|
|
- var count0_1 = schoolKeys.Where(x => x.count >= 0 && x.count <= 1);//至少500,就算没有课例也算一次。
|
|
|
|
- counts.AddRange(count0_1.Page(500));
|
|
|
|
- int field = 1;
|
|
|
|
- foreach (var item in counts)
|
|
|
|
- {
|
|
|
|
- if (item.Any())
|
|
|
|
|
|
+ foreach (var key in teacherKeys)
|
|
{
|
|
{
|
|
- bool stuallstatus = await _azureRedis.GetRedisClient(8).HashSetAsync($"LessonRecord:Unused:Lock:{location}", field,
|
|
|
|
- new UnusedLock { field= field, status = 0, item= item }.ToJsonString());
|
|
|
|
- field += 1;
|
|
|
|
|
|
+ await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, Constant.Teacher).
|
|
|
|
+ GetItemQueryIterator<int>(queryText: $"select value count(1) from c where c.pk='LessonRecord' and c.tmdid='{key.id}'",
|
|
|
|
+ requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey($"LessonRecord") }))
|
|
|
|
+ {
|
|
|
|
+ key.count = item;
|
|
|
|
+ break;
|
|
|
|
+ }
|
|
}
|
|
}
|
|
-
|
|
|
|
|
|
+ schoolKeys.AddRange(teacherKeys);
|
|
|
|
+ //以下代码作用,用于根据当前拥有的课例数判断区分学校和个人使用课例频率越多的,
|
|
|
|
+ //并均分每次需要处理的学校和个人容器个数,避免每次处理批次花费时间相差太大。
|
|
|
|
+ //将每个批次处理数量级控制在500-1000左右。
|
|
|
|
+ List<IEnumerable<IdCodeCount>> counts = new List<IEnumerable<IdCodeCount>>();
|
|
|
|
+ //501-∞
|
|
|
|
+ var count501_ = schoolKeys.Where(x => x.count >= 501); //至少500条
|
|
|
|
+ counts.AddRange(count501_.Page(1));//1个学校或个人
|
|
|
|
+ //201-500
|
|
|
|
+ var count201_500 = schoolKeys.Where(x => x.count >= 201 && x.count <= 500); //至少402条-1000条
|
|
|
|
+ counts.AddRange(count201_500.Page(2));//2个学校或个人
|
|
|
|
+ //100-200
|
|
|
|
+ var count100_200 = schoolKeys.Where(x => x.count >= 100 && x.count <= 200);//至少500条-1000条
|
|
|
|
+ counts.AddRange(count100_200.Page(5));//5个学校或个人
|
|
|
|
+ //51-99
|
|
|
|
+ var count51_99 = schoolKeys.Where(x => x.count >= 51 && x.count <= 99);//至少500条-990条
|
|
|
|
+ counts.AddRange(count51_99.Page(10));//10个学校或个人
|
|
|
|
+ //21-50
|
|
|
|
+ var count21_50 = schoolKeys.Where(x => x.count >= 21 && x.count <= 50);//至少410条-1000条
|
|
|
|
+ counts.AddRange(count21_50.Page(20));//20个学校或个人
|
|
|
|
+ //10-20
|
|
|
|
+ var count10_20 = schoolKeys.Where(x => x.count >= 10 && x.count <= 20);//至少500条-1000条
|
|
|
|
+ counts.AddRange(count10_20.Page(50));//50个学校或个人
|
|
|
|
+ //5-9
|
|
|
|
+ var count0_9 = schoolKeys.Where(x => x.count >= 5 && x.count <= 9);//至少500条-900条
|
|
|
|
+ counts.AddRange(count0_9.Page(100));//100个学校或个人
|
|
|
|
+ //2-4
|
|
|
|
+ var count2_4 = schoolKeys.Where(x => x.count >= 2 && x.count <= 4);//至少500条-1000条
|
|
|
|
+ counts.AddRange(count2_4.Page(250));
|
|
|
|
+ //0-1
|
|
|
|
+ var count0_1 = schoolKeys.Where(x => x.count >= 0 && x.count <= 1);//至少500,就算没有课例也算一次。
|
|
|
|
+ counts.AddRange(count0_1.Page(500));
|
|
|
|
+ int field = 1;
|
|
|
|
+ foreach (var item in counts)
|
|
|
|
+ {
|
|
|
|
+ if (item.Any())
|
|
|
|
+ {
|
|
|
|
+ bool stuallstatus = await _azureRedis.GetRedisClient(8).HashSetAsync($"LessonRecord:Unused:Lock:{location}", field,
|
|
|
|
+ new UnusedLock { field = field, status = 0, item = item }.ToJsonString());
|
|
|
|
+ field += 1;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ }
|
|
|
|
+ await _dingDing.SendBotMsg($"{location},获取到:{schoolKeys.Count} 个学校和个人的容器\n" +
|
|
|
|
+ $"大概要处理:{schoolKeys.Sum(x => x.count) + schoolKeys.Where(x => x.count >= 0).Count()} 条数据\n" +
|
|
|
|
+ $"将数据分为:{field - 1} 次处理,每次处理500-1000条数据。", GroupNames.醍摩豆服務運維群組);
|
|
}
|
|
}
|
|
- await _dingDing.SendBotMsg($"{location},获取到:{schoolKeys.Count} 个学校和个人的容器\n" +
|
|
|
|
- $"大概要处理:{schoolKeys.Sum(x => x.count) + schoolKeys.Where(x => x.count >= 0).Count()} 条数据\n" +
|
|
|
|
- $"将数据分为:{field - 1} 次处理,每次处理500-1000条数据。", GroupNames.醍摩豆服務運維群組);
|
|
|
|
- }
|
|
|
|
- //key存在的时候 开始进行数据处理
|
|
|
|
- var records = await _azureRedis.GetRedisClient(8).HashGetAllAsync($"LessonRecord:Unused:Lock:{location}");
|
|
|
|
- List<UnusedLock> unuseds = new List<UnusedLock>();
|
|
|
|
- foreach (var rcd in records)
|
|
|
|
- {
|
|
|
|
- var value = rcd.Value.ToString().ToObject<UnusedLock>();
|
|
|
|
- unuseds.Add(value);
|
|
|
|
- }
|
|
|
|
- var unused = unuseds.Where(s => s.status == 0).OrderBy(x => x.field).Take(1);
|
|
|
|
- if (unused.Any()) {
|
|
|
|
- var stime = DateTimeOffset.UtcNow;
|
|
|
|
- long s = stime.ToUnixTimeMilliseconds();
|
|
|
|
- string sdata = stime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
|
|
- UnusedLock unusedLock= unused.First();
|
|
|
|
- foreach (IdCodeCount idCode in unusedLock.item) {
|
|
|
|
- try {
|
|
|
|
- var ContainerClient = _azureStorage.GetBlobContainerClient($"{idCode.id}");
|
|
|
|
- List<string> items = await ContainerClient.List("records");
|
|
|
|
- if (items.IsNotEmpty()) {
|
|
|
|
- HashSet<string> set = new HashSet<string>();
|
|
|
|
- items.ForEach(z => {
|
|
|
|
- var uri = z.Split("/");
|
|
|
|
- if (uri.Length > 1)
|
|
|
|
|
|
+ //key存在的时候 开始进行数据处理
|
|
|
|
+ var records = await _azureRedis.GetRedisClient(8).HashGetAllAsync($"LessonRecord:Unused:Lock:{location}");
|
|
|
|
+ List<UnusedLock> unuseds = new List<UnusedLock>();
|
|
|
|
+ foreach (var rcd in records)
|
|
|
|
+ {
|
|
|
|
+ var value = rcd.Value.ToString().ToObject<UnusedLock>();
|
|
|
|
+ unuseds.Add(value);
|
|
|
|
+ }
|
|
|
|
+ int max = unuseds.Max(x => x.field);
|
|
|
|
+ int index = max + 1;
|
|
|
|
+ var unusedLock = unuseds.Where(s => s.status == 0)?.OrderBy(x => x.field)?.First();
|
|
|
|
+ if (unusedLock != null)
|
|
|
|
+ {
|
|
|
|
+ var stime = DateTimeOffset.UtcNow;
|
|
|
|
+ long s = stime.ToUnixTimeMilliseconds();
|
|
|
|
+ string sdata = stime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
|
|
+ await _dingDing.SendBotMsg($"{location}-开始第{unusedLock.field}轮清理冗余的课例记录文件...\n开始时间:{sdata}\n清理容器有{unusedLock.item.Select(x => $"{x.name}-{x.id}").ToJsonString()}", GroupNames.醍摩豆服務運維群組);
|
|
|
|
+ //将当前的标记为1锁定。
|
|
|
|
+ bool stuallstatus = await _azureRedis.GetRedisClient(8).HashSetAsync($"LessonRecord:Unused:Lock:{location}", unusedLock.field,
|
|
|
|
+ new UnusedLock { field = unusedLock.field, status = 1, item = unusedLock.item }.ToJsonString());
|
|
|
|
+ List<KeyValuePair<string, List<string>>> deleteUrls = new List<KeyValuePair<string, List<string>>>();
|
|
|
|
+ foreach (IdCodeCount idCode in unusedLock.item)
|
|
|
|
+ {
|
|
|
|
+ try
|
|
|
|
+ {
|
|
|
|
+ List<string> urls = new List<string>();
|
|
|
|
+ var ContainerClient = _azureStorage.GetBlobContainerClient(idCode.id);
|
|
|
|
+ List<string> items = await ContainerClient.List("records");
|
|
|
|
+ if (items.IsNotEmpty())
|
|
|
|
+ {
|
|
|
|
+ HashSet<string> set = new HashSet<string>();
|
|
|
|
+ items.ForEach(z => {
|
|
|
|
+ var uri = z.Split("/");
|
|
|
|
+ if (uri.Length > 1)
|
|
|
|
+ {
|
|
|
|
+ set.Add(uri[1]);
|
|
|
|
+ }
|
|
|
|
+ });
|
|
|
|
+ if (set.Any())
|
|
{
|
|
{
|
|
- set.Add(uri[1]);
|
|
|
|
|
|
+ string tbname = idCode.code.Equals("Teacher") ? Constant.Teacher : Constant.School;
|
|
|
|
+ string pk = idCode.code.Equals("Teacher") ? "LessonRecord" : $"LessonRecord-{idCode.id}";
|
|
|
|
+ string sql = $"select value c.id from c where c.pk='LessonRecord' and c.id in ({string.Join(",", set.Select(m => $"'{m}'"))})";
|
|
|
|
+ List<string> ids = new List<string>();
|
|
|
|
+ await foreach (var item in _azureCosmos.GetCosmosClient().GetContainer(Constant.TEAMModelOS, tbname)
|
|
|
|
+ .GetItemQueryIterator<string>(queryText: sql, requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey(pk) }))
|
|
|
|
+ {
|
|
|
|
+ ids.Add(item);
|
|
|
|
+ }
|
|
|
|
+ var notin = set.Except(ids);
|
|
|
|
+
|
|
|
|
+ if (notin.Any())
|
|
|
|
+ {
|
|
|
|
+
|
|
|
|
+ foreach (var not in notin)
|
|
|
|
+ {
|
|
|
|
+ string url = $"records/{not}";
|
|
|
|
+ urls.Add(url);
|
|
|
|
+ await _azureStorage.GetBlobServiceClient().DeleteBlobs(_dingDing,idCode.id,new List<string>{ url });
|
|
|
|
+ }
|
|
|
|
+ }
|
|
}
|
|
}
|
|
- });
|
|
|
|
|
|
+ }
|
|
|
|
+ deleteUrls.Add(new KeyValuePair<string, List<string>>($"{idCode.name}-{idCode.id}", urls));
|
|
}
|
|
}
|
|
- } catch (Exception ex) {
|
|
|
|
|
|
+ catch (Exception ex)
|
|
|
|
+ {
|
|
|
|
+ //异常的学校,需要将数据放回redis 重新处理。
|
|
|
|
+ await _azureRedis.GetRedisClient(8).HashSetAsync($"LessonRecord:Unused:Lock:{location}", index,
|
|
|
|
+ new UnusedLock { field = index, status = 0, item = new List<IdCodeCount>() { idCode } }.ToJsonString());
|
|
|
|
+ index += 1;
|
|
|
|
+ await _dingDing.SendBotMsg($"{location}-第{unusedLock.field}轮清理时容器:{idCode.id},类型:{idCode.code}出现异常。\n将{idCode.id}重新加入队列,队列编号:{index}\n" +
|
|
|
|
+ $"异常信息:{ex.Message},{ex.StackTrace}", GroupNames.醍摩豆服務運維群組);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ await _azureRedis.GetRedisClient(8).HashDeleteAsync($"LessonRecord:Unused:Lock:{location}", unusedLock.field);
|
|
|
|
+ var etime = DateTimeOffset.UtcNow;
|
|
|
|
+ long e = etime.ToUnixTimeMilliseconds();
|
|
|
|
+ string edata = etime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
|
|
+ long d = (e - s) / 1000;
|
|
|
|
+ string timeStr = d >= 60 ? Math.Round(d / 60.0) + "分" : d + "秒";
|
|
|
|
+ List<string> content = new List<string>();
|
|
|
|
+ foreach (var url in deleteUrls)
|
|
|
|
+ {
|
|
|
|
+ content.Add($"{url.Key}\n {string.Join(" \t\n", url.Value)}");
|
|
|
|
+ }
|
|
|
|
+ string str = string.Join("\n", content);
|
|
|
|
+ if (max == unusedLock.field) {
|
|
|
|
+ await _azureRedis.GetRedisClient(8).StringSetAsync($"LessonRecord:Unused:Lock:Lock", "用于在清理完所有Blob容器后,锁定不再进行多次清理。");
|
|
}
|
|
}
|
|
-
|
|
|
|
|
|
+ await _dingDing.SendBotMsg($"{location}-结束第{unusedLock.field}轮清理冗余的课例记录文件...\n结束时间:{edata}\n耗时:{timeStr}\n清理内容:\n{str}", GroupNames.醍摩豆服務運維群組);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
+ await _dingDing.SendBotMsg($"{location} 清理时容器出现异常。\n异常信息:{ex.Message},{ex.StackTrace}", GroupNames.醍摩豆服務運維群組);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
@@ -231,6 +299,7 @@ namespace TEAMModelOS.FunctionV4.TimeTrigger
|
|
}
|
|
}
|
|
public class IdCodeCount {
|
|
public class IdCodeCount {
|
|
public string id { get; set; }
|
|
public string id { get; set; }
|
|
|
|
+ public string name { get; set; }
|
|
public string code { get; set; }
|
|
public string code { get; set; }
|
|
public int count { get; set; }
|
|
public int count { get; set; }
|
|
}
|
|
}
|