using Azure; using Azure.Cosmos; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Text.Json; using System.Threading.Tasks; using TEAMModelOS.SDK.DI; using TEAMModelOS.SDK.Extension; using TEAMModelOS.SDK; using HTEXLib.COMM.Helpers; using TEAMModelOS.SDK.Models.Cosmos.Common; using System.Dynamic; using Newtonsoft.Json; using TEAMModelOS.Models; namespace TEAMModelOS.SDK.Models.Service { public static class FixDataService { /// /// 修复学生数据 /// /// /// /// /// /// public static async Task> FixStudentInfo(CosmosClient client, DingDing _dingDing, AzureStorageFactory _azureStorage, JsonElement data) { var code = data.GetProperty("code").GetString(); var ids = data.GetProperty("ids").ToObject>(); var dict = data.GetProperty("dict").ToObject>(); string queryText = $"SELECT VALUE c FROM c WHERE c.id IN ({string.Join(",", ids.Select(o => $"'{o}'"))})"; List students = new List(); await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "Student") .GetItemQueryIterator( queryText: queryText, requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Base-{code}") })) { foreach (var key in dict.Keys) { switch (key) { case "classId": item.classId = $"{ dict[key]}"; break; case "periodId": item.periodId = $"{ dict[key]}"; break; case "schoolId": item.schoolId = $"{ dict[key]}"; break; case "year": int year = DateTime.Now.Year; int.TryParse($"dict[key]", out year); item.year = year; break; default: break; } await client.GetContainer(Constant.TEAMModelOS, "Student").ReplaceItemAsync(item, item.id, new PartitionKey(item.code)); students.Add(item); } } return students; } /// /// 修复内容模块数据 /// /// /// /// /// /// public static async Task FixBlobContent(CosmosClient client, DingDing _dingDing, AzureStorageFactory _azureStorage, JsonElement data) { if (data.TryGetProperty("doPrivate", out JsonElement _doPrivate) && $"{_doPrivate}".Equals("yes", StringComparison.OrdinalIgnoreCase)) { foreach (var cnt in _azureStorage.GetBlobServiceClient().GetBlobContainers()) { if (cnt.Name.Length == 10 && int.TryParse(cnt.Name, out _)) { await doFixBlob(client, _azureStorage, cnt.Name, "private"); } } } List schools = new List(); await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: "select value(c) from c", requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey("Base") })) { schools.Add(item); } foreach (var school in schools) { await doFixBlob(client, _azureStorage, school.id, "school"); } } private static async Task doFixBlob(CosmosClient client, AzureStorageFactory _azureStorage, string name, string scope) { List prefixs = new List() { "audio", "doc", "image", "other", "res", "video" }; var ContainerClient = _azureStorage.GetBlobContainerClient($"{name}"); var tb = "Teacher"; if (scope != "private") { tb = "School"; } List ids = new List(); await foreach (var item in client.GetContainer(Constant.TEAMModelOS, tb).GetItemQueryIterator(queryDefinition: new QueryDefinition("select c.id from c "), requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Bloblog-{name}") })) { ids.Add(item.id); } await client.GetContainer(Constant.TEAMModelOS, tb).DeleteItemsStreamAsync(ids, $"Bloblog-{name}"); long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); foreach (var prefix in prefixs) { if (prefix.Equals("res")) { List itemres = await ContainerClient.List(prefix); if (itemres.IsNotEmpty()) { HashSet set = new HashSet(); itemres.ForEach(x => { var uri = x.Split("/"); set.Add($"res/{uri[1]}"); }); foreach (var item in set) { var urlsSize = await ContainerClient.GetBlobsSize(item); var url = item; if (!item.EndsWith(".hte", StringComparison.OrdinalIgnoreCase) && !item.EndsWith(".HTEX", StringComparison.OrdinalIgnoreCase)) { url += ".HTEX"; } Bloblog bloblog = new Bloblog { id = Guid.NewGuid().ToString(), code = $"Bloblog-{name}", pk = "Bloblog", time = now, url = url, size = urlsSize != null && urlsSize.HasValue ? urlsSize.Value : 0, type = prefix }; await client.GetContainer(Constant.TEAMModelOS, tb).UpsertItemAsync(bloblog, new Azure.Cosmos.PartitionKey(bloblog.code)); } } } else { List items = await ContainerClient.List(prefix); if (items.IsNotEmpty()) { foreach (var item in items) { var urlsSize = await ContainerClient.GetBlobsSize(item); Bloblog bloblog = new Bloblog { id = Guid.NewGuid().ToString(), code = $"Bloblog-{name}", pk = "Bloblog", time = now, url = item, size = urlsSize != null && urlsSize.HasValue ? urlsSize.Value : 0, type = prefix }; await client.GetContainer(Constant.TEAMModelOS, tb).UpsertItemAsync(bloblog, new Azure.Cosmos.PartitionKey(bloblog.code)); } } } } } /// /// 修復學校基本資料 /// /// /// /// public static async Task> FixSchoolPeriodId(CosmosClient client, string schoolCode) { List periodIdList = new List(); await foreach (School schinfo in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c WHERE c.id = '{schoolCode}'", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Base") })) { int periodIndex = 0; foreach (Period periodNow in schinfo.period) { if (periodNow.id.Equals("上學期") || periodNow.id.Equals("上学期") || periodNow.id.Equals("First semester")) { string periodId = Guid.NewGuid().ToString(); schinfo.period[periodIndex].id = periodId; periodIdList.Add(periodId); } periodIndex++; } await client.GetContainer(Constant.TEAMModelOS, "School").ReplaceItemAsync(schinfo, schinfo.id, new PartitionKey("Base")); } return periodIdList; } /// /// 修復學校班級資料 /// /// /// /// /// public static async Task> FixClassInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List classIdList = new List(); await foreach (Class classinfo in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Class-{schoolCode}") })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": classinfo.periodId = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "School").ReplaceItemAsync(classinfo, classinfo.id, new PartitionKey($"Class-{schoolCode}")); classIdList.Add(classinfo.id); } return classIdList; } /// /// 修復學校課程資料 /// /// /// /// /// public static async Task> FixCourseInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List courseIdList = new List(); await foreach (Course courseinfo in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Course-{schoolCode}") })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": courseinfo.period.id = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "School").ReplaceItemAsync(courseinfo, courseinfo.id, new PartitionKey($"Course-{schoolCode}")); courseIdList.Add(courseinfo.id); } return courseIdList; } /// /// 修復學校知識點資料 /// /// /// /// /// public static async Task> FixKnowledgeInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List knowledgeIdList = new List(); await foreach (Knowledge knowledgeinfo in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c WHERE c.pk = 'Knowledge' AND c.owner = '{schoolCode}'", requestOptions: new QueryRequestOptions() { })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": knowledgeinfo.periodId = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "School").ReplaceItemAsync(knowledgeinfo, knowledgeinfo.id, new PartitionKey($"{knowledgeinfo.code}")); knowledgeIdList.Add(knowledgeinfo.id); } return knowledgeIdList; } /// /// 修復學校試卷資料 /// /// /// /// /// public static async Task> FixPaperInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List paperIdList = new List(); await foreach (Paper paperinfo in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Paper-{schoolCode}") })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": paperinfo.periodId = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "School").ReplaceItemAsync(paperinfo, paperinfo.id, new PartitionKey($"Paper-{schoolCode}")); paperIdList.Add(paperinfo.id); } return paperIdList; } /// /// 修復學校課綱資料 /// /// /// /// /// public static async Task> FixVolumeInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List volumeIdList = new List(); await foreach (Volume volumeinfo in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Volume-{schoolCode}") })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": volumeinfo.periodId = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "School").ReplaceItemAsync(volumeinfo, volumeinfo.id, new PartitionKey($"Volume-{schoolCode}")); volumeIdList.Add(volumeinfo.id); } return volumeIdList; } /// /// 修復學校試題資料 /// /// /// /// /// public static async Task> FixItemInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List itemIdList = new List(); await foreach (ItemInfo iteminfo in client.GetContainer(Constant.TEAMModelOS, "School").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Item-{schoolCode}") })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": iteminfo.periodId = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "School").ReplaceItemAsync(iteminfo, iteminfo.id, new PartitionKey($"Item-{schoolCode}")); itemIdList.Add(iteminfo.id); } return itemIdList; } /// /// 修復學校評測資料 /// /// /// /// /// public static async Task> FixExamInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List examIdList = new List(); await foreach (ExamInfo examinfo in client.GetContainer(Constant.TEAMModelOS, "Common").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Exam-{schoolCode}") })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": examinfo.period.id = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "Common").ReplaceItemAsync(examinfo, examinfo.id, new PartitionKey($"Exam-{schoolCode}")); examIdList.Add(examinfo.id); } return examIdList; } /// /// 修復學生資料 /// /// /// /// /// public static async Task> FixStudentInfo(CosmosClient client, string schoolCode, Dictionary dataDic) { List studentIdList = new List(); await foreach (Student studentinfo in client.GetContainer(Constant.TEAMModelOS, "Student").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c", requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Base-{schoolCode}") })) { foreach (KeyValuePair item in dataDic) { switch (item.Key) { case "periodId": studentinfo.periodId = item.Value; break; } } await client.GetContainer(Constant.TEAMModelOS, "Student").ReplaceItemAsync(studentinfo, studentinfo.id, new PartitionKey($"Base-{schoolCode}")); studentIdList.Add(studentinfo.id); } return studentIdList; } /// /// 修復评测内容 /// /// /// /// public static async Task> FixExamPublish(CosmosClient client, DingDing _dingDing, JsonElement data, Option _option) { List infos = new List(); //var dict = data.GetProperty("publish").GetInt32(); //List<(string id, string code)> ps = new List<(string id, string code)>(); List exams = new List(); List>> tasks = new List>>(); try { await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "Common").GetItemQueryStreamIterator(queryText: $"SELECT value(c) FROM c where c.pk = 'Exam' and c.publish = '0'" )) { using var json = await JsonDocument.ParseAsync(item.ContentStream); if (json.RootElement.TryGetProperty("_count", out JsonElement count) && count.GetUInt16() > 0) { foreach (var obj in json.RootElement.GetProperty("Documents").EnumerateArray()) { if (obj.TryGetProperty("publish", out JsonElement publish)) { try { if (string.IsNullOrEmpty(publish.GetString()) || publish.GetString().Equals("0")) { ExamInfo examInfo = new ExamInfo { id = obj.GetProperty("id").GetString(), owner = obj.TryGetProperty("owner", out JsonElement owner) ? owner.GetString() : "", code = obj.TryGetProperty("code", out JsonElement code) ? code.GetString() : "", //examInfo.owner = obj.GetProperty("owner").GetString(); name = obj.GetProperty("name").GetString(), school = obj.GetProperty("school").GetString(), creatorId = obj.GetProperty("creatorId").GetString(), stuCount = obj.TryGetProperty("stuCount", out JsonElement stuCount) ? stuCount.GetInt32() : 0, createTime = obj.GetProperty("createTime").GetInt64(), updateTime = obj.TryGetProperty("updateTime", out JsonElement updateTime) ? updateTime.GetInt64() : DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), startTime = obj.TryGetProperty("startTime", out JsonElement startTime) ? startTime.GetInt64() : DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), endTime = obj.TryGetProperty("endTime", out JsonElement endTime) ? endTime.GetInt64() : DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), year = obj.TryGetProperty("year", out JsonElement year) ? year.GetInt32() : 0, source = obj.TryGetProperty("source", out JsonElement source) ? source.GetString() : "0", classes = obj.TryGetProperty("classes", out JsonElement classes) ? classes.ToObject>() : new List(), stuLists = obj.TryGetProperty("stuLists", out JsonElement stuLists) ? stuLists.ToObject>() : new List(), //examInfo.stuLists = obj.GetProperty("stuLists").ToObject>(); papers = obj.TryGetProperty("papers", out JsonElement papers) ? papers.ToObject>() : new List(), type = obj.TryGetProperty("type", out JsonElement type) ? type.GetString() : "", period = obj.TryGetProperty("period", out JsonElement period) ? period.ToObject() : new PeriodSimple(), grades = obj.TryGetProperty("grades", out JsonElement grades) ? grades.ToObject>() : new List(), subjects = obj.GetProperty("subjects").ToObject>(), progress = obj.TryGetProperty("progress", out JsonElement progress) ? progress.GetString() : "finish", scope = obj.TryGetProperty("scope", out JsonElement scope) ? scope.GetString() : "school", examType = obj.TryGetProperty("examType", out JsonElement examType) ? examType.ToObject() : new Custom(), status = obj.TryGetProperty("status", out JsonElement status) ? status.GetInt32() : 0, average = obj.TryGetProperty("average", out JsonElement average) ? average.GetDouble() : 0, sRate = obj.TryGetProperty("sRate", out JsonElement sRate) ? sRate.GetDouble() : 0, lostStu = obj.TryGetProperty("lostStu", out JsonElement lostStu) ? lostStu.ToObject>() : new List(), standard = obj.TryGetProperty("standard", out JsonElement standard) ? standard.GetDouble() : 0, size = obj.TryGetProperty("size", out JsonElement size) ? size.GetInt64() : 0, income = obj.TryGetProperty("income", out JsonElement income) ? income.GetInt32() : 0, touch = obj.TryGetProperty("touch", out JsonElement touch) ? touch.GetInt32() : 0, publish = 0, groupLists = obj.TryGetProperty("groupLists", out JsonElement groupLists) ? groupLists.ToObject>>>() : new List>>(), targets = obj.TryGetProperty("targets", out JsonElement targets) ? targets.ToObject>() : new List() }; tasks.Add(client.GetContainer(Constant.TEAMModelOS, "Common").ReplaceItemAsync(examInfo, examInfo.id, new PartitionKey(examInfo.code))); infos.Add(examInfo.id); }; } catch (Exception ex) { await _dingDing.SendBotMsg($"OS,{_option.Location} /fix-data/fix-publish \n {ex.Message}\n{ex.StackTrace}", GroupNames.醍摩豆服務運維群組); } } } } } //await Task.WhenAll(tasks); int pagesize = 50; if (tasks.Count <= pagesize) { await Task.WhenAll(tasks); } else { int pages = (tasks.Count + pagesize) / pagesize; //256是批量操作最大值,pages = (total + max -1) / max; for (int i = 0; i < pages; i++) { var listssb = tasks.Skip((i) * pagesize).Take(pagesize).ToList(); await Task.WhenAll(listssb); } } } catch (Exception e) { await _dingDing.SendBotMsg($"OS,{_option.Location} /fix-data/fix-publish \n {e.Message}\n{e.StackTrace}", GroupNames.醍摩豆服務運維群組); } /*foreach (var item in exams) { dynamic dyn = JsonConvert.DeserializeObject(Convert.ToString(item)) ; foreach (var obj in dyn) { if (obj.Name == "publish") { obj.Value = 0; break; } } ExamInfo info = new ExamInfo(); try { string dy = JsonConvert.SerializeObject(dyn); info = JsonConvert.DeserializeObject(dy); } catch (Exception e ) { //string dy = JsonConvert.SerializeObject(dyn); } //await client.GetContainer(Constant.TEAMModelOS, "Common").ReplaceItemAsync(dyn, dyn.id, new PartitionKey(dyn.code)); infos.Add(info.id); }*/ return infos; } public static async Task> FixExamClassResult(CosmosClient client, JsonElement data, DingDing _dingDing, CoreAPIHttpService _coreAPIHttpService, Option _option) { List results = new List(); List ids = new List(); List>> tasks = new List>>(); List>> taskInfo = new List>>(); try { /*await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "Common").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c where c.pk = 'ExamClassResult' and c._ts >= 1640966400")) { results.Add(item); //await Task.WhenAll(tasks); } HashSet examIds = new HashSet(); foreach (var item in results) { ids.Add(item.id); examIds.Add(item.examId); List status = new List(); //List classIds = new List(); //classIds.Add(item.info.id); if (item.status.Count == 0) { if (item.studentAnswers.Count > 0) { foreach (var answer in item.studentAnswers) { if (answer.Count == 0) { item.status.Add(1); } else { item.status.Add(0); } } } } else { List vs = new(); foreach (int st in item.status) { var ans = (st == -2) ? 1 : 0; vs.Add(ans); } item.status = vs; } *//*if (item.progress == true) { int n = 0; foreach (string sta in item.studentIds) { for (int i = 0; i < item.studentScores[n].Count; i++) { if (item.studentScores[n][i] == -1) { item.studentScores[n][i] = 0; } } n++; } }*//* tasks.Add(client.GetContainer(Constant.TEAMModelOS, "Common").ReplaceItemAsync(item, item.id, new PartitionKey(item.code))); }*/ List info = new List(); await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "Common").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c where c.pk = 'Exam' and c.progress = 'finish' and c._ts >= 1640966400")) { info.Add(item); //await Task.WhenAll(tasks); } foreach (var item in info) { item.updateTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); taskInfo.Add(client.GetContainer(Constant.TEAMModelOS, "Common").ReplaceItemAsync(item, item.id, new PartitionKey(item.code))); } //TODO 数据量过大的时候暂未处理 await Task.WhenAll(taskInfo); } catch (Exception ex) { await _dingDing.SendBotMsg($"OS,{_option.Location} /fix-data/fix-classIds \n {ex.Message}\n{ex.StackTrace}", GroupNames.醍摩豆服務運維群組); } return ids; } public static async Task> FixSchoolType(CosmosClient client, JsonElement data, DingDing _dingDing, CoreAPIHttpService _coreAPIHttpService, Option _option) { List>> tasks = new List>>(); List info = new List() ; try { await foreach (var item in client.GetContainer(Constant.TEAMModelOS, "Common").GetItemQueryIterator(queryText: $"SELECT value(c) FROM c ",requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey($"Base") })) { info.Add(item); } foreach (var item in info) { foreach (var period in item.period) { } tasks.Add(client.GetContainer(Constant.TEAMModelOS, "Common").ReplaceItemAsync(item, item.id, new PartitionKey(item.code))); } //TODO 数据量过大的时候暂未处理 await Task.WhenAll(tasks); } catch (Exception ex) { await _dingDing.SendBotMsg($"OS,{_option.Location} /fix-data/fix-type \n {ex.Message}\n{ex.StackTrace}", GroupNames.醍摩豆服務運維群組); } var ids = info.Select(x => x.id).ToList(); return ids; } } }