TEAMModelOS/TEAMModelOS/Controllers/Core/ImportController.cs

using Azure.Storage.Blobs.Models;
using Azure.Storage.Sas;
using HTEXLib;
using HTEXLib.Builders;
using HTEXLib.Helpers.ShapeHelpers;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IdentityModel.Tokens.Jwt;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using TEAMModelOS.SDK.Models;
using TEAMModelOS.Models.Dto;
using TEAMModelOS.SDK;
using TEAMModelOS.SDK.DI;
using TEAMModelOS.SDK.Extension;
using TEAMModelOS.SDK.Module.AzureBlob.Configuration;
using TEAMModelOS.SDK.Module.AzureBlob.Container;
using TEAMModelOS.Services;
using HTEXLib.Translator;
using HTEXLib.DOCX.Models;
using System.Collections.Concurrent;
using TEAMModelOS.Filter;
using Ionic.Zip;

namespace TEAMModelOS.Controllers
{
    [Route("import")]
    [ApiController]
    public class ImportController : ControllerBase
    {
        public readonly DingDing _dingDing;

        public PPTX2HTEXTranslator _PPTX2HTEXTranslator { get; set; }
        public DOXC2HTMLTranslator _DOXC2HTMLTranslator { get; set; }

        public AzureStorageFactory _azureStorage { get; set; }
        private readonly IHttpClientFactory _clientFactory;
        public HTML2ITEMV3Translator _HTML2ITEMV3Translator { get; set; }
        public ImportController(
            PPTX2HTEXTranslator PPTX2HTEXTranslator, IHttpClientFactory clientFactory,
            DOXC2HTMLTranslator DOXC2HTMLTranslator, HTML2ITEMV3Translator HTML2ITEMV3Translator, AzureStorageFactory azureStorage, DingDing ding)
        {
            _HTML2ITEMV3Translator = HTML2ITEMV3Translator;
            _DOXC2HTMLTranslator = DOXC2HTMLTranslator;
            _clientFactory = clientFactory;
            _azureStorage = azureStorage;
            this._PPTX2HTEXTranslator = PPTX2HTEXTranslator;
            _dingDing = ding;

        }
        private static string ReplaceLast(string input, string oldValue, string newValue)
        {
            int index = input.LastIndexOf(oldValue);
            if (index < 0)
            {
                return input;
            }
            else
            {
                StringBuilder sb = new StringBuilder(input.Length - oldValue.Length + newValue.Length);
                sb.Append(input.Substring(0, index));
                sb.Append(newValue);
                sb.Append(input.Substring(index + oldValue.Length,
                   input.Length - index - oldValue.Length));

                return sb.ToString();
            }
        }

        /// <summary>
        ///  {"file":"www....xxxx.pptx","scope":"private/school"}
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        [HttpPost("parse-doc")]
        //[RequestSizeLimit(102_400_000_00)] //最大10000m左右
        [AuthToken(Roles = "admin,teacher")]
        public async Task<IActionResult> ParseDoc(JsonElement request)
        {
            //string id_token = HttpContext.GetXAuth("IdToken");
            //if (string.IsNullOrEmpty(id_token)) return BadRequest();
            //var jwt = new JwtSecurityToken(id_token);
            //if (!jwt.Payload.Iss.Equals("account.teammodel", StringComparison.OrdinalIgnoreCase)) return BadRequest();
            //var id = jwt.Payload.Sub;
            var (id, _, _, school) = HttpContext.GetAuthTokenInfo();
            var containerid = id;
            if (request.TryGetProperty("scope", out JsonElement jscope))
            {
                if (jscope.GetString().Equals("school"))
                {
                    containerid = school;
                }
            }
            request.TryGetProperty("file", out JsonElement code);
            string azureBlobSAS = System.Web.HttpUtility.UrlDecode(code.ToString(), Encoding.UTF8);
            (string, string) a = BlobUrlString(azureBlobSAS);
            string ContainerName = a.Item1;
            string BlobName = a.Item2;
            bool flg = IsBlobName(BlobName);
            var codes = azureBlobSAS.Split("/");
            var file = codes[codes.Length - 1].Split(".");
            var ext = file[file.Length - 1];
            var FileName = ReplaceLast(codes[codes.Length - 1], "." + ext, "");
            if (flg)
            {
                BlobAuth blobAuth = _azureStorage.GetBlobSasUriRead(ContainerName, BlobName);
                var response = await _clientFactory.CreateClient().GetAsync(new Uri(blobAuth.url));
                response.EnsureSuccessStatusCode();
                Stream stream = await response.Content.ReadAsStreamAsync();

                if (ext.ToLower() == "pptx" || ext.ToLower() == "xml")
                {
                    string index = await PPTXTranslator(containerid, FileName, stream);
                   // await _azureStorage.GetBlobServiceClient().DeleteBlobs(_dingDing, containerid, new List<string>() { $"{FileName}.{ext}" });
                    return Ok(new { index = index });
                }
                else if (ext.ToLower() == "docx" || ext.ToLower() == "doc")
                {
                    return Ok(new { index = "" });
                    // await _azureStorage.GetBlobServiceClient().DeleteBlobs(containerid, new List<string>() { $"{FileName}.{ext}" });
                }
                else if (ext.ToLower() == "htex")
                {
                    var index = await HTEXTranslator(containerid, FileName, stream);
                   //await _azureStorage.GetBlobServiceClient().DeleteBlobs(_dingDing, containerid, new List<string>() { $"{FileName}.{ext}" });
                    return Ok(new { index = index });
                }
                else
                {
                    return Ok(new { index = "" });
                }
            }
            else { return BadRequest("不是正确的Blob链接！"); }
        }

        private async Task<string> HTEXTranslator(string containerid, string FileName, Stream stream)
        {
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
            //处理中文乱码问题
            Encoding encoding = Encoding.GetEncoding("GB2312");
            var options = new ReadOptions { Encoding = encoding };
            string index = null;
            bool hasindex = false;
            List<Task<string>> tasks = new List<Task<string>>();
            ZipFile zip = ZipFile.Read(stream, options);
            zip.AlternateEncoding = encoding;
            List<Stream> streams = new List<Stream>();
            foreach (var f in zip.Entries)
            {
                string name = FileName + "/" + f.FileName;
                if (f.IsDirectory)
                {
                    continue;
                }
                var uploadStream = f.OpenReader();
                byte[] buffer = new byte[uploadStream.Length];
                uploadStream.Read(buffer, 0, buffer.Length);
                Stream blobstream = new MemoryStream(buffer);
                streams.Add(blobstream);
                tasks.Add(_azureStorage.UploadFileByContainer(containerid, blobstream, "res", $"{name}", false));
                if (name.Contains($"{FileName}/index.json"))
                {
                    hasindex = true;
                }
                uploadStream.Close();
            }

            zip.Dispose();
            stream.Close();
            if (hasindex)
            {
                await Task.WhenAll(tasks);
                foreach (var task in tasks)
                {
                    var url = System.Web.HttpUtility.UrlDecode(task.Result, Encoding.UTF8);
                    if (url.Contains($"{FileName}/index.json"))
                    {
                        index = url;
                    }
                }
            }
            //释放资源
            streams.ForEach(x => { x.Close(); });
            return index;
        }

        private static (string, string) BlobUrlString(string sasUrl)
        {
            sasUrl = sasUrl.Substring(8);
            string[] sasUrls = sasUrl.Split("/");
            string ContainerName;
            ContainerName = sasUrls[1].Clone().ToString();
            string item = sasUrls[0] + "/" + sasUrls[1] + "/";
            string blob = sasUrl.Replace(item, "");
            return (ContainerName, blob);
        }
        public static bool IsBlobName(string BlobName)
        {
            return System.Text.RegularExpressions.Regex.IsMatch(BlobName,
             @"(?!((^(con)$)|^(con)\\..*|(^(prn)$)|^(prn)\\..*|(^(aux)$)|^(aux)\\..*|(^(nul)$)|^(nul)\\..*|(^(com)[1-9]$)|^(com)[1-9]\\..*|(^(lpt)[1-9]$)|^(lpt)[1-9]\\..*)|^\\s+|.*\\s$)(^[^\\\\\\:\\<\\>\\*\\?\\\\\\""\\\\|]{1,255}$)");
        }
        /// <summary>
        /// 
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        [HttpPost("upload-pptx")]
        [RequestSizeLimit(102_400_000_00)] //最大10000m左右
        public async Task<IActionResult> UploadPPTX([FromForm] IFormFile file)
        {
            var (id, _, _, school) = HttpContext.GetAuthTokenInfo();
            if (FileType.GetExtention(file.FileName).ToLower().Equals("pptx") || FileType.GetExtention(file.FileName).ToLower().Equals("xml"))
            {
                string FileName = file.FileName.Split(".")[0];
                Stream streamFile = file.OpenReadStream();
                string index = await PPTXTranslator(id, FileName, streamFile);
                return Ok(new { index = index });
            }
            else
            {
                return BadRequest("type is not pptx or xml !");
            }
        }

        /// <summary>
        /// docUrl
        /// folder
        /// shaCode
        /// 
        /// UploadWord
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        [HttpPost("upload-word")]
        [RequestSizeLimit(102_400_000_00)] //最大10000m左右
        public IActionResult UploadWord([FromForm] IFormFile file)
        {
            //  ResponseBuilder responseBuilder = new ResponseBuilder();
            if (!FileType.GetExtention(file.FileName).ToLower().Equals("docx"))
            {
                return BadRequest(new Dictionary<string, object> { { "msg", "type is not docx！" }, { "code", ResponseCode.FAILED } });
            }

            var doc = _DOXC2HTMLTranslator.Translate(file.OpenReadStream());
            // Dictionary<string, object> model = await ImportExerciseService.UploadWord(_azureStorage, file);
            return Ok(new { htmlString = doc });
        }

        /// <summary>
        /// word直接转题目
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        [HttpPost("parse-word")]
        public IActionResult ParseWord([FromForm] IFormFile file)
        {
            if (!FileType.GetExtention(file.FileName).ToLower().Equals("docx"))
            {
                return BadRequest(new Dictionary<string, object> { { "msg", "type is not docx！" }, { "code", ResponseCode.FAILED } });
            }
            var doc = _DOXC2HTMLTranslator.Translate(file.OpenReadStream());

            (List<HTEXLib.DOCX.Models.ItemInfo> tests, List<string> error) = _HTML2ITEMV3Translator.Translate(doc);

            return Ok(new { tests, emferror = error });
        }


        /// <summary>
        /// word转html
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        [HttpPost("parse-docx")]
        public IActionResult ParseDocx([FromForm] IFormFile file)
        {
            if (!FileType.GetExtention(file.FileName).ToLower().Equals("docx"))
            {
                return BadRequest(new Dictionary<string, object> { { "msg", "type is not docx！" }, { "code", ResponseCode.FAILED } });
            }
            var doc = _DOXC2HTMLTranslator.Translate(file.OpenReadStream());
            // (List<HTEXLib.DOCX.Models.ItemInfo> tests, List<string> error) = _HTML2ITEMV3Translator.Translate(doc);
            return Ok(new { html = doc });
        }
        /// <summary>
        /// html转题目
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        [HttpPost("parse-html")]
        public IActionResult AnalyzeHtml(JsonElement request)
        {
            if (!request.TryGetProperty("html", out JsonElement html)) { return BadRequest(); }
            (List<HTEXLib.DOCX.Models.ItemInfo> tests, List<string> error) = _HTML2ITEMV3Translator.Translate(html.GetString());
            return Ok(new { tests, emferror = error });
        }


        private async Task<string> PPTXTranslator(string containerid, string FileName, Stream streamFile)
        {
            if (string.IsNullOrWhiteSpace(containerid))
            {
                containerid = "teammodelos";
            }
            //var status = await _azureStorage.GetBlobServiceClient().DeleteBlobs(_dingDing, containerid, new List<string> { $"res/{FileName}" });
            string shaCode = Guid.NewGuid().ToString();
            HTEXLib.Htex htex = _PPTX2HTEXTranslator.Translate(streamFile);
            htex.name = FileName;
            var slides = htex.slides;
            List<Task<string>> tasks = new List<Task<string>>();
            HTEXIndex index = new HTEXIndex() { name = FileName, size = htex.size, thumbnail = htex.thumbnail, id = shaCode };

            List<KeyValuePair<string, string>> blobslidenames = new List<KeyValuePair<string, string>>();
            foreach (var slide in slides)
            {
                string json = JsonHelper.ToJson(slide, ignoreNullValue: false);
                string guid = Guid.NewGuid().ToString();
                blobslidenames.Add(new KeyValuePair<string, string>(guid, json));
            }
            List<Sld> slds = new List<Sld>();
            foreach (var key in blobslidenames)
            {
                slds.Add(new Sld { type = "normal", url = $"{key.Key}.json", scoring = null }); ;
                tasks.Add(_azureStorage.UploadFileByContainer(containerid, key.Value, "res", $"{FileName}/{key.Key}.json", false));
            }
            await Task.WhenAll(tasks);
            // Dictionary<string, Store> dict = new Dictionary<string, Store>();
            List<Task> tasksFiles = new List<Task>();
            foreach (var key in htex.stores.Keys)
            {
                if (key.EndsWith(".wdp") || key.EndsWith(".xlsx"))
                {
                    htex.stores.Remove(key);
                    continue;
                }
                var store = htex.stores[key];
                Store str = new Store() { path = key, contentType = store.contentType, isLazy = store.isLazy };
                if (!store.isLazy && store.contentType != null && ContentTypeDict.extdict.TryGetValue(store.contentType, out string ext) && store.url.Contains(";base64,"))
                {
                    string[] strs = store.url.Split(',');
                    Stream stream = new MemoryStream(Convert.FromBase64String(strs[1]));
                    // var urlstrs = key.Split("/");
                    var name = key.Replace("/", "");
                    str.url = $"{name}";
                    tasksFiles.Add(_azureStorage.UploadFileByContainer(containerid, stream, "res", $"{FileName}/{name}", false));
                }
                else
                {
                    str.url = System.Web.HttpUtility.UrlDecode(store.url, Encoding.UTF8);
                }
                // dict.TryAdd(key, str);
            }
            await Task.WhenAll(tasksFiles);
            // index.stores = dict;
            index.slides = slds;
            var BlobUrl = await _azureStorage.UploadFileByContainer(containerid, JsonHelper.ToJson(index, ignoreNullValue: false), "res", FileName + "/" + "index.json", false);
            return System.Web.HttpUtility.UrlDecode(BlobUrl, Encoding.UTF8);
        }
    }

    public class HTEXIndex
    {
        public string id { get; set; }
        public string version { get; set; } = "1.0.20201210";
        public string name { get; set; }
        public HTEXLib.HtexSize size { get; set; }
        public List<Sld> slides { get; set; }
        //缩略图
        public string thumbnail { get; set; }
        //  public int page { get; set; }
        // public Dictionary<string, Store> stores { get; set; }
        public List<string> knowledge { get; set; }
        public string periodId { get; set; }
        public List<string> gradeIds { get; set; }
        public string subjectId { get; set; }
        public string subjectName { get; set; }
        public string score { get; set; }
        public string code { get; set; }
        public string scope { get; set; }
        public int? multipleRule { get; set; }
    }
    public class Sld
    {
        /// <summary>
        /// normal,普通的hte页面 single 单选题 multiple 多选题 judge 判断题 complete 填空题 subjective 问答题 compose 综合题 
        /// </summary>
        public string type { get; set; }
        /// <summary>
        /// 单页PPTx htex 的解析链接或一个题目的链接
        /// </summary>
        public string url { get; set; }
        /// <summary>
        /// 题目的配分，如果为type为normal 及compose ,则 scoring=null
        /// </summary>
        public Scoring scoring { get; set; }
        /// <summary>
        /// 单页PPTx htex 的缩略图
        /// </summary>
        public string thumbnail { get; set; }
    }

    public class Scoring
    {
        public double score { get; set; }
        public List<string> ans { get; set; } = new List<string>();
    }
}