From 1e7280cb4241b059d9853b7b8695f8464b1e5b8d Mon Sep 17 00:00:00 2001 From: Administrator Date: Sat, 26 Jul 2025 23:51:29 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A4=A7=E6=96=87=E4=BB=B6=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 43 +- .../aipan/component/LocalFileStoreEngine.java | 27 +- .../aipan/component/MinIOFileStoreEngine.java | 59 ++- .../aipan/component/OSSFileStoreEngine.java | 23 +- .../ycloud/aipan/component/StoreEngine.java | 49 +- .../aipan/config/InterceptorConfig.java | 25 +- .../aipan/controller/FileController.java | 51 +++ .../controller/req/FileChunkInitTaskReq.java | 26 ++ .../controller/req/FileChunkMergeReq.java | 17 + .../org/ycloud/aipan/dto/FileChunkDTO.java | 66 +++ .../aipan/interceptor/LoginInterceptor.java | 3 +- .../org/ycloud/aipan/model/FileChunkDO.java | 3 + .../aipan/service/AccountFileService.java | 3 + .../aipan/service/FileChunkService.java | 28 ++ .../service/impl/AccountFileServiceImpl.java | 1 + .../service/impl/FileChunkServiceImpl.java | 172 +++++++ src/main/resources/application.yml | 1 - src/main/resources/static/fileupload.html | 425 ++++++++++++++++++ src/main/resources/static/spark-md5.min.js | 1 + .../org/ycloud/aipan/AmazonS3ClientTests.java | 2 +- .../org/ycloud/aipan/BigFileUploadTest.java | 150 +++++++ .../ycloud/aipan/FileChunkUploadTests.java | 158 +++++++ 22 files changed, 1295 insertions(+), 38 deletions(-) create mode 100644 src/main/java/org/ycloud/aipan/controller/req/FileChunkInitTaskReq.java create mode 100644 src/main/java/org/ycloud/aipan/controller/req/FileChunkMergeReq.java create mode 100644 src/main/java/org/ycloud/aipan/dto/FileChunkDTO.java create mode 100644 src/main/java/org/ycloud/aipan/service/FileChunkService.java create mode 100644 src/main/java/org/ycloud/aipan/service/impl/FileChunkServiceImpl.java create mode 100644 src/main/resources/static/fileupload.html create mode 100644 src/main/resources/static/spark-md5.min.js create mode 100644 src/test/java/org/ycloud/aipan/BigFileUploadTest.java create mode 100644 src/test/java/org/ycloud/aipan/FileChunkUploadTests.java diff --git a/README.md b/README.md index 793373a..b9c0b21 100644 --- a/README.md +++ b/README.md @@ -4244,7 +4244,6 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ } ``` - ### 大文件上传接口开发和全链路测试 @@ -5856,9 +5855,9 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 因为市场人员缺少太多，企业招聘不到人员，就会降低学历提高待遇，这个就是机会 - 大专以上学历+后端/前端、测试能力 ，就可以学！！ - ![image-20250215115400776](file:///./img/image-20250215115400776.png?lastModify=1750210947) + ![image-20250215115400776](./img/image-20250215115400776.png?lastModify=1750210947) - ![image-20250215115329505](file:///./img/image-20250215115329505.png?lastModify=1750210947) + ![image-20250215115329505](./img/image-20250215115329505.png?lastModify=1750210947) - 学习我们小滴课堂的兄弟们的基本画像说明 @@ -6121,7 +6120,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ | **典型错误类型** | 可能偏离指令或生成不相关内容 | 逻辑漏洞、计算错误或步骤缺失 | | **资源消耗** | 通常更轻量（可部署较小参数模型） | 需要更大参数量支持复杂推理 | -![image-20250215170037818](file:///./img/image-20250215170037818.png?lastModify=1750210947) +![image-20250215170037818](./img/image-20250215170037818.png?lastModify=1750210947) - 案例应用说明 @@ -6131,7 +6130,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 模型选择决策 - ![image-20250215165633327](file:///./img/image-20250215165633327.png?lastModify=1750210947) + ![image-20250215165633327](./img/image-20250215165633327.png?lastModify=1750210947) - 案例代码参考 @@ -6336,7 +6335,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 找个LLM进行测试 `我今天被公司解雇了，很难过` 0.2和0.7温度的区别 - ![image-20250217140255457](file:///./img/image-20250217140255457.png?lastModify=1750210947) + ![image-20250217140255457](./img/image-20250217140255457.png?lastModify=1750210947) - 预训练（Pre-training） @@ -6358,7 +6357,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 通俗解释：通过人类评分优化模型输出的"AI教练系统" - 训练流程 - ![export_3t84b](file:///./img/export_3t84b.png?lastModify=1750210947) + ![export_3t84b](./img/export_3t84b.png?lastModify=1750210947) - 模型蒸馏（Knowledge Distillation） @@ -6443,7 +6442,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 很多同学看到大模型的文档里有token，好奇这个是做啥的，和JWT的token啥区别？ - 包括很多在线的LLM大模型接口里面，按照token进行收费的 -![image-20250217154619961](file:///./img/image-20250217154619961.png?lastModify=1750210947) +![image-20250217154619961](./img/image-20250217154619961.png?lastModify=1750210947) - 什么是LLM里面的token - Token 是文本的基本单位，用于将文本分解为模型能够处理的最小单元 @@ -6492,7 +6491,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 私有化部署大模型后，使用方式也都是调用接口 - 不同企业、经费、数据安全和项目领域也决定如何选择 - ![image-20250217162419684](file:///./img/image-20250217162419684.png?lastModify=1750210947) + ![image-20250217162419684](./img/image-20250217162419684.png?lastModify=1750210947) - 关键差异点对比 @@ -6547,7 +6546,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 包括后续框架整合也是，请求协议基本和OpenAI一样, 毕竟龙头老大，也方便迁移 - 由于国内网络访问限制，OpenAI应用的开发也可以直接换国内的大模型，开发一样 - ![image-20250217162403779](file:///./img/image-20250217162403779.png?lastModify=1750210947) + ![image-20250217162403779](./img/image-20250217162403779.png?lastModify=1750210947) - OpenAI 提供的SDK 来调用大模型 @@ -6573,13 +6572,13 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 地址：https://bailian.console.aliyun.com/ - ![image-20250217145235399](file:///./img/image-20250217145235399.png?lastModify=1750210947) + ![image-20250217145235399](./img/image-20250217145235399.png?lastModify=1750210947) - **DeepSeek**（深度求索） - 地址：https://api-docs.deepseek.com/zh-cn/ - ![image-20250217145923315](file:///./img/image-20250217145923315.png?lastModify=1750210947) + ![image-20250217145923315](./img/image-20250217145923315.png?lastModify=1750210947) - **Kimi Chat**（月之暗面） @@ -6614,7 +6613,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 官网：https://lmstudio.ai/ - 部署过程相对简单，支持Win、Mac、Linux - ![image-20250217171804241](file:///./img/image-20250217171804241.png?lastModify=1750210947) + ![image-20250217171804241](./img/image-20250217171804241.png?lastModify=1750210947) - 本地部署 DeepSeek 硬件配置 @@ -6664,7 +6663,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ | **成本范围** | ￥400,000+ | ￥20,000,000+ | | **生态支持** | HuggingFace加速库优化 | 定制化CUDA内核+混合精度训练 | -![image-20250217174017791](file:///./img/image-20250217174017791.png?lastModify=1750210947) +![image-20250217174017791](./img/image-20250217174017791.png?lastModify=1750210947) #### Ollama介绍和本地快速安装 @@ -6684,7 +6683,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 文档：https://github.com/ollama/ollama/blob/main/README.md#quickstart - 安装实操（官网下载对应的包，不同系统选择不一样） -![image-20250217175605827](file:///./img/image-20250217175605827.png?lastModify=1750210947) +![image-20250217175605827](./img/image-20250217175605827.png?lastModify=1750210947) @@ -6721,7 +6720,7 @@ public JsonData list(@RequestParam(value = "parent_id")Long parentId){ - 使用ollama部署DeepSeek大模型，deepseek-r1:7b，deepseek-r1:14b - 注意：电脑配置不高的，不要部署14b哈 - ![image-20250217180343600](file:///./img/image-20250217180343600.png?lastModify=1750210947) + ![image-20250217180343600](./img/image-20250217180343600.png?lastModify=1750210947) - 部署实战 @@ -6735,7 +6734,7 @@ ollama run deepseek-r1:14b - 问题: `算下deeeeep里面有几个e` -![image-20250218215557400](file:///./img/image-20250218215557400.png?lastModify=1750210947) +![image-20250218215557400](./img/image-20250218215557400.png?lastModify=1750210947) #### AI大模型可视化界面介绍和部署实战 @@ -6756,7 +6755,7 @@ ollama run deepseek-r1:14b - 支持多服务商集成的AI对话客户端 - 地址：https://cherry-ai.com/ - ![image-20250217183246768](file:///./img/image-20250217183246768.png?lastModify=1750210947) + ![image-20250217183246768](./img/image-20250217183246768.png?lastModify=1750210947) - Chatbox @@ -6764,14 +6763,14 @@ ollama run deepseek-r1:14b - 配置Ollama允许远程连接 https://chatboxai.app/zh/help-center/connect-chatbox-remote-ollama-service-guide - 地址：https://chatboxai.app/ - ![image-20250217183301366](file:///./img/image-20250217183301366.png?lastModify=1750210947) + ![image-20250217183301366](./img/image-20250217183301366.png?lastModify=1750210947) - Chatbox安装和实操 - 安装包：官网直接下载，根据自己的系统选择 - 配置大模型服务 - ![image-20250217184140019](file:///./img/image-20250217184140019.png?lastModify=1750210947) + ![image-20250217184140019](./img/image-20250217184140019.png?lastModify=1750210947) @@ -6883,7 +6882,7 @@ ollama run deepseek-r1:14b - 参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/use-qwen-by-calling-api - 下面的返回协议 -![image-20250220160752180](file:///./img/image-20250220160752180.png?lastModify=1750210947) +![image-20250220160752180](./img/image-20250220160752180.png?lastModify=1750210947) @@ -6994,7 +6993,7 @@ ollama run deepseek-r1:14b - 技术选型架构图 - ![image-20250221114009228](file:///./img/image-20250221114009228.png?lastModify=1750210947) + ![image-20250221114009228](./img/image-20250221114009228.png?lastModify=1750210947) - 开发AI大模型的Python技术选型（部分技术选型） diff --git a/src/main/java/org/ycloud/aipan/component/LocalFileStoreEngine.java b/src/main/java/org/ycloud/aipan/component/LocalFileStoreEngine.java index 505fe20..193f87a 100644 --- a/src/main/java/org/ycloud/aipan/component/LocalFileStoreEngine.java +++ b/src/main/java/org/ycloud/aipan/component/LocalFileStoreEngine.java @@ -1,12 +1,15 @@ package org.ycloud.aipan.component; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.HttpMethod; +import com.amazonaws.services.s3.model.*; import jakarta.servlet.http.HttpServletResponse; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; +import java.net.URL; +import java.util.Date; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; //@Component @@ -65,4 +68,24 @@ public class LocalFileStoreEngine implements StoreEngine{ public void download2Response(String bucketName, String objectKey, HttpServletResponse response) { } + + @Override + public PartListing listMultipart(String bucketName, String objectKey, String uploadId) { + return null; + } + + @Override + public InitiateMultipartUploadResult initMultipartUploadTask(String bucketName, String objectKey, ObjectMetadata metadata) { + return null; + } + + @Override + public URL genePreSignedUrl(String bucketName, String objectKey, HttpMethod httpMethod, Date expiration, Map params) { + return null; + } + + @Override + public CompleteMultipartUploadResult mergeChunks(String bucketName, String objectKey, String uploadId, List partETags) { + return null; + } } \ No newline at end of file diff --git a/src/main/java/org/ycloud/aipan/component/MinIOFileStoreEngine.java b/src/main/java/org/ycloud/aipan/component/MinIOFileStoreEngine.java index 7f165a2..900d84f 100644 --- a/src/main/java/org/ycloud/aipan/component/MinIOFileStoreEngine.java +++ b/src/main/java/org/ycloud/aipan/component/MinIOFileStoreEngine.java @@ -1,19 +1,19 @@ package org.ycloud.aipan.component; +import com.amazonaws.HttpMethod; import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.*; import jakarta.annotation.Resource; import jakarta.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.tomcat.util.http.fileupload.IOUtils; +import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.Date; import java.util.List; import java.util.Map; @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; @Slf4j @Component +@Primary public class MinIOFileStoreEngine implements StoreEngine { @Resource @@ -223,4 +224,54 @@ public class MinIOFileStoreEngine implements StoreEngine { log.error("下载 bucket {} 中对象 {} 失败: {}", bucketName, objectKey, e.getMessage(), e); } } + + @Override + public PartListing listMultipart(String bucketName, String objectKey, String uploadId) { + try { + ListPartsRequest request = new ListPartsRequest(bucketName, objectKey, uploadId); + return amazonS3Client.listParts(request); + } catch (Exception e) { + log.error("errorMsg={}", e); + return null; + } + } + + @Override + public InitiateMultipartUploadResult initMultipartUploadTask(String bucketName, String objectKey, ObjectMetadata metadata) { + try { + InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectKey, metadata); + return amazonS3Client.initiateMultipartUpload(request); + } catch (Exception e) { + log.error("errorMsg={}", e); + return null; + } + } + + + @Override + public URL genePreSignedUrl(String bucketName, String objectKey, HttpMethod httpMethod, Date expiration, Map params) { + try { + GeneratePresignedUrlRequest genePreSignedUrlReq = + new GeneratePresignedUrlRequest(bucketName, objectKey, httpMethod) + .withExpiration(expiration); + //遍历params作为参数加到genePreSignedUrlReq里面，比如 添加上传ID和分片编号作为请求参数 + //genePreSignedUrlReq.addRequestParameter("uploadId", uploadId); + //genePreSignedUrlReq.addRequestParameter("partNumber", String.valueOf(i)); + for (Map.Entry entry : params.entrySet()) { + genePreSignedUrlReq.addRequestParameter(entry.getKey(), String.valueOf(entry.getValue())); + } + // 生成并获取预签名URL + return amazonS3Client.generatePresignedUrl(genePreSignedUrlReq); + } catch (Exception e) { + log.error("errorMsg={}", e); + return null; + } + } + + @Override + public CompleteMultipartUploadResult mergeChunks(String bucketName, String objectKey, String uploadId, List partETags) { + CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(bucketName, objectKey, uploadId, partETags); + return amazonS3Client.completeMultipartUpload(request); + + } } diff --git a/src/main/java/org/ycloud/aipan/component/OSSFileStoreEngine.java b/src/main/java/org/ycloud/aipan/component/OSSFileStoreEngine.java index 7615922..c9a59d9 100644 --- a/src/main/java/org/ycloud/aipan/component/OSSFileStoreEngine.java +++ b/src/main/java/org/ycloud/aipan/component/OSSFileStoreEngine.java @@ -1,5 +1,6 @@ package org.ycloud.aipan.component; +import com.amazonaws.HttpMethod; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.*; import jakarta.annotation.Resource; @@ -12,6 +13,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.Date; import java.util.List; import java.util.Map; @@ -20,7 +22,6 @@ import java.util.concurrent.TimeUnit; @Slf4j @Component -@Primary public class OSSFileStoreEngine implements StoreEngine { @Resource private AmazonS3Client amazonS3Client; @@ -148,6 +149,26 @@ public class OSSFileStoreEngine implements StoreEngine { } } + @Override + public PartListing listMultipart(String bucketName, String objectKey, String uploadId) { + return null; + } + + @Override + public InitiateMultipartUploadResult initMultipartUploadTask(String bucketName, String objectKey, ObjectMetadata metadata) { + return null; + } + + @Override + public URL genePreSignedUrl(String bucketName, String objectKey, HttpMethod httpMethod, Date expiration, Map params) { + return null; + } + + @Override + public CompleteMultipartUploadResult mergeChunks(String bucketName, String objectKey, String uploadId, List partETags) { + return null; + } + // 拼接路径 // public static void main(String[] args) { // String fileSeparator = System.getProperty("file.separator"); diff --git a/src/main/java/org/ycloud/aipan/component/StoreEngine.java b/src/main/java/org/ycloud/aipan/component/StoreEngine.java index 3ea4dc1..cc1ebc7 100644 --- a/src/main/java/org/ycloud/aipan/component/StoreEngine.java +++ b/src/main/java/org/ycloud/aipan/component/StoreEngine.java @@ -1,11 +1,14 @@ package org.ycloud.aipan.component; -import com.amazonaws.services.s3.model.Bucket; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.HttpMethod; +import com.amazonaws.services.s3.model.*; import jakarta.servlet.http.HttpServletResponse; import org.springframework.web.multipart.MultipartFile; +import java.net.URL; +import java.util.Date; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; public interface StoreEngine { @@ -107,4 +110,46 @@ public interface StoreEngine { * @param response HTTP响应对象，用于输出下载的对象 */ void download2Response(String bucketName, String objectKey, HttpServletResponse response); + + + /*===================分片上传相关=============================*/ + /** + * 查询分片数据 + * @param bucketName 存储桶名称 + * @param objectKey 对象名称 + * @param uploadId 分片上传ID + * @return 分片列表对象 + */ + PartListing listMultipart(String bucketName, String objectKey, String uploadId); + + /** + * 1-初始化分片上传任务,获取uploadId,如果初始化时有 uploadId，说明是断点续传，不能重新生成 uploadId + * @param bucketName 存储桶名称 + * @param objectKey 对象名称 + * @param metadata 对象元数据 + * @return 初始化分片上传结果对象，包含uploadId等信息 + */ + InitiateMultipartUploadResult initMultipartUploadTask(String bucketName, String objectKey, ObjectMetadata metadata); + + + /** + * 2-生成分片上传地址，返回给前端 + * @param bucketName 存储桶名称 + * @param objectKey 对象名称 + * @param httpMethod HTTP方法，如GET、PUT等 + * @param expiration 签名过期时间 + * @param params 签名中包含的参数 + * @return 生成的预签名URL + */ + URL genePreSignedUrl(String bucketName, String objectKey, HttpMethod httpMethod, Date expiration, Map params); + + /** + * 3-合并分片 + * @param bucketName 存储桶名称 + * @param objectKey 对象名称 + * @param uploadId 分片上传ID + * @param partETags 分片ETag列表，用于验证分片的完整性 + * @return 完成分片上传结果对象 + */ + CompleteMultipartUploadResult mergeChunks(String bucketName, String objectKey, String uploadId, List partETags); } diff --git a/src/main/java/org/ycloud/aipan/config/InterceptorConfig.java b/src/main/java/org/ycloud/aipan/config/InterceptorConfig.java index d97625e..92daeab 100644 --- a/src/main/java/org/ycloud/aipan/config/InterceptorConfig.java +++ b/src/main/java/org/ycloud/aipan/config/InterceptorConfig.java @@ -1,10 +1,12 @@ package org.ycloud.aipan.config; +import org.springframework.context.annotation.Bean; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import org.ycloud.aipan.interceptor.LoginInterceptor; @@ -19,11 +21,26 @@ public class InterceptorConfig implements WebMvcConfigurer { public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(loginInterceptor) //添加拦截的路径 - .addPathPatterns("/api/account/*/**","/api/file/*/**","/api/share/*/**") + .addPathPatterns("/api/account/*/**", "/api/file/*/**", "/api/share/*/**") //排除不拦截 - .excludePathPatterns("/api/account/*/register","/api/account/*/login","/api/account/*/upload_avatar", - "/api/share/*/check_share_code","/api/share/*/visit","/api/share/*/detail_no_code","/api/share/*/detail_with_code"); + .excludePathPatterns("/api/account/*/register", "/api/account/*/login", "/api/account/*/upload_avatar", + "/api/share/*/check_share_code", "/api/share/*/visit", "/api/share/*/detail_no_code", "/api/share/*/detail_with_code"); + } + + + @Bean + public WebMvcConfigurer corsConfigurer() { + return new WebMvcConfigurer() { + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/api/**") + .allowedOriginPatterns("*") // 或指定域名 + .allowedMethods("GET", "POST", "PUT", "OPTIONS") + .allowedHeaders("*") + .allowCredentials(true); + } + }; } } \ No newline at end of file diff --git a/src/main/java/org/ycloud/aipan/controller/FileController.java b/src/main/java/org/ycloud/aipan/controller/FileController.java index 745bd6c..ae1fb76 100644 --- a/src/main/java/org/ycloud/aipan/controller/FileController.java +++ b/src/main/java/org/ycloud/aipan/controller/FileController.java @@ -7,9 +7,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import org.ycloud.aipan.controller.req.*; import org.ycloud.aipan.dto.AccountFileDTO; +import org.ycloud.aipan.dto.FileChunkDTO; import org.ycloud.aipan.dto.FolderTreeNodeDTO; import org.ycloud.aipan.interceptor.LoginInterceptor; import org.ycloud.aipan.service.AccountFileService; +import org.ycloud.aipan.service.FileChunkService; import org.ycloud.aipan.util.JsonData; import java.util.List; @@ -22,6 +24,9 @@ public class FileController { @Autowired private AccountFileService accountFileService; + @Autowired + private FileChunkService fileChunkService; + /** * 查询文件列表接口 */ @@ -124,4 +129,50 @@ public class FileController { Boolean flag = accountFileService.secondUpload(req); return JsonData.buildSuccess(flag); } + + // 大文件上传、分享、回收站、下载、搜索 + + /** + * 1-创建分片上传任务 + */ + @PostMapping("init_file_chunk_task") + public JsonData initFileChunkTask(@RequestBody FileChunkInitTaskReq req) { + req.setAccountId(LoginInterceptor.threadLocal.get().getId()); + FileChunkDTO fileChunkDTO = fileChunkService.initFileChunkTask(req); + return JsonData.buildSuccess(fileChunkDTO); + } + + /** + * 2-获取分片上传地址，返回minio临时签名地址 + */ + @GetMapping("/get_file_chunk_upload_url/{identifier}/{partNumber}") + public JsonData getFileChunkUploadUrl(@PathVariable("identifier") String identifier,@PathVariable("partNumber") int partNumber){ + Long accountId = LoginInterceptor.threadLocal.get().getId(); + String url = fileChunkService.genPreSignUploadUrl(accountId, identifier, partNumber); + return JsonData.buildSuccess(url); + } + + /** + * 3-合并分片 + */ + @PostMapping("merge_file_chunk") + public JsonData mergeFileChunk(@RequestBody FileChunkMergeReq req) { + req.setAccountId(LoginInterceptor.threadLocal.get().getId()); + fileChunkService.mergeFileChunk(req); + return JsonData.buildSuccess(); + } + + + /** + * 查询分片上传进度 + */ + @GetMapping("/chunk_upload_progress/{identifier}") + public JsonData getUploadProgress(@PathVariable("identifier") String identifier) { + Long accountId = LoginInterceptor.threadLocal.get().getId(); + FileChunkDTO fileChunkDTO = fileChunkService.listFileChunk(accountId, identifier); + return JsonData.buildSuccess(fileChunkDTO); + } + + + } diff --git a/src/main/java/org/ycloud/aipan/controller/req/FileChunkInitTaskReq.java b/src/main/java/org/ycloud/aipan/controller/req/FileChunkInitTaskReq.java new file mode 100644 index 0000000..177d64f --- /dev/null +++ b/src/main/java/org/ycloud/aipan/controller/req/FileChunkInitTaskReq.java @@ -0,0 +1,26 @@ +package org.ycloud.aipan.controller.req; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class FileChunkInitTaskReq { + + private Long accountId; + + private String filename; + + private String identifier; + + /*** + * 总大小 + */ + private Long totalSize; + + /** + * 分片大小 + */ + private Long chunkSize; + +} \ No newline at end of file diff --git a/src/main/java/org/ycloud/aipan/controller/req/FileChunkMergeReq.java b/src/main/java/org/ycloud/aipan/controller/req/FileChunkMergeReq.java new file mode 100644 index 0000000..45c2b20 --- /dev/null +++ b/src/main/java/org/ycloud/aipan/controller/req/FileChunkMergeReq.java @@ -0,0 +1,17 @@ +package org.ycloud.aipan.controller.req; + +import lombok.Data; +import lombok.experimental.Accessors; + + +@Data +@Accessors(chain = true) +public class FileChunkMergeReq { + + private String identifier; + + private Long parentId; + + private Long accountId; + +} \ No newline at end of file diff --git a/src/main/java/org/ycloud/aipan/dto/FileChunkDTO.java b/src/main/java/org/ycloud/aipan/dto/FileChunkDTO.java new file mode 100644 index 0000000..bcabffd --- /dev/null +++ b/src/main/java/org/ycloud/aipan/dto/FileChunkDTO.java @@ -0,0 +1,66 @@ +package org.ycloud.aipan.dto; + +import com.amazonaws.services.s3.model.PartSummary; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; +import org.ycloud.aipan.model.FileChunkDO; +import org.ycloud.aipan.util.SpringBeanUtil; + + +import java.util.List; + + +@Data +@NoArgsConstructor +@Accessors(chain = true) +public class FileChunkDTO { + + + public FileChunkDTO(FileChunkDO fileChunkDO){ + SpringBeanUtil.copyProperties(fileChunkDO,this); + } + + + private Long id; + + @Schema(description = "文件唯一标识（md5）") + private String identifier; + + @Schema(description = "分片上传ID") + private String uploadId; + + @Schema(description = "文件名") + private String fileName; + + @Schema(description = "所属桶名") + private String bucketName; + + @Schema(description = "文件的key") + private String objectKey; + + @Schema(description = "总文件大小（byte）") + private Long totalSize; + + @Schema(description = "每个分片大小（byte）") + private Long chunkSize; + + @Schema(description = "分片数量") + private Integer chunkNum; + + @Schema(description = "用户ID") + private Long accountId; + + + /** + * 是否完成上传 + */ + private boolean finished; + + /** + * 返回已经存在的分片 + */ + private List exitPartList; + +} \ No newline at end of file diff --git a/src/main/java/org/ycloud/aipan/interceptor/LoginInterceptor.java b/src/main/java/org/ycloud/aipan/interceptor/LoginInterceptor.java index 8040f6c..00481bf 100644 --- a/src/main/java/org/ycloud/aipan/interceptor/LoginInterceptor.java +++ b/src/main/java/org/ycloud/aipan/interceptor/LoginInterceptor.java @@ -32,7 +32,8 @@ public class LoginInterceptor implements HandlerInterceptor { return true; } - String token = request.getHeader("token"); +// String token = request.getHeader("token"); + String token ="YUANeyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJZVUFOIiwiYWNjb3VudElkIjoxODkwNDI0MTU2NjgwMzgwNDE4LCJ1c2VybmFtZSI6Inl1YW4iLCJpYXQiOjE3NTM1MzM1MTgsImV4cCI6MTc1NDEzODMxOH0.WeyTmVvbHqwaBIYLm0tiK7nOJWRQ4Q-jkLilJW0hR8U"; if(StringUtils.isBlank(token)){ token = request.getParameter("token"); } diff --git a/src/main/java/org/ycloud/aipan/model/FileChunkDO.java b/src/main/java/org/ycloud/aipan/model/FileChunkDO.java index 7c3d051..faa3a4e 100644 --- a/src/main/java/org/ycloud/aipan/model/FileChunkDO.java +++ b/src/main/java/org/ycloud/aipan/model/FileChunkDO.java @@ -7,8 +7,10 @@ import com.baomidou.mybatisplus.annotation.TableName; import java.io.Serializable; import java.util.Date; import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Builder; import lombok.Getter; import lombok.Setter; +import lombok.experimental.Accessors; /** *

@@ -22,6 +24,7 @@ import lombok.Setter; @Setter @TableName("file_chunk") @Schema(name = "FileChunkDO", description = "文件分片信息表") +@Accessors(chain = true) public class FileChunkDO implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/main/java/org/ycloud/aipan/service/AccountFileService.java b/src/main/java/org/ycloud/aipan/service/AccountFileService.java index 7a039e8..9d6aedf 100644 --- a/src/main/java/org/ycloud/aipan/service/AccountFileService.java +++ b/src/main/java/org/ycloud/aipan/service/AccountFileService.java @@ -72,4 +72,7 @@ public interface AccountFileService { * 3、建立关系 */ Boolean secondUpload(FileSecondUploadReq req); + + + void saveFileAndAccountFile(FileUploadReq req, String storeFileObjectKey); } diff --git a/src/main/java/org/ycloud/aipan/service/FileChunkService.java b/src/main/java/org/ycloud/aipan/service/FileChunkService.java new file mode 100644 index 0000000..cf44817 --- /dev/null +++ b/src/main/java/org/ycloud/aipan/service/FileChunkService.java @@ -0,0 +1,28 @@ +package org.ycloud.aipan.service; + + +import org.ycloud.aipan.controller.req.FileChunkInitTaskReq; +import org.ycloud.aipan.controller.req.FileChunkMergeReq; +import org.ycloud.aipan.dto.FileChunkDTO; + +public interface FileChunkService { + /** + * 初始化分片上传 + */ + FileChunkDTO initFileChunkTask(FileChunkInitTaskReq req); + + /** + * 获取临时文件上传地址 + */ + String genPreSignUploadUrl(Long accountId, String identifier, Integer partNumber); + + /** + * 合并分片 + */ + void mergeFileChunk(FileChunkMergeReq req); + + /** + * 查询分片上传进度 + */ + FileChunkDTO listFileChunk(Long accountId, String identifier); +} diff --git a/src/main/java/org/ycloud/aipan/service/impl/AccountFileServiceImpl.java b/src/main/java/org/ycloud/aipan/service/impl/AccountFileServiceImpl.java index 2dd6882..fa5e4bf 100644 --- a/src/main/java/org/ycloud/aipan/service/impl/AccountFileServiceImpl.java +++ b/src/main/java/org/ycloud/aipan/service/impl/AccountFileServiceImpl.java @@ -383,6 +383,7 @@ public class AccountFileServiceImpl implements AccountFileService { * @param req * @param storeFileObjectKey */ + @Override public void saveFileAndAccountFile(FileUploadReq req, String storeFileObjectKey) { //保存文件 FileDO fileDO = saveFile(req, storeFileObjectKey); diff --git a/src/main/java/org/ycloud/aipan/service/impl/FileChunkServiceImpl.java b/src/main/java/org/ycloud/aipan/service/impl/FileChunkServiceImpl.java new file mode 100644 index 0000000..4c7f596 --- /dev/null +++ b/src/main/java/org/ycloud/aipan/service/impl/FileChunkServiceImpl.java @@ -0,0 +1,172 @@ +package org.ycloud.aipan.service.impl; + +import cn.hutool.core.date.DateUtil; +import com.amazonaws.HttpMethod; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.http.MediaTypeFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.ycloud.aipan.component.StoreEngine; +import org.ycloud.aipan.config.MinioConfig; +import org.ycloud.aipan.controller.req.FileChunkInitTaskReq; +import org.ycloud.aipan.controller.req.FileChunkMergeReq; +import org.ycloud.aipan.controller.req.FileUploadReq; +import org.ycloud.aipan.dto.FileChunkDTO; +import org.ycloud.aipan.enums.BizCodeEnum; +import org.ycloud.aipan.exception.BizException; +import org.ycloud.aipan.mapper.FileChunkMapper; +import org.ycloud.aipan.mapper.StorageMapper; +import org.ycloud.aipan.model.FileChunkDO; +import org.ycloud.aipan.model.StorageDO; +import org.ycloud.aipan.service.AccountFileService; +import org.ycloud.aipan.service.FileChunkService; +import org.ycloud.aipan.util.CommonUtil; +import com.amazonaws.services.s3.model.*; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class FileChunkServiceImpl implements FileChunkService { + @Autowired + private StorageMapper storageMapper; + @Autowired + private StoreEngine fileStoreEngine; + @Autowired + private MinioConfig minioConfig; + @Autowired + private FileChunkMapper fileChunkMapper; + @Autowired + private AccountFileService accountFileService; + + @Override + @Transactional(rollbackFor = Exception.class) + public FileChunkDTO initFileChunkTask(FileChunkInitTaskReq req) { + //检查存储空间是否够 + StorageDO storageDO = storageMapper.selectOne(new QueryWrapper().eq("account_id", req.getAccountId())); + if (storageDO.getUsedSize() + req.getTotalSize() > storageDO.getTotalSize()) { + throw new BizException(BizCodeEnum.FILE_STORAGE_NOT_ENOUGH); + } + + String objectKey = CommonUtil.getFilePath(req.getFilename()); + // 根据文件名推断内容类型 + String contentType = MediaTypeFactory.getMediaType(objectKey).orElse(MediaType.APPLICATION_OCTET_STREAM).toString(); + // 设置文件元数据 + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentType(contentType); + // 初始化分片上传,获取上传ID + String uploadId = fileStoreEngine.initMultipartUploadTask(minioConfig.getBucketName(), objectKey, objectMetadata).getUploadId(); + + // 创建上传任务实体并设置相关属性 + FileChunkDO task = new FileChunkDO(); + int chunkNum = (int) Math.ceil(req.getTotalSize() * 1.0 / req.getChunkSize()); + task.setBucketName(minioConfig.getBucketName()) + .setChunkNum(chunkNum) + .setChunkSize(req.getChunkSize()) + .setTotalSize(req.getTotalSize()) + .setIdentifier(req.getIdentifier()) + .setFileName(req.getFilename()) + .setObjectKey(objectKey) + .setUploadId(uploadId) + .setAccountId(req.getAccountId()); + // 将任务插入数据库 + fileChunkMapper.insert(task); + // 构建并返回任务信息DTO + return new FileChunkDTO(task).setFinished(false).setExitPartList(new ArrayList<>()); + } + + + @Override + public String genPreSignUploadUrl(Long accountId, String identifier, Integer partNumber) { + FileChunkDO task = fileChunkMapper.selectOne(new QueryWrapper().lambda().eq(FileChunkDO::getIdentifier, identifier).eq(FileChunkDO::getAccountId, accountId)); + if (task == null) { + throw new BizException(BizCodeEnum.FILE_CHUNK_TASK_NOT_EXISTS); + } + //配置预签名过期时间 + Date expireDate = DateUtil.offsetMillisecond(new Date(), minioConfig.getPRE_SIGN_URL_EXPIRE().intValue()); + // 生成预签名URL + Map params = new HashMap<>(); + params.put("partNumber", partNumber.toString()); + params.put("uploadId", task.getUploadId()); + URL preSignedUrl = fileStoreEngine.genePreSignedUrl(minioConfig.getBucketName(), task.getObjectKey(), HttpMethod.PUT, expireDate, params); + log.info("生成预签名URL地址 identifier={},partNumber={}, preSignedUrl={}", identifier, partNumber, preSignedUrl.toString()); + return preSignedUrl.toString(); + } + + + public void mergeFileChunk(FileChunkMergeReq req) { + //获取任务和分片列表，检查是否足够合并 + FileChunkDO task = fileChunkMapper.selectOne(new QueryWrapper() + .eq("account_id", req.getAccountId()) + .eq("identifier", req.getIdentifier())); + if(task == null){ + throw new BizException(BizCodeEnum.FILE_CHUNK_TASK_NOT_EXISTS); + } + PartListing partListing = fileStoreEngine.listMultipart(task.getBucketName(), task.getObjectKey(), task.getUploadId()); + List parts = partListing.getParts(); + if(parts.size() != task.getChunkNum()){ + //上传的分片数量和记录中不对应，合并失败 + throw new BizException(BizCodeEnum.FILE_CHUNK_NOT_ENOUGH); + } + //检查更新存储空间 + StorageDO storageDO = storageMapper.selectOne(new QueryWrapper<>(new StorageDO()) + .eq("account_id", req.getAccountId())); + long realFileTotalSize = parts.stream().map(PartSummary::getSize).mapToLong(Long::valueOf).sum(); + if(storageDO.getUsedSize() + realFileTotalSize > storageDO.getTotalSize()){ + throw new BizException(BizCodeEnum.FILE_STORAGE_NOT_ENOUGH); + } + storageDO.setUsedSize(storageDO.getUsedSize() +realFileTotalSize); + storageMapper.updateById(storageDO); + + //2-合并文件 + CompleteMultipartUploadResult result = fileStoreEngine.mergeChunks(task.getBucketName(), + task.getObjectKey(), task.getUploadId(), + parts.stream().map(partSummary -> + new PartETag(partSummary.getPartNumber(), partSummary.getETag())) + .collect(Collectors.toList())); + //【判断是否合并成功 + if(result.getETag()!=null){ + FileUploadReq fileUploadReq = new FileUploadReq(); + fileUploadReq.setAccountId(req.getAccountId()) + .setFilename(task.getFileName()) + .setIdentifier(task.getIdentifier()) + .setParentId(req.getParentId()) + .setFileSize(realFileTotalSize) + .setFile(null); + + //存储文件和关联信息到数据库 + accountFileService.saveFileAndAccountFile(fileUploadReq,task.getObjectKey()); + //删除相关任务记录 + fileChunkMapper.deleteById(task.getId()); + log.info("合并成功"); + } + } + + @Override + public FileChunkDTO listFileChunk(Long accountId, String identifier) { + // 获取任务和分片列表，检查是否足够 + FileChunkDO task = fileChunkMapper.selectOne(new QueryWrapper().lambda().eq(FileChunkDO::getAccountId, accountId)); + if (task == null || !identifier.equals(task.getIdentifier())) { + return null; + } + FileChunkDTO result = new FileChunkDTO(task); + boolean doesObjectExist = fileStoreEngine.doesObjectExist(task.getBucketName(), task.getObjectKey()); + if (!doesObjectExist) { + // 不存在，表示未上传完，返回已上传的分片 + PartListing partListing = fileStoreEngine.listMultipart(task.getBucketName(), task.getObjectKey(), task.getUploadId()); + if(task.getChunkNum() == partListing.getParts().size()){ + //已经存在，合并 + result.setFinished(true).setExitPartList(partListing.getParts()); + }else { + result.setFinished(false).setExitPartList(partListing.getParts()); + } + } + return result; + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 18d3368..1a1bc3e 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -58,4 +58,3 @@ oss: access-key: LTAI5tRQFFPQWHPZksM9XGHG access-secret: z4ZSJffdH525Konxz7LBxOSAZP2BXN bucket-name: forward-tech - diff --git a/src/main/resources/static/fileupload.html b/src/main/resources/static/fileupload.html new file mode 100644 index 0000000..4446f16 --- /dev/null +++ b/src/main/resources/static/fileupload.html @@ -0,0 +1,425 @@ + + + + + 大文件上传（断点续传版） + + + +

+
+

大文件上传（断点续传）

+

支持秒传、断点续传，最大支持2GB

+
+ + +
+
📁
+

点击或拖拽文件到此处上传

+ +
+ + +
+
+
+
+ + +
+
+
+
+
0%
+
+
+ + + + + + +
✅ 上传成功
+
❌ 上传失败
+
+ + + + + + \ No newline at end of file diff --git a/src/main/resources/static/spark-md5.min.js b/src/main/resources/static/spark-md5.min.js new file mode 100644 index 0000000..2ef527d --- /dev/null +++ b/src/main/resources/static/spark-md5.min.js @@ -0,0 +1 @@ +(function(factory){if(typeof exports==="object"){module.exports=factory()}else if(typeof define==="function"&&define.amd){define(factory)}else{var glob;try{glob=window}catch(e){glob=self}glob.SparkMD5=factory()}})(function(undefined){"use strict";var add32=function(a,b){return a+b&4294967295},hex_chr=["0","1","2","3","4","5","6","7","8","9","a","b","c","d","e","f"];function cmn(q,a,b,x,s,t){a=add32(add32(a,q),add32(x,t));return add32(a<>>32-s,b)}function md5cycle(x,k){var a=x[0],b=x[1],c=x[2],d=x[3];a+=(b&c|~b&d)+k[0]-680876936|0;a=(a<<7|a>>>25)+b|0;d+=(a&b|~a&c)+k[1]-389564586|0;d=(d<<12|d>>>20)+a|0;c+=(d&a|~d&b)+k[2]+606105819|0;c=(c<<17|c>>>15)+d|0;b+=(c&d|~c&a)+k[3]-1044525330|0;b=(b<<22|b>>>10)+c|0;a+=(b&c|~b&d)+k[4]-176418897|0;a=(a<<7|a>>>25)+b|0;d+=(a&b|~a&c)+k[5]+1200080426|0;d=(d<<12|d>>>20)+a|0;c+=(d&a|~d&b)+k[6]-1473231341|0;c=(c<<17|c>>>15)+d|0;b+=(c&d|~c&a)+k[7]-45705983|0;b=(b<<22|b>>>10)+c|0;a+=(b&c|~b&d)+k[8]+1770035416|0;a=(a<<7|a>>>25)+b|0;d+=(a&b|~a&c)+k[9]-1958414417|0;d=(d<<12|d>>>20)+a|0;c+=(d&a|~d&b)+k[10]-42063|0;c=(c<<17|c>>>15)+d|0;b+=(c&d|~c&a)+k[11]-1990404162|0;b=(b<<22|b>>>10)+c|0;a+=(b&c|~b&d)+k[12]+1804603682|0;a=(a<<7|a>>>25)+b|0;d+=(a&b|~a&c)+k[13]-40341101|0;d=(d<<12|d>>>20)+a|0;c+=(d&a|~d&b)+k[14]-1502002290|0;c=(c<<17|c>>>15)+d|0;b+=(c&d|~c&a)+k[15]+1236535329|0;b=(b<<22|b>>>10)+c|0;a+=(b&d|c&~d)+k[1]-165796510|0;a=(a<<5|a>>>27)+b|0;d+=(a&c|b&~c)+k[6]-1069501632|0;d=(d<<9|d>>>23)+a|0;c+=(d&b|a&~b)+k[11]+643717713|0;c=(c<<14|c>>>18)+d|0;b+=(c&a|d&~a)+k[0]-373897302|0;b=(b<<20|b>>>12)+c|0;a+=(b&d|c&~d)+k[5]-701558691|0;a=(a<<5|a>>>27)+b|0;d+=(a&c|b&~c)+k[10]+38016083|0;d=(d<<9|d>>>23)+a|0;c+=(d&b|a&~b)+k[15]-660478335|0;c=(c<<14|c>>>18)+d|0;b+=(c&a|d&~a)+k[4]-405537848|0;b=(b<<20|b>>>12)+c|0;a+=(b&d|c&~d)+k[9]+568446438|0;a=(a<<5|a>>>27)+b|0;d+=(a&c|b&~c)+k[14]-1019803690|0;d=(d<<9|d>>>23)+a|0;c+=(d&b|a&~b)+k[3]-187363961|0;c=(c<<14|c>>>18)+d|0;b+=(c&a|d&~a)+k[8]+1163531501|0;b=(b<<20|b>>>12)+c|0;a+=(b&d|c&~d)+k[13]-1444681467|0;a=(a<<5|a>>>27)+b|0;d+=(a&c|b&~c)+k[2]-51403784|0;d=(d<<9|d>>>23)+a|0;c+=(d&b|a&~b)+k[7]+1735328473|0;c=(c<<14|c>>>18)+d|0;b+=(c&a|d&~a)+k[12]-1926607734|0;b=(b<<20|b>>>12)+c|0;a+=(b^c^d)+k[5]-378558|0;a=(a<<4|a>>>28)+b|0;d+=(a^b^c)+k[8]-2022574463|0;d=(d<<11|d>>>21)+a|0;c+=(d^a^b)+k[11]+1839030562|0;c=(c<<16|c>>>16)+d|0;b+=(c^d^a)+k[14]-35309556|0;b=(b<<23|b>>>9)+c|0;a+=(b^c^d)+k[1]-1530992060|0;a=(a<<4|a>>>28)+b|0;d+=(a^b^c)+k[4]+1272893353|0;d=(d<<11|d>>>21)+a|0;c+=(d^a^b)+k[7]-155497632|0;c=(c<<16|c>>>16)+d|0;b+=(c^d^a)+k[10]-1094730640|0;b=(b<<23|b>>>9)+c|0;a+=(b^c^d)+k[13]+681279174|0;a=(a<<4|a>>>28)+b|0;d+=(a^b^c)+k[0]-358537222|0;d=(d<<11|d>>>21)+a|0;c+=(d^a^b)+k[3]-722521979|0;c=(c<<16|c>>>16)+d|0;b+=(c^d^a)+k[6]+76029189|0;b=(b<<23|b>>>9)+c|0;a+=(b^c^d)+k[9]-640364487|0;a=(a<<4|a>>>28)+b|0;d+=(a^b^c)+k[12]-421815835|0;d=(d<<11|d>>>21)+a|0;c+=(d^a^b)+k[15]+530742520|0;c=(c<<16|c>>>16)+d|0;b+=(c^d^a)+k[2]-995338651|0;b=(b<<23|b>>>9)+c|0;a+=(c^(b|~d))+k[0]-198630844|0;a=(a<<6|a>>>26)+b|0;d+=(b^(a|~c))+k[7]+1126891415|0;d=(d<<10|d>>>22)+a|0;c+=(a^(d|~b))+k[14]-1416354905|0;c=(c<<15|c>>>17)+d|0;b+=(d^(c|~a))+k[5]-57434055|0;b=(b<<21|b>>>11)+c|0;a+=(c^(b|~d))+k[12]+1700485571|0;a=(a<<6|a>>>26)+b|0;d+=(b^(a|~c))+k[3]-1894986606|0;d=(d<<10|d>>>22)+a|0;c+=(a^(d|~b))+k[10]-1051523|0;c=(c<<15|c>>>17)+d|0;b+=(d^(c|~a))+k[1]-2054922799|0;b=(b<<21|b>>>11)+c|0;a+=(c^(b|~d))+k[8]+1873313359|0;a=(a<<6|a>>>26)+b|0;d+=(b^(a|~c))+k[15]-30611744|0;d=(d<<10|d>>>22)+a|0;c+=(a^(d|~b))+k[6]-1560198380|0;c=(c<<15|c>>>17)+d|0;b+=(d^(c|~a))+k[13]+1309151649|0;b=(b<<21|b>>>11)+c|0;a+=(c^(b|~d))+k[4]-145523070|0;a=(a<<6|a>>>26)+b|0;d+=(b^(a|~c))+k[11]-1120210379|0;d=(d<<10|d>>>22)+a|0;c+=(a^(d|~b))+k[2]+718787259|0;c=(c<<15|c>>>17)+d|0;b+=(d^(c|~a))+k[9]-343485551|0;b=(b<<21|b>>>11)+c|0;x[0]=a+x[0]|0;x[1]=b+x[1]|0;x[2]=c+x[2]|0;x[3]=d+x[3]|0}function md5blk(s){var md5blks=[],i;for(i=0;i<64;i+=4){md5blks[i>>2]=s.charCodeAt(i)+(s.charCodeAt(i+1)<<8)+(s.charCodeAt(i+2)<<16)+(s.charCodeAt(i+3)<<24)}return md5blks}function md5blk_array(a){var md5blks=[],i;for(i=0;i<64;i+=4){md5blks[i>>2]=a[i]+(a[i+1]<<8)+(a[i+2]<<16)+(a[i+3]<<24)}return md5blks}function md51(s){var n=s.length,state=[1732584193,-271733879,-1732584194,271733878],i,length,tail,tmp,lo,hi;for(i=64;i<=n;i+=64){md5cycle(state,md5blk(s.substring(i-64,i)))}s=s.substring(i-64);length=s.length;tail=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0];for(i=0;i>2]|=s.charCodeAt(i)<<(i%4<<3)}tail[i>>2]|=128<<(i%4<<3);if(i>55){md5cycle(state,tail);for(i=0;i<16;i+=1){tail[i]=0}}tmp=n*8;tmp=tmp.toString(16).match(/(.*?)(.{0,8})$/);lo=parseInt(tmp[2],16);hi=parseInt(tmp[1],16)||0;tail[14]=lo;tail[15]=hi;md5cycle(state,tail);return state}function md51_array(a){var n=a.length,state=[1732584193,-271733879,-1732584194,271733878],i,length,tail,tmp,lo,hi;for(i=64;i<=n;i+=64){md5cycle(state,md5blk_array(a.subarray(i-64,i)))}a=i-64>2]|=a[i]<<(i%4<<3)}tail[i>>2]|=128<<(i%4<<3);if(i>55){md5cycle(state,tail);for(i=0;i<16;i+=1){tail[i]=0}}tmp=n*8;tmp=tmp.toString(16).match(/(.*?)(.{0,8})$/);lo=parseInt(tmp[2],16);hi=parseInt(tmp[1],16)||0;tail[14]=lo;tail[15]=hi;md5cycle(state,tail);return state}function rhex(n){var s="",j;for(j=0;j<4;j+=1){s+=hex_chr[n>>j*8+4&15]+hex_chr[n>>j*8&15]}return s}function hex(x){var i;for(i=0;i>16)+(y>>16)+(lsw>>16);return msw<<16|lsw&65535}}if(typeof ArrayBuffer!=="undefined"&&!ArrayBuffer.prototype.slice){(function(){function clamp(val,length){val=val|0||0;if(val<0){return Math.max(val+length,0)}return Math.min(val,length)}ArrayBuffer.prototype.slice=function(from,to){var length=this.byteLength,begin=clamp(from,length),end=length,num,target,targetArray,sourceArray;if(to!==undefined){end=clamp(to,length)}if(begin>end){return new ArrayBuffer(0)}num=end-begin;target=new ArrayBuffer(num);targetArray=new Uint8Array(target);sourceArray=new Uint8Array(this,begin,num);targetArray.set(sourceArray);return target}})()}function toUtf8(str){if(/[\u0080-\uFFFF]/.test(str)){str=unescape(encodeURIComponent(str))}return str}function utf8Str2ArrayBuffer(str,returnUInt8Array){var length=str.length,buff=new ArrayBuffer(length),arr=new Uint8Array(buff),i;for(i=0;i>2]|=buff.charCodeAt(i)<<(i%4<<3)}this._finish(tail,length);ret=hex(this._hash);if(raw){ret=hexToBinaryString(ret)}this.reset();return ret};SparkMD5.prototype.reset=function(){this._buff="";this._length=0;this._hash=[1732584193,-271733879,-1732584194,271733878];return this};SparkMD5.prototype.getState=function(){return{buff:this._buff,length:this._length,hash:this._hash.slice()}};SparkMD5.prototype.setState=function(state){this._buff=state.buff;this._length=state.length;this._hash=state.hash;return this};SparkMD5.prototype.destroy=function(){delete this._hash;delete this._buff;delete this._length};SparkMD5.prototype._finish=function(tail,length){var i=length,tmp,lo,hi;tail[i>>2]|=128<<(i%4<<3);if(i>55){md5cycle(this._hash,tail);for(i=0;i<16;i+=1){tail[i]=0}}tmp=this._length*8;tmp=tmp.toString(16).match(/(.*?)(.{0,8})$/);lo=parseInt(tmp[2],16);hi=parseInt(tmp[1],16)||0;tail[14]=lo;tail[15]=hi;md5cycle(this._hash,tail)};SparkMD5.hash=function(str,raw){return SparkMD5.hashBinary(toUtf8(str),raw)};SparkMD5.hashBinary=function(content,raw){var hash=md51(content),ret=hex(hash);return raw?hexToBinaryString(ret):ret};SparkMD5.ArrayBuffer=function(){this.reset()};SparkMD5.ArrayBuffer.prototype.append=function(arr){var buff=concatenateArrayBuffers(this._buff.buffer,arr,true),length=buff.length,i;this._length+=arr.byteLength;for(i=64;i<=length;i+=64){md5cycle(this._hash,md5blk_array(buff.subarray(i-64,i)))}this._buff=i-64>2]|=buff[i]<<(i%4<<3)}this._finish(tail,length);ret=hex(this._hash);if(raw){ret=hexToBinaryString(ret)}this.reset();return ret};SparkMD5.ArrayBuffer.prototype.reset=function(){this._buff=new Uint8Array(0);this._length=0;this._hash=[1732584193,-271733879,-1732584194,271733878];return this};SparkMD5.ArrayBuffer.prototype.getState=function(){var state=SparkMD5.prototype.getState.call(this);state.buff=arrayBuffer2Utf8Str(state.buff);return state};SparkMD5.ArrayBuffer.prototype.setState=function(state){state.buff=utf8Str2ArrayBuffer(state.buff,true);return SparkMD5.prototype.setState.call(this,state)};SparkMD5.ArrayBuffer.prototype.destroy=SparkMD5.prototype.destroy;SparkMD5.ArrayBuffer.prototype._finish=SparkMD5.prototype._finish;SparkMD5.ArrayBuffer.hash=function(arr,raw){var hash=md51_array(new Uint8Array(arr)),ret=hex(hash);return raw?hexToBinaryString(ret):ret};return SparkMD5}); diff --git a/src/test/java/org/ycloud/aipan/AmazonS3ClientTests.java b/src/test/java/org/ycloud/aipan/AmazonS3ClientTests.java index fa7363a..1cff506 100644 --- a/src/test/java/org/ycloud/aipan/AmazonS3ClientTests.java +++ b/src/test/java/org/ycloud/aipan/AmazonS3ClientTests.java @@ -41,7 +41,7 @@ class AmazonS3ClientTests { */ @Test public void testCreateBucket() { - String bucketName = "avatar"; + String bucketName = "ai-pan"; Bucket bucket = amazonS3Client.createBucket(bucketName); log.info("bucket:{}", bucket); } diff --git a/src/test/java/org/ycloud/aipan/BigFileUploadTest.java b/src/test/java/org/ycloud/aipan/BigFileUploadTest.java new file mode 100644 index 0000000..38ba0c8 --- /dev/null +++ b/src/test/java/org/ycloud/aipan/BigFileUploadTest.java @@ -0,0 +1,150 @@ +package org.ycloud.aipan; + +import com.amazonaws.HttpMethod; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.*; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; + +@SpringBootTest +@Slf4j +public class BigFileUploadTest { + + + @Autowired + private AmazonS3Client amazonS3Client; + + //=====================大文件上传相关接口=========================== + + /** + * 第一步：初始化大文件分片上传任务，获取uploadId + * 如果初始化时有 uploadId，说明是断点续传，不能重新生成 uploadId + */ + @Test + public void testInitiateMultipartUploadTask() { + String bucketName = "ai-pan"; + String objectKey = "/meta/test5.txt"; + + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentType("text/plain"); +// objectMetadata.setContentType("application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + //初始化分片上传请求 + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, objectKey, objectMetadata); + + //初始化分片上传任务 + InitiateMultipartUploadResult uploadResult = amazonS3Client.initiateMultipartUpload(initRequest); + String uploadId = uploadResult.getUploadId(); + log.info("uploadId:{}", uploadId); + + } + + @Test + public void testGenePreSignedUrls() { + // 定义对象键名 + String objectKey = "/meta/test5.txt"; + // 定义存储桶名称 + String bucket = "ai-pan"; + // 定义分片数量，这里设置为4个分片 + int chunkCount = 4; + + String uploadId = "NzViMDY5NGUtM2IwOC00ZDhkLTk1ODMtM2EyYThhZGFmNmI3LjVhMDlmZDJkLWE2ODgtNDgwNS1hODQwLWZhMTEwNWI0NTUwZA"; + + // 创建用于存储分片URL的列表 + List partList = new ArrayList<>(); + // 遍历每个分片，生成预签名的URL + for (int i = 1; i <= chunkCount; i++) { + // 生成预签名的 URL, 设置过期时间，例如 1 小时后 + Date expiration = new Date(System.currentTimeMillis() + 3600 * 1000); + // 创建生成预签名URL的请求，并指定HTTP方法为PUT + GeneratePresignedUrlRequest genePreSignedUrlReq = new GeneratePresignedUrlRequest(bucket, objectKey, HttpMethod.PUT).withExpiration(expiration); + // 添加上传ID和分片编号作为请求参数 + genePreSignedUrlReq.addRequestParameter("uploadId", uploadId); + genePreSignedUrlReq.addRequestParameter("partNumber", String.valueOf(i)); + // 生成并获取预签名URL + URL url = amazonS3Client.generatePresignedUrl(genePreSignedUrlReq); + // 将生成的URL添加到列表中 + partList.add(url.toString()); + // 日志输出当前分片的URL列表 + log.info("partList:{}", partList); + } + } + + + // 测试合并分片的方法 + @Test + public void testMergeChunks() { + // 定义对象键名 + String objectKey = "/meta/test5.txt"; + // 定义存储桶名称 + String bucket = "ai-pan"; + // 定义分片数量，这里设置为4个分片 + int chunkCount = 4; + // 定义上传ID，用于标识特定的分片上传事件 + String uploadId = "NzViMDY5NGUtM2IwOC00ZDhkLTk1ODMtM2EyYThhZGFmNmI3LjVhMDlmZDJkLWE2ODgtNDgwNS1hODQwLWZhMTEwNWI0NTUwZA"; + + // 创建一个列出分片请求对象 + ListPartsRequest listPartsRequest = new ListPartsRequest(bucket, objectKey, uploadId); + // 获取分片列表 + PartListing partListing = amazonS3Client.listParts(listPartsRequest); + List parts = partListing.getParts(); + // 检查分片数量是否与预期一致 + if (chunkCount != parts.size()) { + // 已上传分块数量与记录中的数量不对应，不能合并分片 + throw new RuntimeException("分片缺失，请重新上传"); + } + + // 创建一个完成分片上传请求对象 + CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest() + .withUploadId(uploadId) + .withKey(objectKey) + .withBucketName(bucket) + .withPartETags(parts.stream() + // 将每个分片的编号和ETag封装到PartETag对象中 + .map(partSummary -> new PartETag(partSummary.getPartNumber(), partSummary.getETag())) + .collect(Collectors.toList())); + + // 完成分片上传并获取结果 + CompleteMultipartUploadResult result = amazonS3Client.completeMultipartUpload(completeMultipartUploadRequest); + log.info("result:{}", result.getBucketName()); + } + + @Test + // 测试列出分片上传的各个分片信息 + public void testListParts() { + // 定义对象键名 + String objectKey = "/meta/test5.txt"; + // 定义存储桶名称 + String bucket = "ai-pan"; + // 定义上传ID，用于标识特定的分片上传事件 + String uploadId = "ZjFkZjRhN2UtNzMzOS04NTUxLTgwOTEtNWViNzUwNmRmYTEzLmE4NTUyMmQyLTM1NjUtNGMwMS05ZTY2LWQ5MWQ4NDUyBmIyA"; + + // 检查指定的存储桶中是否存在具有指定对象键名的对象 + boolean doesObjectExist = amazonS3Client.doesObjectExist(bucket, objectKey); + if (!doesObjectExist) { + // 未上传完，返回已上传的分片 + ListPartsRequest listPartsRequest = new ListPartsRequest(bucket, objectKey, uploadId); + PartListing partListing = amazonS3Client.listParts(listPartsRequest); + List parts = partListing.getParts(); + // 创建一个结果映射，用于存放上传状态和分片列表 + Map result = new HashMap<>(); + result.put("finished", false); + result.put("exitPartList", parts); + //前端可以通过这个判断是否要调用合并merge接口 + log.info("result:{}", result); + + // 遍历并打印每个分片的信息 + for (PartSummary partSummary : parts) { + System.out.println("getPartNumber：" + partSummary.getPartNumber() + "，getETag=" + partSummary.getETag() + "，getSize= " + partSummary.getSize() + "，getLastModified=" + partSummary.getLastModified()); + } + // 打印存储桶名称 + System.out.println(partListing.getBucketName()); + } + + } +} diff --git a/src/test/java/org/ycloud/aipan/FileChunkUploadTests.java b/src/test/java/org/ycloud/aipan/FileChunkUploadTests.java new file mode 100644 index 0000000..1394af7 --- /dev/null +++ b/src/test/java/org/ycloud/aipan/FileChunkUploadTests.java @@ -0,0 +1,158 @@ +package org.ycloud.aipan; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.FileEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.assertj.core.util.Lists; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.ycloud.aipan.controller.req.FileChunkInitTaskReq; +import org.ycloud.aipan.controller.req.FileChunkMergeReq; +import org.ycloud.aipan.dto.FileChunkDTO; +import org.ycloud.aipan.service.FileChunkService; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; + +@SpringBootTest +@Slf4j +class FileChunkUploadTests { + + @Autowired + private FileChunkService fileChunkService; + + private Long accountId = 3L; + + private String identifier = "abcsdfsd"; + + /** + * 存储分片后端的文件路径和名称 + */ + private final List chunkFilePaths = Lists.newArrayList(); + + /** + * 存储分片上传地址 + */ + private final List chunkUploadUrls = Lists.newArrayList(); + + /** + * 上传ID + */ + private String uploadId; + + /** + * 分片大小，5MB + */ + private final long chunkSize = 5 * 1024 * 1024; + + /** + * 用一个10MB以上的文件，按5MB分片大小进行分片 + */ + + @Test + public void testCreateChunkFiles() { + + // 将文件分片存储 + String filePath = "/Users/xdclass/Desktop/chunk/es_note.pdf"; + File file = new File(filePath); + long fileSize = file.length(); + //int chunkCount = (int) Math.ceil((double) fileSize / CHUNK_SIZE); + int chunkCount = (int) Math.ceil(fileSize * 1.0 / chunkSize); + log.info("创建分片数量是: {} chunks", chunkCount); + try (FileInputStream fis = new FileInputStream(file)) { + byte[] buffer = new byte[(int) chunkSize]; + for (int i = 0; i < chunkCount; i++) { + String chunkFileName = filePath + ".part" + (i + 1); + try (FileOutputStream fos = new FileOutputStream(chunkFileName)) { + int bytesRead = fis.read(buffer); + fos.write(buffer, 0, bytesRead); + log.info("创建的分片文件名: {} ({} bytes)", chunkFileName, bytesRead); + chunkFilePaths.add(chunkFileName); + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + + /** + * 第1步，创建分片上传任务 + */ + private void testInitFileChunkTask() { + FileChunkInitTaskReq req = new FileChunkInitTaskReq(); + req.setAccountId(accountId).setFilename("es_note.pdf") + .setTotalSize((long) (20552959))//20552959 + .setChunkSize((long) (5 * 1024 * 1024))//5242880 + .setIdentifier(identifier); + FileChunkDTO fileChunkDTO = fileChunkService.initFileChunkTask(req); + log.info("分片上传初始化结果: {}", fileChunkDTO); + + uploadId = fileChunkDTO.getUploadId(); + + testGetFileChunkUploadUrl(); + } + /** + * 第2步，获取分片上传地址，返回临时MinIO地址，前端直接上传到Minio里面 + */ + private void testGetFileChunkUploadUrl() { + + for (int i = 1; i <= chunkFilePaths.size(); i++) { + String uploadUrl = fileChunkService.genPreSignUploadUrl(accountId, identifier, i); + log.info("分片上传地址: {}", uploadUrl); + //存储4个分片地址 + chunkUploadUrls.add(uploadUrl); + } + + uploadChunk(); + } + + /** + * 模拟前端直接上传分片 + */ + @SneakyThrows + private void uploadChunk() { + CloseableHttpClient httpClient = HttpClients.createDefault(); + for (int i = 0; i < chunkUploadUrls.size(); i++) { + // PUT直接上传到minio + String chunkUploadId = chunkUploadUrls.get(i); + HttpPut httpPut = new HttpPut(chunkUploadId); + httpPut.setHeader("Content-Type","application/octet-stream"); + File chunkFile = new File(chunkFilePaths.get(i)); + FileEntity chunkFileEntity = new FileEntity(chunkFile); + httpPut.setEntity(chunkFileEntity); + CloseableHttpResponse chunkUploadResp = httpClient.execute(httpPut); + httpPut.releaseConnection(); + } + } + + /** + * 测试合并分片 + */ + @Test + public void testMergeFileChunk() { + FileChunkMergeReq req = new FileChunkMergeReq(); + req.setAccountId(accountId).setIdentifier(identifier).setParentId(233L); + fileChunkService.mergeFileChunk(req); + } + + + /** + * 查询分片上传进度 + */ + @Test + public void testChunkUploadProgress() { + FileChunkDTO fileChunkDTO = fileChunkService.listFileChunk(accountId, identifier); + log.info("分片上传进度: {}", fileChunkDTO); + } +} \ No newline at end of file