22 changed files with 1295 additions and 38 deletions
@ -0,0 +1,26 @@ |
|||||
|
package org.ycloud.aipan.controller.req; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import lombok.experimental.Accessors; |
||||
|
|
||||
|
@Data |
||||
|
@Accessors(chain = true) |
||||
|
public class FileChunkInitTaskReq { |
||||
|
|
||||
|
private Long accountId; |
||||
|
|
||||
|
private String filename; |
||||
|
|
||||
|
private String identifier; |
||||
|
|
||||
|
/*** |
||||
|
* 总大小 |
||||
|
*/ |
||||
|
private Long totalSize; |
||||
|
|
||||
|
/** |
||||
|
* 分片大小 |
||||
|
*/ |
||||
|
private Long chunkSize; |
||||
|
|
||||
|
} |
@ -0,0 +1,17 @@ |
|||||
|
package org.ycloud.aipan.controller.req; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import lombok.experimental.Accessors; |
||||
|
|
||||
|
|
||||
|
@Data |
||||
|
@Accessors(chain = true) |
||||
|
public class FileChunkMergeReq { |
||||
|
|
||||
|
private String identifier; |
||||
|
|
||||
|
private Long parentId; |
||||
|
|
||||
|
private Long accountId; |
||||
|
|
||||
|
} |
@ -0,0 +1,66 @@ |
|||||
|
package org.ycloud.aipan.dto; |
||||
|
|
||||
|
import com.amazonaws.services.s3.model.PartSummary; |
||||
|
import io.swagger.v3.oas.annotations.media.Schema; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import lombok.experimental.Accessors; |
||||
|
import org.ycloud.aipan.model.FileChunkDO; |
||||
|
import org.ycloud.aipan.util.SpringBeanUtil; |
||||
|
|
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
|
||||
|
@Data |
||||
|
@NoArgsConstructor |
||||
|
@Accessors(chain = true) |
||||
|
public class FileChunkDTO { |
||||
|
|
||||
|
|
||||
|
public FileChunkDTO(FileChunkDO fileChunkDO){ |
||||
|
SpringBeanUtil.copyProperties(fileChunkDO,this); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private Long id; |
||||
|
|
||||
|
@Schema(description = "文件唯一标识(md5)") |
||||
|
private String identifier; |
||||
|
|
||||
|
@Schema(description = "分片上传ID") |
||||
|
private String uploadId; |
||||
|
|
||||
|
@Schema(description = "文件名") |
||||
|
private String fileName; |
||||
|
|
||||
|
@Schema(description = "所属桶名") |
||||
|
private String bucketName; |
||||
|
|
||||
|
@Schema(description = "文件的key") |
||||
|
private String objectKey; |
||||
|
|
||||
|
@Schema(description = "总文件大小(byte)") |
||||
|
private Long totalSize; |
||||
|
|
||||
|
@Schema(description = "每个分片大小(byte)") |
||||
|
private Long chunkSize; |
||||
|
|
||||
|
@Schema(description = "分片数量") |
||||
|
private Integer chunkNum; |
||||
|
|
||||
|
@Schema(description = "用户ID") |
||||
|
private Long accountId; |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* 是否完成上传 |
||||
|
*/ |
||||
|
private boolean finished; |
||||
|
|
||||
|
/** |
||||
|
* 返回已经存在的分片 |
||||
|
*/ |
||||
|
private List<PartSummary> exitPartList; |
||||
|
|
||||
|
} |
@ -0,0 +1,28 @@ |
|||||
|
package org.ycloud.aipan.service; |
||||
|
|
||||
|
|
||||
|
import org.ycloud.aipan.controller.req.FileChunkInitTaskReq; |
||||
|
import org.ycloud.aipan.controller.req.FileChunkMergeReq; |
||||
|
import org.ycloud.aipan.dto.FileChunkDTO; |
||||
|
|
||||
|
public interface FileChunkService { |
||||
|
/** |
||||
|
* 初始化分片上传 |
||||
|
*/ |
||||
|
FileChunkDTO initFileChunkTask(FileChunkInitTaskReq req); |
||||
|
|
||||
|
/** |
||||
|
* 获取临时文件上传地址 |
||||
|
*/ |
||||
|
String genPreSignUploadUrl(Long accountId, String identifier, Integer partNumber); |
||||
|
|
||||
|
/** |
||||
|
* 合并分片 |
||||
|
*/ |
||||
|
void mergeFileChunk(FileChunkMergeReq req); |
||||
|
|
||||
|
/** |
||||
|
* 查询分片上传进度 |
||||
|
*/ |
||||
|
FileChunkDTO listFileChunk(Long accountId, String identifier); |
||||
|
} |
@ -0,0 +1,172 @@ |
|||||
|
package org.ycloud.aipan.service.impl; |
||||
|
|
||||
|
import cn.hutool.core.date.DateUtil; |
||||
|
import com.amazonaws.HttpMethod; |
||||
|
import com.amazonaws.services.s3.model.ObjectMetadata; |
||||
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.http.MediaType; |
||||
|
import org.springframework.http.MediaTypeFactory; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.transaction.annotation.Transactional; |
||||
|
import org.ycloud.aipan.component.StoreEngine; |
||||
|
import org.ycloud.aipan.config.MinioConfig; |
||||
|
import org.ycloud.aipan.controller.req.FileChunkInitTaskReq; |
||||
|
import org.ycloud.aipan.controller.req.FileChunkMergeReq; |
||||
|
import org.ycloud.aipan.controller.req.FileUploadReq; |
||||
|
import org.ycloud.aipan.dto.FileChunkDTO; |
||||
|
import org.ycloud.aipan.enums.BizCodeEnum; |
||||
|
import org.ycloud.aipan.exception.BizException; |
||||
|
import org.ycloud.aipan.mapper.FileChunkMapper; |
||||
|
import org.ycloud.aipan.mapper.StorageMapper; |
||||
|
import org.ycloud.aipan.model.FileChunkDO; |
||||
|
import org.ycloud.aipan.model.StorageDO; |
||||
|
import org.ycloud.aipan.service.AccountFileService; |
||||
|
import org.ycloud.aipan.service.FileChunkService; |
||||
|
import org.ycloud.aipan.util.CommonUtil; |
||||
|
import com.amazonaws.services.s3.model.*; |
||||
|
import java.net.URL; |
||||
|
import java.util.*; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
@Service |
||||
|
@Slf4j |
||||
|
public class FileChunkServiceImpl implements FileChunkService { |
||||
|
@Autowired |
||||
|
private StorageMapper storageMapper; |
||||
|
@Autowired |
||||
|
private StoreEngine fileStoreEngine; |
||||
|
@Autowired |
||||
|
private MinioConfig minioConfig; |
||||
|
@Autowired |
||||
|
private FileChunkMapper fileChunkMapper; |
||||
|
@Autowired |
||||
|
private AccountFileService accountFileService; |
||||
|
|
||||
|
@Override |
||||
|
@Transactional(rollbackFor = Exception.class) |
||||
|
public FileChunkDTO initFileChunkTask(FileChunkInitTaskReq req) { |
||||
|
//检查存储空间是否够
|
||||
|
StorageDO storageDO = storageMapper.selectOne(new QueryWrapper<StorageDO>().eq("account_id", req.getAccountId())); |
||||
|
if (storageDO.getUsedSize() + req.getTotalSize() > storageDO.getTotalSize()) { |
||||
|
throw new BizException(BizCodeEnum.FILE_STORAGE_NOT_ENOUGH); |
||||
|
} |
||||
|
|
||||
|
String objectKey = CommonUtil.getFilePath(req.getFilename()); |
||||
|
// 根据文件名推断内容类型
|
||||
|
String contentType = MediaTypeFactory.getMediaType(objectKey).orElse(MediaType.APPLICATION_OCTET_STREAM).toString(); |
||||
|
// 设置文件元数据
|
||||
|
ObjectMetadata objectMetadata = new ObjectMetadata(); |
||||
|
objectMetadata.setContentType(contentType); |
||||
|
// 初始化分片上传,获取上传ID
|
||||
|
String uploadId = fileStoreEngine.initMultipartUploadTask(minioConfig.getBucketName(), objectKey, objectMetadata).getUploadId(); |
||||
|
|
||||
|
// 创建上传任务实体并设置相关属性
|
||||
|
FileChunkDO task = new FileChunkDO(); |
||||
|
int chunkNum = (int) Math.ceil(req.getTotalSize() * 1.0 / req.getChunkSize()); |
||||
|
task.setBucketName(minioConfig.getBucketName()) |
||||
|
.setChunkNum(chunkNum) |
||||
|
.setChunkSize(req.getChunkSize()) |
||||
|
.setTotalSize(req.getTotalSize()) |
||||
|
.setIdentifier(req.getIdentifier()) |
||||
|
.setFileName(req.getFilename()) |
||||
|
.setObjectKey(objectKey) |
||||
|
.setUploadId(uploadId) |
||||
|
.setAccountId(req.getAccountId()); |
||||
|
// 将任务插入数据库
|
||||
|
fileChunkMapper.insert(task); |
||||
|
// 构建并返回任务信息DTO
|
||||
|
return new FileChunkDTO(task).setFinished(false).setExitPartList(new ArrayList<>()); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public String genPreSignUploadUrl(Long accountId, String identifier, Integer partNumber) { |
||||
|
FileChunkDO task = fileChunkMapper.selectOne(new QueryWrapper<FileChunkDO>().lambda().eq(FileChunkDO::getIdentifier, identifier).eq(FileChunkDO::getAccountId, accountId)); |
||||
|
if (task == null) { |
||||
|
throw new BizException(BizCodeEnum.FILE_CHUNK_TASK_NOT_EXISTS); |
||||
|
} |
||||
|
//配置预签名过期时间
|
||||
|
Date expireDate = DateUtil.offsetMillisecond(new Date(), minioConfig.getPRE_SIGN_URL_EXPIRE().intValue()); |
||||
|
// 生成预签名URL
|
||||
|
Map<String, Object> params = new HashMap<>(); |
||||
|
params.put("partNumber", partNumber.toString()); |
||||
|
params.put("uploadId", task.getUploadId()); |
||||
|
URL preSignedUrl = fileStoreEngine.genePreSignedUrl(minioConfig.getBucketName(), task.getObjectKey(), HttpMethod.PUT, expireDate, params); |
||||
|
log.info("生成预签名URL地址 identifier={},partNumber={}, preSignedUrl={}", identifier, partNumber, preSignedUrl.toString()); |
||||
|
return preSignedUrl.toString(); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
public void mergeFileChunk(FileChunkMergeReq req) { |
||||
|
//获取任务和分片列表,检查是否足够合并
|
||||
|
FileChunkDO task = fileChunkMapper.selectOne(new QueryWrapper<FileChunkDO>() |
||||
|
.eq("account_id", req.getAccountId()) |
||||
|
.eq("identifier", req.getIdentifier())); |
||||
|
if(task == null){ |
||||
|
throw new BizException(BizCodeEnum.FILE_CHUNK_TASK_NOT_EXISTS); |
||||
|
} |
||||
|
PartListing partListing = fileStoreEngine.listMultipart(task.getBucketName(), task.getObjectKey(), task.getUploadId()); |
||||
|
List<PartSummary> parts = partListing.getParts(); |
||||
|
if(parts.size() != task.getChunkNum()){ |
||||
|
//上传的分片数量和记录中不对应,合并失败
|
||||
|
throw new BizException(BizCodeEnum.FILE_CHUNK_NOT_ENOUGH); |
||||
|
} |
||||
|
//检查更新存储空间
|
||||
|
StorageDO storageDO = storageMapper.selectOne(new QueryWrapper<>(new StorageDO()) |
||||
|
.eq("account_id", req.getAccountId())); |
||||
|
long realFileTotalSize = parts.stream().map(PartSummary::getSize).mapToLong(Long::valueOf).sum(); |
||||
|
if(storageDO.getUsedSize() + realFileTotalSize > storageDO.getTotalSize()){ |
||||
|
throw new BizException(BizCodeEnum.FILE_STORAGE_NOT_ENOUGH); |
||||
|
} |
||||
|
storageDO.setUsedSize(storageDO.getUsedSize() +realFileTotalSize); |
||||
|
storageMapper.updateById(storageDO); |
||||
|
|
||||
|
//2-合并文件
|
||||
|
CompleteMultipartUploadResult result = fileStoreEngine.mergeChunks(task.getBucketName(), |
||||
|
task.getObjectKey(), task.getUploadId(), |
||||
|
parts.stream().map(partSummary -> |
||||
|
new PartETag(partSummary.getPartNumber(), partSummary.getETag())) |
||||
|
.collect(Collectors.toList())); |
||||
|
//【判断是否合并成功
|
||||
|
if(result.getETag()!=null){ |
||||
|
FileUploadReq fileUploadReq = new FileUploadReq(); |
||||
|
fileUploadReq.setAccountId(req.getAccountId()) |
||||
|
.setFilename(task.getFileName()) |
||||
|
.setIdentifier(task.getIdentifier()) |
||||
|
.setParentId(req.getParentId()) |
||||
|
.setFileSize(realFileTotalSize) |
||||
|
.setFile(null); |
||||
|
|
||||
|
//存储文件和关联信息到数据库
|
||||
|
accountFileService.saveFileAndAccountFile(fileUploadReq,task.getObjectKey()); |
||||
|
//删除相关任务记录
|
||||
|
fileChunkMapper.deleteById(task.getId()); |
||||
|
log.info("合并成功"); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public FileChunkDTO listFileChunk(Long accountId, String identifier) { |
||||
|
// 获取任务和分片列表,检查是否足够
|
||||
|
FileChunkDO task = fileChunkMapper.selectOne(new QueryWrapper<FileChunkDO>().lambda().eq(FileChunkDO::getAccountId, accountId)); |
||||
|
if (task == null || !identifier.equals(task.getIdentifier())) { |
||||
|
return null; |
||||
|
} |
||||
|
FileChunkDTO result = new FileChunkDTO(task); |
||||
|
boolean doesObjectExist = fileStoreEngine.doesObjectExist(task.getBucketName(), task.getObjectKey()); |
||||
|
if (!doesObjectExist) { |
||||
|
// 不存在,表示未上传完,返回已上传的分片
|
||||
|
PartListing partListing = fileStoreEngine.listMultipart(task.getBucketName(), task.getObjectKey(), task.getUploadId()); |
||||
|
if(task.getChunkNum() == partListing.getParts().size()){ |
||||
|
//已经存在,合并
|
||||
|
result.setFinished(true).setExitPartList(partListing.getParts()); |
||||
|
}else { |
||||
|
result.setFinished(false).setExitPartList(partListing.getParts()); |
||||
|
} |
||||
|
} |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
} |
@ -0,0 +1,425 @@ |
|||||
|
<!DOCTYPE html> |
||||
|
<html lang="zh-CN"> |
||||
|
<head> |
||||
|
<meta charset="UTF-8"/> |
||||
|
<title>大文件上传(断点续传版)</title> |
||||
|
<style> |
||||
|
/* 原有样式不变,只新增 .hidden 控制隐藏 */ |
||||
|
* { |
||||
|
margin: 0; |
||||
|
padding: 0; |
||||
|
box-sizing: border-box |
||||
|
} |
||||
|
|
||||
|
body { |
||||
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif; |
||||
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); |
||||
|
display: flex; |
||||
|
min-height: 100vh; |
||||
|
align-items: center; |
||||
|
justify-content: center; |
||||
|
padding: 20px |
||||
|
} |
||||
|
|
||||
|
.upload-container { |
||||
|
background: rgba(255, 255, 255, .95); |
||||
|
border-radius: 20px; |
||||
|
padding: 40px; |
||||
|
width: 100%; |
||||
|
max-width: 600px; |
||||
|
box-shadow: 0 20px 40px rgba(0, 0, 0, .1) |
||||
|
} |
||||
|
|
||||
|
.upload-header { |
||||
|
text-align: center; |
||||
|
margin-bottom: 30px |
||||
|
} |
||||
|
|
||||
|
.upload-header h1 { |
||||
|
font-size: 32px; |
||||
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); |
||||
|
-webkit-background-clip: text; |
||||
|
-webkit-text-fill-color: transparent |
||||
|
} |
||||
|
|
||||
|
.upload-area { |
||||
|
border: 2px dashed #667eea; |
||||
|
border-radius: 15px; |
||||
|
padding: 60px 20px; |
||||
|
text-align: center; |
||||
|
cursor: pointer; |
||||
|
transition: all .3s; |
||||
|
position: relative |
||||
|
} |
||||
|
|
||||
|
.upload-area:hover { |
||||
|
border-color: #764ba2; |
||||
|
background: rgba(118, 75, 162, .05) |
||||
|
} |
||||
|
|
||||
|
.upload-area.dragover { |
||||
|
border-color: #764ba2; |
||||
|
background: rgba(118, 75, 162, .1); |
||||
|
transform: scale(1.02) |
||||
|
} |
||||
|
|
||||
|
.upload-icon { |
||||
|
font-size: 48px; |
||||
|
margin-bottom: 15px; |
||||
|
animation: float 3s ease-in-out infinite |
||||
|
} |
||||
|
|
||||
|
@keyframes float { |
||||
|
0%, 100% { |
||||
|
transform: translateY(0) |
||||
|
} |
||||
|
50% { |
||||
|
transform: translateY(-10px) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
.file-input { |
||||
|
display: none |
||||
|
} |
||||
|
|
||||
|
.file-info, .progress-container, .success-message, .error-message { |
||||
|
margin-top: 20px; |
||||
|
display: none |
||||
|
} |
||||
|
|
||||
|
.file-info.active, .progress-container.active { |
||||
|
display: block; |
||||
|
animation: fadeIn .3s |
||||
|
} |
||||
|
|
||||
|
@keyframes fadeIn { |
||||
|
from { |
||||
|
opacity: 0 |
||||
|
} |
||||
|
to { |
||||
|
opacity: 1 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
.progress-bar-bg { |
||||
|
background: #e0e0e0; |
||||
|
border-radius: 10px; |
||||
|
height: 20px; |
||||
|
overflow: hidden; |
||||
|
position: relative |
||||
|
} |
||||
|
|
||||
|
.progress-bar { |
||||
|
background: linear-gradient(90deg, #667eea 0%, #764ba2 100%); |
||||
|
height: 100%; |
||||
|
width: 0; |
||||
|
transition: width .3s; |
||||
|
position: relative; |
||||
|
overflow: hidden |
||||
|
} |
||||
|
|
||||
|
.progress-bar::after { |
||||
|
content: ''; |
||||
|
position: absolute; |
||||
|
top: 0; |
||||
|
left: 0; |
||||
|
bottom: 0; |
||||
|
right: 0; |
||||
|
background: linear-gradient(90deg, transparent, rgba(255, 255, 255, .3), transparent); |
||||
|
animation: shimmer 2s infinite |
||||
|
} |
||||
|
|
||||
|
@keyframes shimmer { |
||||
|
0% { |
||||
|
transform: translateX(-100%) |
||||
|
} |
||||
|
100% { |
||||
|
transform: translateX(100%) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
.progress-text { |
||||
|
text-align: center; |
||||
|
margin-top: 10px; |
||||
|
color: #666; |
||||
|
font-size: 14px |
||||
|
} |
||||
|
|
||||
|
.upload-btn { |
||||
|
margin-top: 20px; |
||||
|
padding: 12px 40px; |
||||
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); |
||||
|
color: #fff; |
||||
|
border: none; |
||||
|
border-radius: 25px; |
||||
|
font-size: 16px; |
||||
|
cursor: pointer; |
||||
|
transition: all .3s; |
||||
|
display: none |
||||
|
} |
||||
|
|
||||
|
.upload-btn:hover { |
||||
|
transform: translateY(-2px); |
||||
|
box-shadow: 0 5px 15px rgba(0, 0, 0, .2) |
||||
|
} |
||||
|
|
||||
|
.upload-btn.active { |
||||
|
display: inline-block; |
||||
|
animation: bounceIn .5s |
||||
|
} |
||||
|
|
||||
|
@keyframes bounceIn { |
||||
|
0% { |
||||
|
transform: scale(.8); |
||||
|
opacity: 0 |
||||
|
} |
||||
|
50% { |
||||
|
transform: scale(1.1) |
||||
|
} |
||||
|
100% { |
||||
|
transform: scale(1); |
||||
|
opacity: 1 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
.hidden { |
||||
|
display: none |
||||
|
} |
||||
|
|
||||
|
.loading-spinner { |
||||
|
display: none; |
||||
|
width: 40px; |
||||
|
height: 40px; |
||||
|
margin: 20px auto; |
||||
|
border: 4px solid #f3f3f3; |
||||
|
border-top: 4px solid #667eea; |
||||
|
border-radius: 50%; |
||||
|
animation: spin 1s linear infinite |
||||
|
} |
||||
|
|
||||
|
@keyframes spin { |
||||
|
0% { |
||||
|
transform: rotate(0deg) |
||||
|
} |
||||
|
100% { |
||||
|
transform: rotate(360deg) |
||||
|
} |
||||
|
} |
||||
|
</style> |
||||
|
</head> |
||||
|
<body> |
||||
|
<div class="upload-container"> |
||||
|
<div class="upload-header"> |
||||
|
<h1>大文件上传(断点续传)</h1> |
||||
|
<p>支持秒传、断点续传,最大支持2GB</p> |
||||
|
</div> |
||||
|
|
||||
|
<!-- 拖拽区域 --> |
||||
|
<div class="upload-area" id="uploadArea"> |
||||
|
<div class="upload-icon">📁</div> |
||||
|
<p>点击或拖拽文件到此处上传</p> |
||||
|
<input type="file" id="fileInput" class="file-input"/> |
||||
|
</div> |
||||
|
|
||||
|
<!-- 文件信息 --> |
||||
|
<div class="file-info" id="fileInfo"> |
||||
|
<div class="file-name" id="fileName"></div> |
||||
|
<div class="file-size" id="fileSize"></div> |
||||
|
</div> |
||||
|
|
||||
|
<!-- 进度 --> |
||||
|
<div class="progress-container" id="progressContainer"> |
||||
|
<div class="progress-bar-bg"> |
||||
|
<div class="progress-bar" id="progressBar"></div> |
||||
|
</div> |
||||
|
<div class="progress-text" id="progressText">0%</div> |
||||
|
<div class="chunk-info" id="chunkInfo"></div> |
||||
|
</div> |
||||
|
|
||||
|
<!-- 操作按钮 --> |
||||
|
<button class="upload-btn" id="uploadBtn">开始上传</button> |
||||
|
<div class="loading-spinner" id="loadingSpinner"></div> |
||||
|
|
||||
|
<!-- 结果提示 --> |
||||
|
<div class="success-message" id="successMessage">✅ 上传成功</div> |
||||
|
<div class="error-message" id="errorMessage">❌ 上传失败</div> |
||||
|
</div> |
||||
|
|
||||
|
<!-- SparkMD5 计算 MD5 --> |
||||
|
<script src="https://cdn.jsdelivr.net/npm/spark-md5@3.0.2/spark-md5.min.js"></script> |
||||
|
<script> |
||||
|
const uploadArea = document.getElementById('uploadArea'); |
||||
|
const fileInput = document.getElementById('fileInput'); |
||||
|
const fileName = document.getElementById('fileName'); |
||||
|
const fileSize = document.getElementById('fileSize'); |
||||
|
const progressBar = document.getElementById('progressBar'); |
||||
|
const progressText = document.getElementById('progressText'); |
||||
|
const chunkInfo = document.getElementById('chunkInfo'); |
||||
|
const uploadBtn = document.getElementById('uploadBtn'); |
||||
|
const loadingSpinner = document.getElementById('loadingSpinner'); |
||||
|
const successBox = document.getElementById('successMessage'); |
||||
|
const errorBox = document.getElementById('errorMessage'); |
||||
|
const fileInfoBox = document.getElementById('fileInfo'); |
||||
|
const progressBox = document.getElementById('progressContainer'); |
||||
|
|
||||
|
let file = null; |
||||
|
let identifier = ''; |
||||
|
const chunkSize = 5 * 1024 * 1024; // 5MB |
||||
|
let totalChunks = 0; |
||||
|
let uploadedParts = new Set(); // 已上传分片编号 |
||||
|
|
||||
|
/* ---------- 拖拽&选择 ---------- */ |
||||
|
uploadArea.addEventListener('click', () => fileInput.click()); |
||||
|
fileInput.addEventListener('change', e => handleFile(e.target.files[0])); |
||||
|
uploadArea.addEventListener('dragover', e => { |
||||
|
e.preventDefault(); |
||||
|
uploadArea.classList.add('dragover'); |
||||
|
}); |
||||
|
uploadArea.addEventListener('dragleave', () => uploadArea.classList.remove('dragover')); |
||||
|
uploadArea.addEventListener('drop', e => { |
||||
|
e.preventDefault(); |
||||
|
uploadArea.classList.remove('dragover'); |
||||
|
handleFile(e.dataTransfer.files[0]); |
||||
|
}); |
||||
|
|
||||
|
function handleFile(f) { |
||||
|
if (!f) return; |
||||
|
file = f; |
||||
|
fileName.textContent = f.name; |
||||
|
fileSize.textContent = formatSize(f.size); |
||||
|
fileInfoBox.classList.add('active'); |
||||
|
uploadBtn.classList.add('active'); |
||||
|
} |
||||
|
|
||||
|
function formatSize(b) { |
||||
|
const units = ['B', 'KB', 'MB', 'GB']; |
||||
|
let i = 0; |
||||
|
while (b >= 1024 && i < 3) { |
||||
|
b /= 1024; |
||||
|
i++; |
||||
|
} |
||||
|
return b.toFixed(2) + ' ' + units[i]; |
||||
|
} |
||||
|
|
||||
|
/* ---------- 工具 ---------- */ |
||||
|
async function md5File(f) { |
||||
|
return new Promise((resolve, reject) => { |
||||
|
const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice; |
||||
|
const chunks = Math.ceil(f.size / chunkSize); |
||||
|
let currentChunk = 0; |
||||
|
const spark = new SparkMD5.ArrayBuffer(); |
||||
|
const reader = new FileReader(); |
||||
|
|
||||
|
reader.onload = e => { |
||||
|
spark.append(e.target.result); |
||||
|
currentChunk++; |
||||
|
if (currentChunk < chunks) loadNext(); |
||||
|
else resolve(spark.end()); |
||||
|
}; |
||||
|
reader.onerror = () => reject('FileReader error'); |
||||
|
|
||||
|
function loadNext() { |
||||
|
const start = currentChunk * chunkSize; |
||||
|
const end = Math.min(start + chunkSize, f.size); |
||||
|
reader.readAsArrayBuffer(blobSlice.call(f, start, end)); |
||||
|
} |
||||
|
|
||||
|
loadNext(); |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
/* ---------- 上传主流程 ---------- */ |
||||
|
uploadBtn.addEventListener('click', startUpload); |
||||
|
|
||||
|
async function startUpload() { |
||||
|
if (!file) return; |
||||
|
uploadBtn.style.display = 'none'; |
||||
|
loadingSpinner.style.display = 'block'; |
||||
|
progressBox.classList.add('active'); |
||||
|
|
||||
|
try { |
||||
|
identifier = await md5File(file); |
||||
|
totalChunks = Math.ceil(file.size / chunkSize); |
||||
|
|
||||
|
// 查询已上传的分片 |
||||
|
const progressResp = await fetch(`http://localhost:8081/api/file/v1/chunk_upload_progress/${identifier}`); |
||||
|
const progressData = await progressResp.json(); |
||||
|
if (progressData.code ===0 && progressData.data != null && progressData.data.exitPartList) { |
||||
|
progressData.data.exitPartList.forEach(p => uploadedParts.add(p.partNumber)); |
||||
|
} |
||||
|
|
||||
|
// 初始化任务(已存在会秒传) |
||||
|
const initResp = await fetch('http://localhost:8081/api/file/v1/init_file_chunk_task', { |
||||
|
method: 'POST', |
||||
|
headers: {'Content-Type': 'application/json'}, |
||||
|
body: JSON.stringify({ |
||||
|
filename: file.name, |
||||
|
identifier, |
||||
|
totalSize: file.size, |
||||
|
chunkSize |
||||
|
}) |
||||
|
}); |
||||
|
const initData = await initResp.json(); |
||||
|
if (initData.code !== 0) throw new Error(initData.msg || '初始化失败'); |
||||
|
const uploadId = initData.data.uploadId; |
||||
|
|
||||
|
// 上传缺失分片 |
||||
|
let uploaded = uploadedParts.size; |
||||
|
for (let i = 0; i < totalChunks; i++) { |
||||
|
const partNumber = i + 1; |
||||
|
if (uploadedParts.has(partNumber)) continue; |
||||
|
|
||||
|
const start = i * chunkSize; |
||||
|
const end = Math.min(start + chunkSize, file.size); |
||||
|
const chunk = file.slice(start, end); |
||||
|
|
||||
|
// 获取预签名 URL |
||||
|
const urlResp = await fetch(`http://localhost:8081/api/file/v1/get_file_chunk_upload_url/${identifier}/${partNumber}`); |
||||
|
|
||||
|
const urlData = await urlResp.json(); |
||||
|
if (urlData.code !== 0) throw new Error(urlData.msg || '获取URL失败'); |
||||
|
|
||||
|
// PUT 上传 |
||||
|
await fetch(urlData.data, {method: 'PUT', body: chunk}); |
||||
|
uploaded++; |
||||
|
const percent = (uploaded / totalChunks) * 100; |
||||
|
progressBar.style.width = percent + '%'; |
||||
|
progressText.textContent = Math.round(percent) + '%'; |
||||
|
chunkInfo.textContent = `正在上传第 ${partNumber} / ${totalChunks} 个分片`; |
||||
|
} |
||||
|
|
||||
|
// 合并 |
||||
|
const mergeResp = await fetch('http://localhost:8081/api/file/v1/merge_file_chunk', { |
||||
|
method: 'POST', |
||||
|
headers: {'Content-Type': 'application/json'}, |
||||
|
body: JSON.stringify({identifier: identifier, parentId: 0}) |
||||
|
}); |
||||
|
const mergeData = await mergeResp.json(); |
||||
|
if (mergeData.code !== 0) throw new Error(mergeData.msg || '合并失败'); |
||||
|
|
||||
|
loadingSpinner.style.display = 'none'; |
||||
|
successBox.style.display = 'block'; |
||||
|
reset(); |
||||
|
} catch (e) { |
||||
|
loadingSpinner.style.display = 'none'; |
||||
|
errorBox.textContent = `❌ ${e.message || '上传出错'}`; |
||||
|
errorBox.style.display = 'block'; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function reset() { |
||||
|
setTimeout(() => { |
||||
|
fileInfoBox.classList.remove('active'); |
||||
|
progressBox.classList.remove('active'); |
||||
|
uploadBtn.classList.remove('active'); |
||||
|
uploadBtn.style.display = ''; |
||||
|
[successBox, errorBox].forEach(b => b.style.display = 'none'); |
||||
|
progressBar.style.width = '0%'; |
||||
|
progressText.textContent = '0%'; |
||||
|
file = null; |
||||
|
fileInput.value = ''; |
||||
|
}, 3000); |
||||
|
} |
||||
|
</script> |
||||
|
</body> |
||||
|
</html> |
File diff suppressed because one or more lines are too long
@ -0,0 +1,150 @@ |
|||||
|
package org.ycloud.aipan; |
||||
|
|
||||
|
import com.amazonaws.HttpMethod; |
||||
|
import com.amazonaws.services.s3.AmazonS3Client; |
||||
|
import com.amazonaws.services.s3.model.*; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.junit.jupiter.api.Test; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.boot.test.context.SpringBootTest; |
||||
|
|
||||
|
import java.net.URL; |
||||
|
import java.util.*; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
@SpringBootTest |
||||
|
@Slf4j |
||||
|
public class BigFileUploadTest { |
||||
|
|
||||
|
|
||||
|
@Autowired |
||||
|
private AmazonS3Client amazonS3Client; |
||||
|
|
||||
|
//=====================大文件上传相关接口===========================
|
||||
|
|
||||
|
/** |
||||
|
* 第一步:初始化大文件分片上传任务,获取uploadId |
||||
|
* 如果初始化时有 uploadId,说明是断点续传,不能重新生成 uploadId |
||||
|
*/ |
||||
|
@Test |
||||
|
public void testInitiateMultipartUploadTask() { |
||||
|
String bucketName = "ai-pan"; |
||||
|
String objectKey = "/meta/test5.txt"; |
||||
|
|
||||
|
ObjectMetadata objectMetadata = new ObjectMetadata(); |
||||
|
objectMetadata.setContentType("text/plain"); |
||||
|
// objectMetadata.setContentType("application/vnd.openxmlformats-officedocument.wordprocessingml.document");
|
||||
|
//初始化分片上传请求
|
||||
|
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, objectKey, objectMetadata); |
||||
|
|
||||
|
//初始化分片上传任务
|
||||
|
InitiateMultipartUploadResult uploadResult = amazonS3Client.initiateMultipartUpload(initRequest); |
||||
|
String uploadId = uploadResult.getUploadId(); |
||||
|
log.info("uploadId:{}", uploadId); |
||||
|
|
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testGenePreSignedUrls() { |
||||
|
// 定义对象键名
|
||||
|
String objectKey = "/meta/test5.txt"; |
||||
|
// 定义存储桶名称
|
||||
|
String bucket = "ai-pan"; |
||||
|
// 定义分片数量,这里设置为4个分片
|
||||
|
int chunkCount = 4; |
||||
|
|
||||
|
String uploadId = "NzViMDY5NGUtM2IwOC00ZDhkLTk1ODMtM2EyYThhZGFmNmI3LjVhMDlmZDJkLWE2ODgtNDgwNS1hODQwLWZhMTEwNWI0NTUwZA"; |
||||
|
|
||||
|
// 创建用于存储分片URL的列表
|
||||
|
List<String> partList = new ArrayList<>(); |
||||
|
// 遍历每个分片,生成预签名的URL
|
||||
|
for (int i = 1; i <= chunkCount; i++) { |
||||
|
// 生成预签名的 URL, 设置过期时间,例如 1 小时后
|
||||
|
Date expiration = new Date(System.currentTimeMillis() + 3600 * 1000); |
||||
|
// 创建生成预签名URL的请求,并指定HTTP方法为PUT
|
||||
|
GeneratePresignedUrlRequest genePreSignedUrlReq = new GeneratePresignedUrlRequest(bucket, objectKey, HttpMethod.PUT).withExpiration(expiration); |
||||
|
// 添加上传ID和分片编号作为请求参数
|
||||
|
genePreSignedUrlReq.addRequestParameter("uploadId", uploadId); |
||||
|
genePreSignedUrlReq.addRequestParameter("partNumber", String.valueOf(i)); |
||||
|
// 生成并获取预签名URL
|
||||
|
URL url = amazonS3Client.generatePresignedUrl(genePreSignedUrlReq); |
||||
|
// 将生成的URL添加到列表中
|
||||
|
partList.add(url.toString()); |
||||
|
// 日志输出当前分片的URL列表
|
||||
|
log.info("partList:{}", partList); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
// 测试合并分片的方法
|
||||
|
@Test |
||||
|
public void testMergeChunks() { |
||||
|
// 定义对象键名
|
||||
|
String objectKey = "/meta/test5.txt"; |
||||
|
// 定义存储桶名称
|
||||
|
String bucket = "ai-pan"; |
||||
|
// 定义分片数量,这里设置为4个分片
|
||||
|
int chunkCount = 4; |
||||
|
// 定义上传ID,用于标识特定的分片上传事件
|
||||
|
String uploadId = "NzViMDY5NGUtM2IwOC00ZDhkLTk1ODMtM2EyYThhZGFmNmI3LjVhMDlmZDJkLWE2ODgtNDgwNS1hODQwLWZhMTEwNWI0NTUwZA"; |
||||
|
|
||||
|
// 创建一个列出分片请求对象
|
||||
|
ListPartsRequest listPartsRequest = new ListPartsRequest(bucket, objectKey, uploadId); |
||||
|
// 获取分片列表
|
||||
|
PartListing partListing = amazonS3Client.listParts(listPartsRequest); |
||||
|
List<PartSummary> parts = partListing.getParts(); |
||||
|
// 检查分片数量是否与预期一致
|
||||
|
if (chunkCount != parts.size()) { |
||||
|
// 已上传分块数量与记录中的数量不对应,不能合并分片
|
||||
|
throw new RuntimeException("分片缺失,请重新上传"); |
||||
|
} |
||||
|
|
||||
|
// 创建一个完成分片上传请求对象
|
||||
|
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest() |
||||
|
.withUploadId(uploadId) |
||||
|
.withKey(objectKey) |
||||
|
.withBucketName(bucket) |
||||
|
.withPartETags(parts.stream() |
||||
|
// 将每个分片的编号和ETag封装到PartETag对象中
|
||||
|
.map(partSummary -> new PartETag(partSummary.getPartNumber(), partSummary.getETag())) |
||||
|
.collect(Collectors.toList())); |
||||
|
|
||||
|
// 完成分片上传并获取结果
|
||||
|
CompleteMultipartUploadResult result = amazonS3Client.completeMultipartUpload(completeMultipartUploadRequest); |
||||
|
log.info("result:{}", result.getBucketName()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
// 测试列出分片上传的各个分片信息
|
||||
|
public void testListParts() { |
||||
|
// 定义对象键名
|
||||
|
String objectKey = "/meta/test5.txt"; |
||||
|
// 定义存储桶名称
|
||||
|
String bucket = "ai-pan"; |
||||
|
// 定义上传ID,用于标识特定的分片上传事件
|
||||
|
String uploadId = "ZjFkZjRhN2UtNzMzOS04NTUxLTgwOTEtNWViNzUwNmRmYTEzLmE4NTUyMmQyLTM1NjUtNGMwMS05ZTY2LWQ5MWQ4NDUyBmIyA"; |
||||
|
|
||||
|
// 检查指定的存储桶中是否存在具有指定对象键名的对象
|
||||
|
boolean doesObjectExist = amazonS3Client.doesObjectExist(bucket, objectKey); |
||||
|
if (!doesObjectExist) { |
||||
|
// 未上传完,返回已上传的分片
|
||||
|
ListPartsRequest listPartsRequest = new ListPartsRequest(bucket, objectKey, uploadId); |
||||
|
PartListing partListing = amazonS3Client.listParts(listPartsRequest); |
||||
|
List<PartSummary> parts = partListing.getParts(); |
||||
|
// 创建一个结果映射,用于存放上传状态和分片列表
|
||||
|
Map<String, Object> result = new HashMap<>(); |
||||
|
result.put("finished", false); |
||||
|
result.put("exitPartList", parts); |
||||
|
//前端可以通过这个判断是否要调用合并merge接口
|
||||
|
log.info("result:{}", result); |
||||
|
|
||||
|
// 遍历并打印每个分片的信息
|
||||
|
for (PartSummary partSummary : parts) { |
||||
|
System.out.println("getPartNumber:" + partSummary.getPartNumber() + ",getETag=" + partSummary.getETag() + ",getSize= " + partSummary.getSize() + ",getLastModified=" + partSummary.getLastModified()); |
||||
|
} |
||||
|
// 打印存储桶名称
|
||||
|
System.out.println(partListing.getBucketName()); |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
} |
@ -0,0 +1,158 @@ |
|||||
|
package org.ycloud.aipan; |
||||
|
|
||||
|
import lombok.SneakyThrows; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.http.client.methods.CloseableHttpResponse; |
||||
|
import org.apache.http.client.methods.HttpPut; |
||||
|
import org.apache.http.entity.FileEntity; |
||||
|
import org.apache.http.impl.client.CloseableHttpClient; |
||||
|
import org.apache.http.impl.client.HttpClients; |
||||
|
import org.assertj.core.util.Lists; |
||||
|
import org.junit.jupiter.api.Test; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.boot.test.context.SpringBootTest; |
||||
|
import org.ycloud.aipan.controller.req.FileChunkInitTaskReq; |
||||
|
import org.ycloud.aipan.controller.req.FileChunkMergeReq; |
||||
|
import org.ycloud.aipan.dto.FileChunkDTO; |
||||
|
import org.ycloud.aipan.service.FileChunkService; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.FileInputStream; |
||||
|
import java.io.FileOutputStream; |
||||
|
import java.io.IOException; |
||||
|
import java.net.http.HttpClient; |
||||
|
import java.net.http.HttpResponse; |
||||
|
import java.util.List; |
||||
|
|
||||
|
@SpringBootTest |
||||
|
@Slf4j |
||||
|
class FileChunkUploadTests { |
||||
|
|
||||
|
@Autowired |
||||
|
private FileChunkService fileChunkService; |
||||
|
|
||||
|
private Long accountId = 3L; |
||||
|
|
||||
|
private String identifier = "abcsdfsd"; |
||||
|
|
||||
|
/** |
||||
|
* 存储分片后端的文件路径和名称 |
||||
|
*/ |
||||
|
private final List<String> chunkFilePaths = Lists.newArrayList(); |
||||
|
|
||||
|
/** |
||||
|
* 存储分片上传地址 |
||||
|
*/ |
||||
|
private final List<String> chunkUploadUrls = Lists.newArrayList(); |
||||
|
|
||||
|
/** |
||||
|
* 上传ID |
||||
|
*/ |
||||
|
private String uploadId; |
||||
|
|
||||
|
/** |
||||
|
* 分片大小,5MB |
||||
|
*/ |
||||
|
private final long chunkSize = 5 * 1024 * 1024; |
||||
|
|
||||
|
/** |
||||
|
* 用一个10MB以上的文件,按5MB分片大小进行分片 |
||||
|
*/ |
||||
|
|
||||
|
@Test |
||||
|
public void testCreateChunkFiles() { |
||||
|
|
||||
|
// 将文件分片存储
|
||||
|
String filePath = "/Users/xdclass/Desktop/chunk/es_note.pdf"; |
||||
|
File file = new File(filePath); |
||||
|
long fileSize = file.length(); |
||||
|
//int chunkCount = (int) Math.ceil((double) fileSize / CHUNK_SIZE);
|
||||
|
int chunkCount = (int) Math.ceil(fileSize * 1.0 / chunkSize); |
||||
|
log.info("创建分片数量是: {} chunks", chunkCount); |
||||
|
try (FileInputStream fis = new FileInputStream(file)) { |
||||
|
byte[] buffer = new byte[(int) chunkSize]; |
||||
|
for (int i = 0; i < chunkCount; i++) { |
||||
|
String chunkFileName = filePath + ".part" + (i + 1); |
||||
|
try (FileOutputStream fos = new FileOutputStream(chunkFileName)) { |
||||
|
int bytesRead = fis.read(buffer); |
||||
|
fos.write(buffer, 0, bytesRead); |
||||
|
log.info("创建的分片文件名: {} ({} bytes)", chunkFileName, bytesRead); |
||||
|
chunkFilePaths.add(chunkFileName); |
||||
|
} |
||||
|
} |
||||
|
} catch (IOException e) { |
||||
|
e.printStackTrace(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* 第1步,创建分片上传任务 |
||||
|
*/ |
||||
|
private void testInitFileChunkTask() { |
||||
|
FileChunkInitTaskReq req = new FileChunkInitTaskReq(); |
||||
|
req.setAccountId(accountId).setFilename("es_note.pdf") |
||||
|
.setTotalSize((long) (20552959))//20552959
|
||||
|
.setChunkSize((long) (5 * 1024 * 1024))//5242880
|
||||
|
.setIdentifier(identifier); |
||||
|
FileChunkDTO fileChunkDTO = fileChunkService.initFileChunkTask(req); |
||||
|
log.info("分片上传初始化结果: {}", fileChunkDTO); |
||||
|
|
||||
|
uploadId = fileChunkDTO.getUploadId(); |
||||
|
|
||||
|
testGetFileChunkUploadUrl(); |
||||
|
} |
||||
|
/** |
||||
|
* 第2步,获取分片上传地址,返回临时MinIO地址,前端直接上传到Minio里面 |
||||
|
*/ |
||||
|
private void testGetFileChunkUploadUrl() { |
||||
|
|
||||
|
for (int i = 1; i <= chunkFilePaths.size(); i++) { |
||||
|
String uploadUrl = fileChunkService.genPreSignUploadUrl(accountId, identifier, i); |
||||
|
log.info("分片上传地址: {}", uploadUrl); |
||||
|
//存储4个分片地址
|
||||
|
chunkUploadUrls.add(uploadUrl); |
||||
|
} |
||||
|
|
||||
|
uploadChunk(); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 模拟前端直接上传分片 |
||||
|
*/ |
||||
|
@SneakyThrows |
||||
|
private void uploadChunk() { |
||||
|
CloseableHttpClient httpClient = HttpClients.createDefault(); |
||||
|
for (int i = 0; i < chunkUploadUrls.size(); i++) { |
||||
|
// PUT直接上传到minio
|
||||
|
String chunkUploadId = chunkUploadUrls.get(i); |
||||
|
HttpPut httpPut = new HttpPut(chunkUploadId); |
||||
|
httpPut.setHeader("Content-Type","application/octet-stream"); |
||||
|
File chunkFile = new File(chunkFilePaths.get(i)); |
||||
|
FileEntity chunkFileEntity = new FileEntity(chunkFile); |
||||
|
httpPut.setEntity(chunkFileEntity); |
||||
|
CloseableHttpResponse chunkUploadResp = httpClient.execute(httpPut); |
||||
|
httpPut.releaseConnection(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 测试合并分片 |
||||
|
*/ |
||||
|
@Test |
||||
|
public void testMergeFileChunk() { |
||||
|
FileChunkMergeReq req = new FileChunkMergeReq(); |
||||
|
req.setAccountId(accountId).setIdentifier(identifier).setParentId(233L); |
||||
|
fileChunkService.mergeFileChunk(req); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* 查询分片上传进度 |
||||
|
*/ |
||||
|
@Test |
||||
|
public void testChunkUploadProgress() { |
||||
|
FileChunkDTO fileChunkDTO = fileChunkService.listFileChunk(accountId, identifier); |
||||
|
log.info("分片上传进度: {}", fileChunkDTO); |
||||
|
} |
||||
|
} |
Loading…
Reference in new issue