![如何将 s3 对象 >5gb 从一个存储桶移动到另一个存储桶?](https://linux22.com/image/800618/%E5%A6%82%E4%BD%95%E5%B0%86%20s3%20%E5%AF%B9%E8%B1%A1%20%3E5gb%20%E4%BB%8E%E4%B8%80%E4%B8%AA%E5%AD%98%E5%82%A8%E6%A1%B6%E7%A7%BB%E5%8A%A8%E5%88%B0%E5%8F%A6%E4%B8%80%E4%B8%AA%E5%AD%98%E5%82%A8%E6%A1%B6%EF%BC%9F.png)
大于 5GB 的对象是支持分段上传。一个存储桶中已有 5GB 以上的文件,我想将其移动到同一个 AWS 账户下的另一个存储桶中。当我使用 s3cmd 发出命令时:
s3cmd mv s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]
我返回错误:
ERROR: S3 error: 400 (InvalidRequest): The specified copy source is larger than the maximum allowable size for a copy source: 5368709120
如果有可能的话,我推测如果不增加带宽/成本,就无法将其转移。即便如此,我仍在尝试弄清楚是否可以通过某种多部分方法移动大文件。
答案1
目前,您尝试执行的操作无法通过单个操作完成。在 S3cmd 中移动到 API 本质上是复制和删除合而为一,这是复制操作的限制。
http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
您可以在 Amazon S3 中存储最多 5 TB 的单个对象。使用此 API,您可以在单个原子操作中创建最大 5 GB 的对象副本。但是,要复制大于 5 GB 的对象,您必须使用分段上传上传部分 - 复制 API
http://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsMPUapi.html
本节中的示例向您展示了如何使用分段上传 API 复制大于 5 GB 的对象。您可以在一次操作中复制小于 5 GB 的对象。
答案2
以下是我如何使用 NodeJS 以节省内存的方式实现此目的。
function copyS3MP(from_bucket, from_key, to_bucket, to_key) {
const AWS = require('aws-sdk');
AWS.config.update({region: 'us-west-2'});
let s3 = new AWS.S3();
let head, uploadId, numParts, fileSize;
let startTime = new Date();
let partNum = 0;
let partSize = 1024 * 1024 * 10; // 10mb chunks except last part
let maxUploadTries = 3;
let multiPartParams = {
Bucket: to_bucket,
Key: to_key,
ContentType: getContentType(to_key)
};
let multipartMap = {
Parts: []
};
function getHead() {
return new Promise(async (resolve, reject) => {
try {
const h = await s3.headObject({
Bucket: from_bucket,
Key: from_key
}).promise();
resolve(h);
} catch (e) {
reject(e);
}
});
}
function createMultipartUpload() {
return new Promise(async (resolve, reject) => {
try {
s3.createMultipartUpload(multiPartParams, function(mpErr, multipart) {
if (mpErr) {
console.error(mpErr);
return reject(mpErr);
}
console.log('Got upload ID', multipart.UploadId);
return resolve(multipart.UploadId);
});
} catch (e) {
reject(e);
}
});
}
function copyPart(start, partNum) {
let tryNum = 1;
function copyLogic(copyParams) {
return new Promise((resolve, reject) => {
s3.uploadPartCopy(copyParams, function(multiErr, mData) {
if (multiErr) {
console.log('Upload part error:', multiErr);
return reject(multiErr);
} else {
multipartMap.Parts[this.request.params.PartNumber - 1] = {
ETag: mData.ETag,
PartNumber: Number(this.request.params.PartNumber)
};
console.log('Completed part', this.request.params.PartNumber);
console.log('mData', mData);
return resolve();
}
}).on('httpUploadProgress', function(progress) { console.log(Math.round(progress.loaded/progress.total*100)+ '% done') });
});
}
return new Promise(async (resolve, reject) => {
let end = Math.min(start + partSize, fileSize);
try {
let partParams = {
Bucket: to_bucket,
Key: to_key,
PartNumber: String(partNum),
UploadId: uploadId,
CopySource: `${from_bucket}/${from_key}`,
CopySourceRange: `bytes=${start}-${end - 1}`
};
while (tryNum <= maxUploadTries) {
try {
await copyLogic(partParams);
return resolve();
} catch (e) {
tryNum++;
if (tryNum <= maxUploadTries) {
console.log('Retrying copy of part: #', partParams.PartNumber);
await module.exports.sleep(1);
} else {
console.log('Failed uploading part: #', partParams.PartNumber);
return reject(e);
}
}
}
resolve();
} catch (e) {
return reject(e);
}
});
}
function completeMultipartUpload() {
return new Promise((resolve, reject) => {
let doneParams = {
Bucket: to_bucket,
Key: to_key,
MultipartUpload: multipartMap,
UploadId: uploadId
};
s3.completeMultipartUpload(doneParams, function(err, data) {
if (err) {
return reject(err);
}
var delta = (new Date() - startTime) / 1000;
console.log('Completed upload in', delta, 'seconds');
console.log('Final upload data:', data);
return resolve();
});
});
}
return new Promise(async (resolve, reject) => {
try {
head = await getHead();
fileSize = head.ContentLength;
} catch (e) {
return reject(e);
}
numParts = Math.ceil(fileSize / partSize);
console.log('Creating multipart upload for:', to_key);
try {
uploadId = await createMultipartUpload();
} catch (e) {
return reject(e);
}
for (let start = 0; start < fileSize; start += partSize) {
partNum++;
console.log("Part Num: " + partNum);
try {
await copyPart(start, partNum);
} catch (e) {
console.error(e);
return reject(e);
}
}
try {
await completeMultipartUpload();
} catch (e) {
return reject(e);
}
resolve();
});
}