☰

如何将 s3 对象 >5gb 从一个存储桶移动到另一个存储桶？

tag-icon • tag-icon amazon-web-services filesystems amazon-s3
如何将 s3 对象 >5gb 从一个存储桶移动到另一个存储桶？

大于 5GB 的对象是支持分段上传。一个存储桶中已有 5GB 以上的文件，我想将其移动到同一个 AWS 账户下的另一个存储桶中。当我使用 s3cmd 发出命令时：

s3cmd mv s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]

我返回错误：

ERROR: S3 error: 400 (InvalidRequest): The specified copy source is larger than the maximum allowable size for a copy source: 5368709120

如果有可能的话，我推测如果不增加带宽/成本，就无法将其转移。即便如此，我仍在尝试弄清楚是否可以通过某种多部分方法移动大文件。

答案1

目前，您尝试执行的操作无法通过单个操作完成。在 S3cmd 中移动到 API 本质上是复制和删除合而为一，这是复制操作的限制。

http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html

您可以在 Amazon S3 中存储最多 5 TB 的单个对象。使用此 API，您可以在单个原子操作中创建最大 5 GB 的对象副本。但是，要复制大于 5 GB 的对象，您必须使用分段上传上传部分 - 复制 API

http://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsMPUapi.html

本节中的示例向您展示了如何使用分段上传 API 复制大于 5 GB 的对象。您可以在一次操作中复制小于 5 GB 的对象。

答案2

以下是我如何使用 NodeJS 以节省内存的方式实现此目的。

function copyS3MP(from_bucket, from_key, to_bucket, to_key) {
    const AWS = require('aws-sdk');
    AWS.config.update({region: 'us-west-2'});

    let s3 = new AWS.S3();

    let head, uploadId, numParts, fileSize;

    let startTime = new Date();
    let partNum = 0;
    let partSize = 1024 * 1024 * 10; // 10mb chunks except last part
    let maxUploadTries = 3;

    let multiPartParams = {
        Bucket: to_bucket,
        Key: to_key,
        ContentType: getContentType(to_key)
    };

    let multipartMap = {
        Parts: []
    };

    function getHead() {
        return new Promise(async (resolve, reject) => {
            try {
                const h = await s3.headObject({
                    Bucket: from_bucket,
                    Key: from_key
                }).promise();

                resolve(h);
            } catch (e) {
                reject(e);
            }
        });
    }

    function createMultipartUpload() {
        return new Promise(async (resolve, reject) => {
            try {
                s3.createMultipartUpload(multiPartParams, function(mpErr, multipart) {
                    if (mpErr) {
                        console.error(mpErr);
                        return reject(mpErr);
                    }

                    console.log('Got upload ID', multipart.UploadId);

                    return resolve(multipart.UploadId);
                });
            } catch (e) {
                reject(e);
            }
        });
    }

    function copyPart(start, partNum) {
        let tryNum = 1;

        function copyLogic(copyParams) {
            return new Promise((resolve, reject) => {
                s3.uploadPartCopy(copyParams, function(multiErr, mData) {
                    if (multiErr) {
                        console.log('Upload part error:', multiErr);
                        return reject(multiErr);
                    } else {
                        multipartMap.Parts[this.request.params.PartNumber - 1] = {
                            ETag: mData.ETag,
                            PartNumber: Number(this.request.params.PartNumber)
                        };

                        console.log('Completed part', this.request.params.PartNumber);
                        console.log('mData', mData);

                        return resolve();
                    }
                }).on('httpUploadProgress', function(progress) {  console.log(Math.round(progress.loaded/progress.total*100)+ '% done') });
            });
        }

        return new Promise(async (resolve, reject) => {
            let end = Math.min(start + partSize, fileSize);

            try {
                let partParams = {
                    Bucket: to_bucket,
                    Key: to_key,
                    PartNumber: String(partNum),
                    UploadId: uploadId,
                    CopySource: `${from_bucket}/${from_key}`,
                    CopySourceRange: `bytes=${start}-${end - 1}`
                };

                while (tryNum <= maxUploadTries) {
                    try {
                        await copyLogic(partParams);
                        return resolve();
                    } catch (e) {
                        tryNum++;
                        if (tryNum <= maxUploadTries) {
                            console.log('Retrying copy of part: #', partParams.PartNumber);
                            await module.exports.sleep(1);
                        } else {
                            console.log('Failed uploading part: #', partParams.PartNumber);
                            return reject(e);
                        }
                    }
                }

                resolve();
            } catch (e) {
                return reject(e);
            }
        });
    }

    function completeMultipartUpload() {
        return new Promise((resolve, reject) => {
            let doneParams = {
                Bucket: to_bucket,
                Key: to_key,
                MultipartUpload: multipartMap,
                UploadId: uploadId
            };

            s3.completeMultipartUpload(doneParams, function(err, data) {
                if (err) {
                    return reject(err);
                }

                var delta = (new Date() - startTime) / 1000;
                console.log('Completed upload in', delta, 'seconds');
                console.log('Final upload data:', data);

                return resolve();
            });
        });
    }

    return new Promise(async (resolve, reject) => {
        try {
            head = await getHead();
            fileSize = head.ContentLength;
        } catch (e) {
            return reject(e);
        }

        numParts = Math.ceil(fileSize / partSize);

        console.log('Creating multipart upload for:', to_key);

        try {
            uploadId = await createMultipartUpload();
        } catch (e) {
            return reject(e);
        }

        for (let start = 0; start < fileSize; start += partSize) {
            partNum++;
            console.log("Part Num: " + partNum);

            try {
                await copyPart(start, partNum);
            } catch (e) {
                console.error(e);
                return reject(e);
            }
        }

        try {
            await completeMultipartUpload();
        } catch (e) {
            return reject(e);
        }

        resolve();
    });
}

相关内容