使用阿里云 OSS SDK 跨地域传输大文件

ossutil 工具只能在同一个地域下操作数据，想要把 oss 里的文件在专有云和公共云之间相互传输，思路是使用 oss sdk，从源 bucket 下载数据，然后上传到目的 bucket。

需要注意的是上传大文件需要分片上传+断点续传+队列重试。

type OssSavePoint struct {
	UploadID              string
	PartsNum              int64 // 总片数
	StartPartNum          int64 // 开始从第几片上传
	MultipartUploadResult oss.InitiateMultipartUploadResult
	UploadParts           []oss.UploadPart
}

type Request struct {
	SourcePath string
	TargetPath string
	UploadID   string
}

func MultipartUpload(ctx context.Context, srcBucket, dstBucket *oss.Bucket, request Request) error {

	props, err := srcBucket.GetObjectDetailedMeta(request.SourcePath)
	if err != nil {
		return err
	}
	fileSize, err := strconv.ParseInt(props.Get("Content-Length"), 10, 64)
	if err != nil {
		return err
	}

	partSize := int64(100 * 1024 * 1024)
	numParts := (fileSize + partSize - 1) / partSize
	if fileSize <= partSize {
		partSize = fileSize
	}

	if numParts >= 10000 {
		return fmt.Errorf("too many parts: %d", numParts)
	}

	sp, err := GetObject(ctx, request.UploadID)
	if err != nil {
		if !errors.Is(err, redis.Nil) {
			return err

		}
		imur, err := dstBucket.InitiateMultipartUpload(request.TargetPath)
		if err != nil {
			return err
		}
		sp = OssSavePoint{
			TaskID:                request.UploadID,
			UploadID:              imur.UploadID,
			PartsNum:              numParts,
			StartPartNum:          0,
			MultipartUploadResult: imur,
			UploadParts:           make([]oss.UploadPart, 0),
		}
	}

	if sp.StartPartNum >= numParts {
		return nil
	}

	for i := sp.StartPartNum; i < numParts; i++ {

		partNumber := i + 1
		start := i * partSize
		end := start + partSize - 1
		if end >= fileSize {
			end = fileSize - 1
		}

		currentPartSize := end - start + 1

		partReader, err := srcBucket.GetObject(request.SourcePath, oss.Range(start, end))
		if err != nil {
			_ = partReader.Close()
			return err
		}

		part, err := dstBucket.UploadPart(sp.MultipartUploadResult, partReader, currentPartSize, int(partNumber))
		if err != nil {
			_ = partReader.Close()
			return err
		}

		sp.UploadParts = append(sp.UploadParts, part)
		sp.StartPartNum = partNumber
		if err := SetObject(ctx, sp.UploadID, sp); err != nil {
			_ = partReader.Close()
			return err
		}

		_ = partReader.Close()
	}

	_, err = dstBucket.CompleteMultipartUpload(sp.MultipartUploadResult, sp.UploadParts, oss.ObjectACL(oss.ACLPrivate))
	if err != nil {
		if abortErr := dstBucket.AbortMultipartUpload(sp.MultipartUploadResult); abortErr != nil {
			return err
		}
		if err := DelObject(ctx, request.UploadID); err != nil {
			return err
		}
		return err
	}
	return nil
}