restic中增加阿里云oss的backend

前提:使用OSS提供的go包

 

代码一上传至Gitee, 地址:

https://gitee.com/hfpiao/restic

增加文件:

　　internal/backend/oss/config.go: oss连接配置文件

　　internal/backend/oss/config_test.go:测试

　　internal/backend/oss/oss.go: oss功能调用

　　internal/backend/oss/oss_test.go:测试

参照 internal/backend/s3

 

其他修改文件:

       cmd/restic/global.go: 环境变量读取, oss的Open和Create方法调用设置

       internal/backend/location/location.go: 增加oss连接配置解析调用

oss/config.go文件

package oss

import (
	"net/url"
	"strings"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/options"
)

// Config contains all configuration necessary to connect to an oss compatible
// server.
type Config struct {
	Endpoint      string
	UseCname	  bool
	KeyID, Secret string
	Bucket        string
	Timeout		  int
	SecurityToken string
	EnableMD5     bool
	EnableCRC     bool
	Proxy		  string
	AuthProxy	  string
	Connections	  uint
	Layout        string
	Prefix	      string
}

// NewConfig returns a new Config with the default values filled in.
func NewConfig() Config {
	return Config{
		Connections: 5,
		UseCname: false,
		EnableMD5: false,
		EnableCRC: true,
	}
}

func init() {
	options.Register("oss", Config{})
}

// ParseConfig parses the string s and extracts the s3 config. The two
// supported configuration formats are oss://host/bucketname/prefix and
// oss:host/bucketname/prefix. The host can also be a valid s3 region
// name. If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
	// assume that a URL has been specified, parse it and
	// use the host as the endpoint and the path as the
	// bucket name and prefix
	url, err := url.Parse(s[4:])
	if err != nil {
		return nil, errors.Wrap(err, "url.Parse")
	}

	path := strings.SplitN(url.Path[1:], "/", 2)
	return createConfig(url.Host, path, url.Scheme == "http")
}

func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
	cfg := NewConfig()
	cfg.Endpoint = endpoint
	cfg.Bucket = p[0]
	return cfg, nil
}

oss/oss.go文件

package oss

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/aliyun/aliyun-oss-go-sdk/oss"
	"github.com/cenkalti/backoff/v4"
)

// Backend stores data on an S3 endpoint.
type Backend struct {
	client *oss.Client
	sem    *backend.Semaphore
	cfg    Config
	backend.Layout
}

// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}

const defaultLayout = "default"

func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
	debug.Log("open, config %#v", cfg)

	client, err := oss.New(cfg.Endpoint, cfg.KeyID, cfg.Secret)
	if err != nil {
		return nil, errors.Wrap(err, "oss.New")
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &Backend{
		client: client,
		sem:    sem,
		cfg:    cfg,
	}

	l, err := backend.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix)
	if err != nil {
		return nil, err
	}

	be.Layout = l

	return be, nil
}

// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	return open(ctx, cfg, rt)
}

// Create opens the OSS backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	be, err := open(ctx, cfg, rt)
	if err != nil {
		return nil, errors.Wrap(err, "open")
	}
	found, err := be.client.IsBucketExist(cfg.Bucket)

	if err != nil && be.IsAccessDenied(err) {
		err = nil
		found = true
	}

	if err != nil {
		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
		return nil, errors.Wrap(err, "client.BucketExists")
	}

	if !found {
		// create new bucket with default ACL in default region
		err = be.client.CreateBucket(cfg.Bucket, oss.StorageClass(oss.StorageIA), oss.ACL(oss.ACLPrivate))
		if err != nil {
			return nil, errors.Wrap(err, "client.CreateBucket")
		}
	}

	return be, nil
}

// IsAccessDenied returns true if the error is caused by Access Denied.
func (be *Backend) IsAccessDenied(err error) bool {
	debug.Log("IsAccessDenied(%T, %#v)", err, err)

	if e, ok := errors.Cause(err).(oss.ServiceError); ok && e.Code == "AccessDenied" {
		return true
	}

	return false
}

// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
	debug.Log("IsNotExist(%T, %#v)", err, err)
	if os.IsNotExist(errors.Cause(err)) {
		return true
	}

	if e, ok := errors.Cause(err).(oss.ServiceError); ok && e.Code == "NoSuchKey" {
		return true
	}

	return false
}

// Join combines path components with slashes.
func (be *Backend) Join(p ...string) string {
	return path.Join(p...)
}

type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	isDir   bool
}

func (fi fileInfo) Name() string       { return fi.name }    // base name of the file
func (fi fileInfo) Size() int64        { return fi.size }    // length in bytes for regular files; system-dependent for others
func (fi fileInfo) Mode() os.FileMode  { return fi.mode }    // file mode bits
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for Mode().IsDir()
func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)

// ReadDir returns the entries for a directory.
func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, err error) {
	debug.Log("ReadDir(%v)", dir)

	// make sure dir ends with a slash
	if dir[len(dir)-1] != '/' {
		dir += "/"
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	debug.Log("using ListObjectsV2")

	bucket, _ := be.client.Bucket(be.cfg.Bucket)
	maxKeys := oss.MaxKeys(1000)
	prefix := oss.Prefix(dir)
	listObjectsResultV2, err := bucket.ListObjectsV2(prefix, maxKeys)
	if err != nil {
		return nil, err
	}
	for obj := range listObjectsResultV2.Objects {

		if listObjectsResultV2.Objects[obj].Key == "" {
			continue
		}

		name := strings.TrimPrefix(listObjectsResultV2.Objects[obj].Key, dir)
		if name == "" {
			continue
		}
		entry := fileInfo{
			name:    name,
			size:    listObjectsResultV2.Objects[obj].Size,
			modTime: listObjectsResultV2.Objects[obj].LastModified,
		}

		if name[len(name)-1] == '/' {
			entry.isDir = true
			entry.mode = os.ModeDir | 0755
			entry.name = name[:len(name)-1]
		} else {
			entry.mode = 0644
		}

		list = append(list, entry)
	}

	return list, nil
}

// Location returns this backend's location (the bucket name).
func (be *Backend) Location() string {
	return be.Join(be.cfg.Bucket, be.cfg.Prefix)
}

// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
	return be.cfg.Prefix
}

// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
	debug.Log("Save %v", h)

	if err := h.Valid(); err != nil {
		return backoff.Permanent(err)
	}

	objName := be.Filename(h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	storageType := oss.ObjectStorageClass(oss.StorageIA)
	objectAcl := oss.ObjectACL(oss.ACLPrivate)

	debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length())
	bucket, _ := be.client.Bucket(be.cfg.Bucket)
	err := bucket.PutObject(objName, ioutil.NopCloser(rd), storageType, objectAcl)

	debug.Log("%v -> %v bytes, err %#v: %v", objName, rd.Length(), err, err)

	return errors.Wrap(err, "client.PutObject")
}

// wrapReader wraps an io.ReadCloser to run an additional function on Close.
type wrapReader struct {
	io.ReadCloser
	f func()
}

func (wr wrapReader) Close() error {
	err := wr.ReadCloser.Close()
	wr.f()
	return err
}

// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
	return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}

func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, backoff.Permanent(err)
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	objName := be.Filename(h)

	var err error
	var opts oss.Option
	if length > 0 {
		debug.Log("range: %v-%v", offset, offset+int64(length)-1)
		opts = oss.Range(offset, offset+int64(length)-1)
	} else if offset > 0 {
		debug.Log("range: %v-", offset)
		opts = oss.Range(offset, 0)
	}

	if err != nil {
		return nil, errors.Wrap(err, "oss.Range")
	}

	be.sem.GetToken()
	bucket, _ := be.client.Bucket(be.cfg.Bucket)
	rd, err := bucket.GetObject(objName, opts)
	if err != nil {
		be.sem.ReleaseToken()
		return nil, err
	}

	closeRd := wrapReader{
		ReadCloser: rd,
		f: func() {
			debug.Log("Close()")
			be.sem.ReleaseToken()
		},
	}

	return closeRd, err
}

// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("%v", h)

	objName := be.Filename(h)


	be.sem.GetToken()
	bucket, _ := be.client.Bucket(be.cfg.Bucket)
	props, err := bucket.GetObjectDetailedMeta(objName)
	if err != nil {
		debug.Log("GetObjectDetailedMeta() err %v", err)
		be.sem.ReleaseToken()
		return restic.FileInfo{}, errors.Wrap(err, "client.GetObjectDetailedMeta")
	}
	size, err := strconv.ParseInt(props.Get("Content-Length"), 10, 64)
	if err != nil {
		return restic.FileInfo{}, errors.Wrap(err, "client.GetSize")
	}
	return restic.FileInfo{Size: size, Name: h.Name}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	found := false
	objName := be.Filename(h)

	be.sem.GetToken()
	bucket, _ := be.client.Bucket(be.cfg.Bucket)
	found, _ = bucket.IsObjectExist(objName)
	be.sem.ReleaseToken()

	// If error, then not found
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
	objName := be.Filename(h)

	be.sem.GetToken()
	bucket, _ := be.client.Bucket(be.cfg.Bucket)
	objFound, err := bucket.IsObjectExist(objName)
	if objFound {
		err = bucket.DeleteObject(objName)
	}
	be.sem.ReleaseToken()

	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)

	return errors.Wrap(err, "bucket.DeleteObject")
}

// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
	debug.Log("listing %v", t)

	prefix, _ := be.Basedir(t)

	// make sure prefix ends with a slash
	if !strings.HasSuffix(prefix, "/") {
		prefix += "/"
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	debug.Log("using ListObjectV2")

	bucket, _ := be.client.Bucket(be.cfg.Bucket)

	listObjectsResultV2, err := bucket.ListObjectsV2(oss.Prefix(prefix), oss.MaxKeys(1000))
	if err != nil {
		return err
	}
	for obj := range listObjectsResultV2.Objects {
		m := strings.TrimPrefix(listObjectsResultV2.Objects[obj].Key, prefix)
		if m == "" {
			continue
		}

		fi := restic.FileInfo{
			Name: path.Base(m),
			Size: listObjectsResultV2.Objects[obj].Size,
		}

		if ctx.Err() != nil {
			return ctx.Err()
		}

		err := fn(fi)
		if err != nil {
			return err
		}

		if ctx.Err() != nil {
			return ctx.Err()
		}
	}

	return ctx.Err()
}

// Remove keys for a specified backend type.
func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	return be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
		return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
	})
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
	allTypes := []restic.FileType{
		restic.PackFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range allTypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}

	return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
}

// Close does nothing
func (be *Backend) Close() error { return nil }

// Rename moves a file based on the new layout l.
func (be *Backend) Rename(ctx context.Context, h restic.Handle, l backend.Layout) error {
	debug.Log("Rename %v to %v", h, l)
	oldName := be.Filename(h)
	newName := l.Filename(h)

	if oldName == newName {
		debug.Log("  %v is already renamed", newName)
		return nil
	}

	debug.Log("  %v -> %v", oldName, newName)

	bucket, _ := be.client.Bucket(be.cfg.Bucket)
	_, err := bucket.CopyObject(oldName, newName)
	if err != nil && be.IsNotExist(err) {
		debug.Log("copy failed: %v, seems to already have been renamed", err)
		return nil
	}

	if err != nil {
		debug.Log("copy failed: %v", err)
		return err
	}

	return bucket.DeleteObject(oldName)
}

cmd/global.go文件

func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
　　　　　....
	case "sftp":
		cfg := loc.Config.(sftp.Config)
		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
			return nil, err
		}

		debug.Log("opening sftp repository at %#v", cfg)
		return cfg, nil

	case "oss":
		cfg := loc.Config.(oss.Config)

		if cfg.KeyID == "" {
			cfg.KeyID = os.Getenv("OSS_ACCESS_KEY_ID")
		}

		if cfg.Secret == "" {
			cfg.Secret = os.Getenv("OSS_SECRET_ACCESS_KEY")
		}

		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
			return nil, err
		}

		debug.Log("opening s3 repository at %#v", cfg)
		return cfg, nil
	case "s3":
	.....

	return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
}

// Open the backend specified by a location config.
func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
	....
	case "sftp":
		be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config))
		// wrap the backend in a LimitBackend so that the throughput is limited
		be = limiter.LimitBackend(be, lim)
	case "oss":
		be, err = oss.Open(globalOptions.ctx, cfg.(oss.Config), rt)
	case "s3":
	....

	return be, nil
}

// Create the backend specified by URI.
func create(s string, opts options.Options) (restic.Backend, error) {
	....
	case "s3":
		return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt)
	case "oss":
		return oss.Create(globalOptions.ctx, cfg.(oss.Config), rt)
	case "gs":
	....
	return nil, errors.Fatalf("invalid scheme %q", loc.Scheme)
}

location/location.go文件

....
var parsers = []parser{
	{"b2", b2.ParseConfig, noPassword},
	{"local", local.ParseConfig, noPassword},
	{"sftp", sftp.ParseConfig, noPassword},
	{"s3", s3.ParseConfig, noPassword},
	{"oss", oss.ParseConfig, noPassword},
	{"gs", gs.ParseConfig, noPassword},
	{"azure", azure.ParseConfig, noPassword},
	{"swift", swift.ParseConfig, noPassword},
	{"rest", rest.ParseConfig, rest.StripPassword},
	{"rclone", rclone.ParseConfig, noPassword},
}
....