ceph 对象存储应用

RGW

# 用户
RGW兼容aws s3和OpenStack Switft，用户数据包含
    用户认证信息
    访问控制权限信息：包含操作访问权限和访问控制列表ACL
    用户配额信息：防止某些用户占用过多内存空间，根据用户实际情况分配存储空间
# 存储桶 bucket
存储桶是对象的容器，为了方便管理和操作具有相同属性的一类对象引起的一级管理单元
    基础信息：RGW关注的信息，包含bucket配额信息，bucket placement rule,bucket中搜索对象数目等
    扩展信息：对RGW透明的一些信息，如用户自定义的元数据信息
# 对象
RGW中的应用对象对应RADOS对象，应用对象上传分整体上传和分段上传，不同的方式应用对象对应RADOS对象的方式不同
    rgw_max_chunk_size: 分块大小，RGW下发至RADOS集群的单个IO大小
    rgw_obj_stripe_size: 条带大小
    class RGWObjManifest: 管理应用对象和RADOS对象的对应关系

服务端ceph 配置对象存储rgw

# 在规划的服务器上部署rgw并检查ceph状态
cd /usr/local/ceph/admin/ && bash ceph.sh rgw
ceph -s
    rgw: 1 daemon active (m3-c3-133)
# 测试部署情况
curl -H "Content-Type: application/json" "http://192.168.43.133:7480"
    <?xml version="1.0" encoding="UTF-8"?>
    <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
    <Owner>
    <ID>anonymous</ID>
    <DisplayName>
    </DisplayName>
    </Owner>
    <Buckets>
    </Buckets>
    </ListAllMyBucketsResult>

# 创建用户(rgw 192.168.43.133)
docker exec -ti rgw \
radosgw-admin user create \
--uid="tao" --display-name="I'm Tao account" --email="88643217@qq.com"
    {
    "user_id": "tao",
    "display_name": "I'm Tao account",
    "email": "88643217@qq.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "tao",
            "access_key": "YA250OWIHH75USSCC82W",
            "secret_key": "Fl5AbVK4RpNNWWWIhuqjUfsKZ9E0JST8Wqsb3zHD"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}


# 查看pool的对象存储相关信息
ceph osd lspools
    default.rgw.control  #控制器信息
    default.rgw.meta     #记录元数据
    default.rgw.log      #日志信息
    default.rgw.buckets.index #对象存储的bucket信息
    default.rgw.buckets.data  #对象存储的数据信息 有文件才显示

客户端测试使用对象存储

# 下载测试文件脚本
wget "https://gist.githubusercontent.com/kairen/e0dec164fa6664f40784f303076233a5/raw/33add5a18cb7d6f18531d8d481562d017557747c/s3client"
chmod +x s3client
yum install python-pip
pip install boto

# 根据创建用户key创建环境参数配置文件并应用
vim s3key.sh
export S3_ACCESS_KEY="YA250OWIHH75USSCC82W"
export S3_SECRET_KEY="Fl5AbVK4RpNNWWWIhuqjUfsKZ9E0JST8Wqsb3zHD"
export S3_HOST="192.168.43.133"
export S3_PORT="7480"
bash s3key.sh

# 列出当前bucket 桶
./s3client list
---------- Bucket List----------

# 创建一个bucket并上传
./s3client upload files s3key.sh /
报错：Upload failed, InvalidRange ...
解决：
    修改配置ceph.conf文件
    添加mon_max_pg_per_osd = 1000 重启所有ceph容器
# 再次执行上传OK
./s3client upload files s3key.sh /
Upload [s3key.sh] success ...
# 查看上传后的文件
./s3client list files
---------- [files] ----------
s3key.sh                162                     2022-05-11T03:42:59.720Z

# 下载上传的文件
mv s3key.sh s3key1.sh
./s3client download files s3key.sh
# 验证下载文件完整性 MD5对比
md5sum s3key.sh s3key1.sh
b746c32b67802fa5149cf63bc5a020c6  s3key.sh
b746c32b67802fa5149cf63bc5a020c6  s3key1.sh

s3测试脚本 s3client 内容

#!/usr/bin/env python
import os
import sys
import boto
from boto.exception import S3ResponseError
import boto.s3.connection
from boto.s3.connection import S3Connection
from boto.s3.key import Key

ACCESS_KEY = os.environ.get('S3_ACCESS_KEY')
SECRET_KEY = os.environ.get('S3_SECRET_KEY')
S3_HOST = os.environ.get('S3_HOST')
S3_PORT = os.environ.get('S3_PORT')

if ACCESS_KEY is None and SECRET_KEY is None \
        and S3_HOST is None and S3_PORT is None:
    sys.exit('ERROR : You must provide a env[S3_ACCESS_KEY], env[S3_SECRET_KEY], env[S3_HOST] and env[S3_PORT]')

def main():
    if len(sys.argv) < 2:
        sys.exit('Usage: s3client {list|upload|download|delete|create}')
    else:
        if sys.argv[1] == 'list':
            s3_list()
        elif sys.argv[1] == 'upload':
            s3_upload()
        elif sys.argv[1] == 'download':
            s3_download()
        elif sys.argv[1] == 'delete':
            s3_delete()
        elif sys.argv[1] == 'create':
            s3_create()
        else:
            sys.exit('Usage: s3client {list|upload|download|delete|create}')

def connection():
    try:
        if S3_HOST != 's3.amazonaws.com':
            s3_connect = boto.connect_s3(
                aws_access_key_id=ACCESS_KEY,
                aws_secret_access_key=SECRET_KEY,
                host=S3_HOST,
                port=int(S3_PORT),
                is_secure=False,
                calling_format=boto.s3.connection.OrdinaryCallingFormat(),
            )
            return s3_connect
        else:
            s3_connect = S3Connection(ACCESS_KEY, SECRET_KEY)
            return s3_connect

    except S3ResponseError, error:
        sys.exit("ERROR : Can\'t connect to %s" % S3_HOST)

def s3_list():
    if len(sys.argv) == 2:
        print('---------- Bucket List----------')
        for bucket in connection().get_all_buckets():
            print "{name:<20} \t {created:<20}".format(
                name=bucket.name,
                created=bucket.creation_date
            )
    elif len(sys.argv) == 3:
        s3_bucket = sys.argv[2]
        try:
            bucket = connection().get_bucket(s3_bucket)
            print("---------- [%s] ----------" % s3_bucket)
            for key in bucket.list():
                print "{name:<20}\t{size:<20}\t{modified:<20}".format(
                    name=key.name,
                    size=key.size,
                    modified=key.last_modified
                )
        except S3ResponseError, error:
            print ("List failed, %s bucket ... " % error.error_code)
    else:
        message_alert(
            "Usage: s3client list {bucket_name}",
            "List file from bucket"
        )

def s3_create():
    if len(sys.argv) == 3:
        s3_bucket = sys.argv[2]
        try:
            if connection().create_bucket(s3_bucket):
                print("Create [%s] success ..." % s3_bucket)
        except S3ResponseError, error:
            print ("Create failed, %s ..." % error.error_code)
    else:
        message_alert(
            "Usage: s3client create {bucket_name}",
            "Create a bucket for bucket_name"
        )

def s3_upload():
    if len(sys.argv) != 5:
        message_alert(
            "Usage: s3client upload {bucket_name} {local_file_path} {s3_path}",
            "Upload file to bucket"
        )
    else:
        s3_bucket = sys.argv[2]
        local_file_path = sys.argv[3]
        s3_path = sys.argv[4]
        try:
            bucket = connection().get_bucket(s3_bucket)
            key = Key(bucket)
            file_name_to_use_in_s3 = "%s/%s" % (s3_path, os.path.basename(local_file_path))
            key.name = file_name_to_use_in_s3
            key.set_contents_from_filename(local_file_path)
            print ("Upload [%s] success ..." % local_file_path)
            sys.exit(0)
        except S3ResponseError, error:
            print ("Upload failed, %s ..." % error.error_code)

def s3_delete():
    if len(sys.argv) == 4:
        s3_bucket = sys.argv[2]
        s3_path = sys.argv[3]
        try:
            bucket = connection().get_bucket(s3_bucket)
            bucket.delete_key(s3_path)
            sys.exit("Delete [%s] file success ..." % s3_path)
        except S3ResponseError, error:
            sys.exit("Delete failed, %s ..." % error.error_code)
    elif len(sys.argv) == 3:
        s3_bucket = sys.argv[2]
        try:
            connection().delete_bucket(s3_bucket)
            print("Delete [%s] bucket success ..." % s3_bucket)
        except S3ResponseError, error:
            sys.exit("Delete failed, %s ..." % error.error_code)
    else:
        message_alert(
            "Usage: s3client delete {bucket_name}",
            "Delete a bucket"
        )
        message_alert(
            "Usage: s3client delete {bucket_name} {s3_path}",
            "Delete file in bucket"
        )
def s3_download():
    if len(sys.argv) != 4:
        message_alert(
            "Usage: s3client download {bucket_name} {s3_file}",
            "Download file from bucket"
        )
    else:
        try:
            s3_bucket = sys.argv[2]
            s3_file = sys.argv[3]
            bucket = connection().get_bucket(s3_bucket)
            if bucket.get_key(s3_file):
                key = bucket.get_key(s3_file)
                key.get_contents_to_filename('./%s' % s3_file)
                sys.exit("Download [%s] success ..." % s3_file)
            else:
                print ("Download failed, Not found file ...")

        except S3ResponseError, error:
            print ("Download failed, %s ..." % error.error_code)

def message_alert(usage, description):
    print("{usage:<50} \t {description:<50}".format(
        usage=usage,
        description=description
    ))

if __name__ == '__main__':
    main()