实现流程可控的镜像下载和存储(一)

基于https实现镜像所有相关元信息的获取

在弱网环境下，下载镜像很慢且容易出错，基于这个原因需要开发更加可靠且支持断点续传的镜像下载程序
由于Docker Hub在国内无法访问，用自己的阿里云镜像加速替代来进行测试
下面以下载linux/amd64的ubuntu22.04镜像为例

Authentication

例中的阿里云镜像加速无需校验，如果镜像源需要校验，那么首先获取token，然后加入header

    -H "Authorization: Bearer <Token>"

获取Manifest

curl -H "Accept: application/vnd.oci.image.index.v1+json" \
     -H "Accept: application/vnd.oci.image.manifest.v1+json" \
     -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
     -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
     -s https://no8ju4hi.mirror.aliyuncs.com/v2/library/ubuntu/manifests/22.04

对于多个架构的镜像仓库，会获取到所有架构的Digest[1]

ubuntu:22.04->Manifests

{
    "manifests": [
        {
            "digest": "sha256:3c3de9608507804525ff4303874525760ea36d62606e8105f515adaa761b80cb",
            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
            "platform": {
                "architecture": "amd64",
                "os": "linux"
            },
            "size": 529
        },
        {
            "digest": "sha256:77e15e9408b4057b6cb6b66fb004a488d42496b60d2fdc02ac73c38e3206d886",
            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
            "platform": {
                "architecture": "arm",
                "os": "linux",
                "variant": "v7"
            },
            "size": 529
        },
        {
            "digest": "sha256:c0c089b5b5b1ea9e860a12a35042d832dcb1733f69fe57089b3152577d96ea35",
            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
            "platform": {
                "architecture": "arm64",
                "os": "linux",
                "variant": "v8"
            },
            "size": 529
        },
        {
            "digest": "sha256:b49ca9c2c0555ed157127cd429996e11900085b8b982414cc48483a1ca5c046b",
            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
            "platform": {
                "architecture": "ppc64le",
                "os": "linux"
            },
            "size": 529
        },
        {
            "digest": "sha256:dce81b3b890c5f5637821a97f45ea9d42dda08c06a3069ae9559ce32385af4e0",
            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
            "platform": {
                "architecture": "riscv64",
                "os": "linux"
            },
            "size": 529
        },
        {
            "digest": "sha256:e45f2138cf3a542f701097c780c96a4a7b8a77d76e27f9ffc1986fda63debd43",
            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
            "platform": {
                "architecture": "s390x",
                "os": "linux"
            },
            "size": 529
        }
    ],
    "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
    "schemaVersion": 2
}

根据上面得到的Manifest，可以获取linux/amd64的镜像对应的信息

• digest: "sha256:3c3de9608507804525ff4303874525760ea36d62606e8105f515adaa761b80cb"
• architecture : "amd64"
• os : "linux"
• size : 424

curl -H "Accept: application/vnd.oci.image.index.v1+json" \
     -H "Accept: application/vnd.oci.image.manifest.v1+json" \
     -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
     -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
     -s https://no8ju4hi.mirror.aliyuncs.com/v2/library/ubuntu/manifests/sha256:3c3de9608507804525ff4303874525760ea36d62606e8105f515adaa761b80cb

对于只有一个架构的镜像，执行第一段命令行，会跳过步骤[1]，直接得到下面的结果

ubuntu:22.04->sha256:3c3de9608507804525ff4303874525760ea36d62606e8105f515adaa761b80cb

{
   "schemaVersion": 2,
   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
   "config": {
      "mediaType": "application/vnd.docker.container.image.v1+json",
      "size": 1462,
      "digest": "sha256:9d28ccdc1fc782ec635c98e55ff68b05e6de1df2c7fcbbb4385f023368eec716"
   },
   "layers": [
      {
         "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
         "size": 29886454,
         "digest": "sha256:aee1767db0dd7da5c30ffaef2282976fe8730cdb0a7b1ecda17cd3c85374fa57"
      }
   ]
}

获取Metadata

根据上面的Manifest，可以得到镜像信息文件的路径
另外可以知道文件的sha256值为9d28ccdc1fc782ec635c98e55ff68b05e6de1df2c7fcbbb4385f023368eec716，大小为1462

wget https://no8ju4hi.mirror.aliyuncs.com/v2/library/ubuntu/blobs/sha256:9d28ccdc1fc782ec635c98e55ff68b05e6de1df2c7fcbbb4385f023368eec716

linux/amd64 ubuntu:22.04镜像信息

{
    "architecture": "amd64",
    "config": {
        "Hostname": "",
        "Domainname": "",
        "User": "",
        "AttachStdin": false,
        "AttachStdout": false,
        "AttachStderr": false,
        "Tty": false,
        "OpenStdin": false,
        "StdinOnce": false,
        "Env": [
            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
        ],
        "Cmd": [
            "bash"
        ],
        "Image": "sha256:c0ffde4acd3f60443d26c66a72e3f18f9da32f570a45ab54ef0d56a301a50150",
        "Volumes": null,
        "WorkingDir": "",
        "Entrypoint": null,
        "OnBuild": null,
        "Labels": null
    },
    "container": "a2b4f5c93fe1e35e98c8624c70be1f9f61147f6568992d1036640ed69e40de6c",
    "container_config": {
        "Hostname": "a2b4f5c93fe1",
        "Domainname": "",
        "User": "",
        "AttachStdin": false,
        "AttachStdout": false,
        "AttachStderr": false,
        "Tty": false,
        "OpenStdin": false,
        "StdinOnce": false,
        "Env": [
            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
        ],
        "Cmd": [
            "/bin/sh",
            "-c",
            "#(nop) ",
            "CMD [\"bash\"]"
        ],
        "Image": "sha256:c0ffde4acd3f60443d26c66a72e3f18f9da32f570a45ab54ef0d56a301a50150",
        "Volumes": null,
        "WorkingDir": "",
        "Entrypoint": null,
        "OnBuild": null,
        "Labels": {}
    },
    "created": "2021-12-04T02:21:12.903923136Z",
    "docker_version": "20.10.7",
    "history": [
        {
            "created": "2021-12-04T02:21:12.507294562Z",
            "created_by": "/bin/sh -c #(nop) ADD file:1e60bfbe5a32672bfcd507fc81d927e28eb8b8b12907819058ba9bed8d6b8ac1 in / "
        },
        {
            "created": "2021-12-04T02:21:12.903923136Z",
            "created_by": "/bin/sh -c #(nop)  CMD [\"bash\"]",
            "empty_layer": true
        }
    ],
    "os": "linux",
    "rootfs": {
        "type": "layers",
        "diff_ids": [
            "sha256:950d1cd211572857ec798f05767ad7614025ec505d4e85df9d5e62662ed4fea9"
        ]
    }
}

下载镜像的所有blob

根据Manifest，可以知道当前镜像只有一个Layer，其对应的blob为sha256:aee1767db0dd7da5c30ffaef2282976fe8730cdb0a7b1ecda17cd3c85374fa57，大小为29886454
下载对应blob

wget https://no8ju4hi.mirror.aliyuncs.com/v2/library/ubuntu/blobs/sha256:aee1767db0dd7da5c30ffaef2282976fe8730cdb0a7b1ecda17cd3c85374fa57

元信息校验

所有的元信息都有sha256和size两个参数，当下载完成时，可以比较文件大小和sha256sum值，完全一致则下载的文件无误，否则需要重新下载

golang实现

package client

import (
	"crypto/tls"
	"encoding/json"
	"io"
	"net/http"
	"strconv"

	log "github.com/sirupsen/logrus"
)

type HarborClient struct {
	uri   string
	token string
}

func (cli *HarborClient) Init(uri, token string) {
	cli.uri = uri
	cli.token = token
}

func (cli *HarborClient) GetManifest(imageName, version, arch string) []byte {
	target := cli.uri + "/v2/" + imageName + "/manifests/" + version
	req, err := http.NewRequest("GET", target, nil)
	if err != nil {
		log.Errorln("GetHttpSkip Request Error:", err)
		return nil
	}
	req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
	req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.list.v2+json")
	req.Header.Add("Accept", "application/vnd.oci.image.manifest.v1+json")
	req.Header.Add("Accept", "application/vnd.oci.image.index.v1+json")

	req.Header.Add("Authorization", "Bearer "+cli.token)
	httpCli := http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	res, err := httpCli.Do(req)
	if err != nil {
		log.Errorln("GetHttpSkip Response Error:", err)
		return nil
	}
	body, _ := io.ReadAll(res.Body)
	data := make(map[string]interface{})
	json.Unmarshal(body, &data)
	_, ok := data["layers"].([]interface{})
	if ok {
		return body
	}

	// image generate by docker buildx
	manifests, ok := data["manifests"].([]interface{})
	if ok {
		for _, info := range manifests {
			tmp := info.(map[string]interface{})
			digest := tmp["digest"].(string)

			platform := tmp["platform"].(map[string]interface{})
			architecture := platform["architecture"].(string)
			// os := platform["os"].(string)
			if architecture == arch {
				return cli.GetManifest(imageName, digest, arch)
			}
		}
	} else {
		log.Errorln("Fail to parse response: " + string(body))
		return nil
	}
	log.Errorln("Fail to get image: " + string(body))
	return nil
}

func (cli *HarborClient) GetMetaData(imageName, blob string) ([]byte, bool) {
	target := cli.uri + "/v2/" + imageName + "/blobs/" + blob
	req, err := http.NewRequest("GET", target, nil)
	if err != nil {
		log.Errorln("GetHttpSkip Request Error:", err)
		return nil, false
	}
	req.Header.Add("Authorization", "Bearer "+cli.token)
	httpCli := http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	res, err := httpCli.Do(req)
	if err != nil {
		log.Errorln("GetHttpSkip Response Error:", err)
		return nil, false
	}
	body, _ := io.ReadAll(res.Body)
	return body, true
}

func (cli *HarborClient) DownloadBlob(w *utils.FileWriter, imageName, blob string, start, len int64) (int, bool) {
	target := cli.uri + "/v2/" + imageName + "/blobs/" + blob
	req, err := http.NewRequest("GET", target, nil)
	if err != nil {
		log.Errorln("GetHttpSkip Request Error:", err)
		return -1, false
	}
	req.Header.Add("Authorization", "Bearer "+cli.token)
	end := start + len - 1
	req.Header.Add("Range", "bytes="+strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(end, 10))
	httpCli := http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	res, err := httpCli.Do(req)
	if err != nil {
		log.Errorln("GetHttpSkip Response Error:", err)
		return -1, false
	}
	body, _ := io.ReadAll(res.Body)
	ret := w.Write(body)
	return ret, true
}