edgecore
安装docker
systemctl enable docker && systemctl start docker cat > /etc/docker/daemon.json << EOF { "registry-mirrors": ["https://b9pmyelo.mirror.aliyuncs.com"] } EOF
root@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge# go version
go version go1.14.6 linux/arm64
root@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge# make all WHAT=edgecore
root@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge/edge# go build cmd/edgecore/edgecore.go
root@cloud:~/cmd# apt-get install mosquitto Reading package lists... Done Building dependency tree Reading state information... Done The following package was automatically installed and is no longer required: libusb-0.1-4
root@cloud:~/cmd# mosquitto -v 1617867134: mosquitto version 1.4.15 (build date Tue, 18 Jun 2019 11:42:22 -0300) starting 1617867134: Using default config. 1617867134: Opening ipv4 listen socket on port 1883. 1617867134: Error: Address already in use root@cloud:~/cmd#
1617867134: Error: Address already in use root@cloud:~/cmd# mkdir -p /etc/kubeedge/config/ root@cloud:~/cmd# ~/cmd/edgecore --minconfig > /etc/kubeedge/config/edgecore.yaml 2021-04-08 15:33:55.191605 I | INFO: Install client plugin, protocol: rest 2021-04-08 15:33:55.193395 I | INFO: Installed service discovery plugin: edge root@cloud:~/cmd#
拷贝cloudedge的
改成
tlsCertFile: /etc/kubeedge/certs/stream.crt
tlsPrivateKeyFile: /etc/kubeedge/certs/stream.key
更改httpServer 和erver
cloudcore端口
By default ports 10000 and 10002 in your cloudcore needs to be accessible for your edge nodes.
原来是 cloudcore的CLOUDCOREIPS有问题
root@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge# export CLOUDCOREIPS="192.168.56.105" root@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge# build/tools/certgen.sh stream
CLOUDCOREIPS应该是
keadm init --advertise-address="THE-EXPOSED-IP"(only work since 1.3 release)
tlsCertFile: /etc/kubeedge/certs/stream.crt
tlsPrivateKeyFile: /etc/kubeedge/certs/stream.key
1.3版本不再需要手动生成证书，改为用 k8s secret 方式，因此，必须先运行云端，才会生成 secret(至少成功运行一次，以产生 secret)，否则无法得到 token，无法配置边缘端。
token问题
F0408 16:59:14.457130 75821 certmanager.go:92] Error: token credentials are in the wrong format
kubectl get secret tokensecret -n kubeedge -oyaml apiVersion: v1 data: tokendata: Y2EwODdkYTA2OTM3OTEwMWU5NDIxMGRjODQzNzcxMjU0MGVmYTdkNWQyOThkN2RmZWE5MGRmNTI1NTJjZjE2Mi5leUpoYkdjaU9pSklVekkxTmlJc0luUjVjQ0k2SWtwWFZDSjkuZXlKbGVIQWlPakUyTVRjNU5UWTVNalo5LlZoYl9lbC1Xb3dJZkdOYzFxaER2cHJyWElnaGtsekl0OExmWEhNVW01dE0= kind: Secret metadata: creationTimestamp: "2021-04-08T06:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:data: .: {} f:tokendata: {} f:type: {} manager: cloudcore operation: Update time: "2021-04-08T08:28:46Z" name: tokensecret namespace: kubeedge resourceVersion: "32970197" selfLink: /api/v1/namespaces/kubeedge/secrets/tokensecret uid: 99e013e4-a3f8-48dd-a2a9-0075143dfa20 type: Opaque
tokendata="Y2EwODdkYTA2OTM3OTEwMWU5NDIxMGRjODQzNzcxMjU0MGVmYTdkNWQyOThkN2RmZWE5MGRmNTI1NTJjZjE2Mi5leUpoYkdjaU9pSklVekkxTmlJc0luUjVjQ0k2SWtwWFZDSjkuZXlKbGVIQWlPakUyTVRjNU5UWTVNalo5LlZoYl9lbC1Xb3dJZkdOYzFxaER2cHJyWElnaGtsekl0OExmWEhNVW01dE0=" echo ${tokendata} | | base64 -d
ca087da069379101e94210dc8437712540efa7d5d298d7dfea90df52552cf162.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MTc5NTY5MjZ9.Vhb_el-WowIfGNc1qhDvprrXIghklzIt8LfXHMUm5tM
和
./keadm gettoken --kube-config=$KUBECONFIG获得的不一样root@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge# ~/cmd/keadm gettroot@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge# ~/cmd/keadm gettoken ca087da069379101e94210dc8437712540efa7d5d298d7dfea90df52552cf162.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MTgwMjQ2MjZ9.VxYI8ATgvp4yLc317AEArVsqdCEQRBRC1oa3ZOrxH-E
root@ubuntu:~/cmd# ./keadm gettoken --kube-config=$HOME/.kube/config
57803e2a5572d38cb27216149ff17151ab20ef5f2e09b6c18ad7f5f031f37c82.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MTgyOTgzNTl9.f2FyYcFqCzuB_hsSb0FwjZG9NnmsWRUqVpgRgjOJM0A
root@ubuntu:~/cmd#
root@cloud:~/cmd# cat /etc/kubeedge/config/edgecore.yaml # With --minconfig , you can easily used this configurations as reference. # It's useful to users who are new to KubeEdge, and you can modify/create your own configs accordingly. # This configuration is suitable for beginners. apiVersion: edgecore.config.kubeedge.io/v1alpha1 database: dataSource: /var/lib/kubeedge/edgecore.db kind: EdgeCore modules: edgeHub: heartbeat: 15 httpServer: https://10.18.18.82:10002 tlsCaFile: /etc/kubeedge/ca/rootCA.crt tlsCertFile: /etc/kubeedge/certs/server.crt tlsPrivateKeyFile: /etc/kubeedge/certs/server.key token: "ca087da069379101e94210dc8437712540efa7d5d298d7dfea90df52552cf162.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MTc5NTY5MjZ9.Vhb_el-WowIfGNc1qhDvprrXIghklzIt8LfXHMUm5tM" websocket: enable: true handshakeTimeout: 30 readDeadline: 15 server: 10.18.18.82:10000 writeDeadline: 15 edged: cgroupDriver: cgroupfs cgroupRoot: "" cgroupsPerQOS: true clusterDNS: "" clusterDomain: "" devicePluginEnabled: false dockerAddress: unix:///var/run/docker.sock gpuPluginEnabled: false hostnameOverride: cloud nodeIP: 10.18.18.47 podSandboxImage: kubeedge/pause-arm64:3.1 remoteImageEndpoint: unix:///var/run/dockershim.sock remoteRuntimeEndpoint: unix:///var/run/dockershim.sock runtimeType: docker eventBus: mqttMode: 2 mqttQOS: 0 mqttRetain: false mqttServerExternal: tcp://127.0.0.1:1883 mqttServerInternal: tcp://127.0.0.1:1884
首先找到OpenSSL工具配置文件openssl.cnf，对于Centos,目录在/etc/pki/tls/中
ubuntu /etc/ssl/openssl.cnf
[ v3_ca ] subjectAltName = IP:172.10.15.110
[SAN] subjectAltName=@alt_names [alt_names] IP.1 = 10.10.16.82 [ v3_ca ] #subjectAltName = IP:10.10.16.82
F0409 11:19:25.059920 85883 certmanager.go:92] Error: failed to get edge certificate from the cloudcore, error: Get "https://10.10.16.82:10002/edge.crt": x509: cannot validate certificate for 10.10.16.82 because it doesn't contain any IP SANs
echo subjectAltName = IP:10.30.0.163 > extfile.cnf openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -extfile extfile.cnf -out server.crt -days 5000 ————————————————
export HOST="my.host"
export IP="127.0.0.1"
openssl req -newkey rsa:4096 -nodes -keyout ${HOST}.key -x509 -days 365 -out ${HOST}.crt -addext 'subjectAltName = IP:${IP}' -subj '/C=US/ST=CA/L=SanFrancisco/O=MyCompany/OU=RND/CN=${HOST}/'
Inspired by link
启动 ./edgecore报错
E0409 17:23:15.633955 113274 helper.go:34] Failed to unmarshal configfile /etc/kubeedge/config/edgecore.yaml: error converting YAML to JSON: yaml: line 45: mapping values are not allowed in this context F0409 17:23:15.633986 113274 server.go:64] error converting YAML to JSON: yaml: line 45: mapping values are not allowed in this context
原来事keadm启动了edgecore，吧keadm干掉
root@cloud:~/cmd# ps -elf | grep edgecore 4 S root 202460 1 2 80 0 - 1080888 futex_ 18:50 ? 00:00:10 /usr/local/bin/edgecore 0 S root 203204 86080 0 80 0 - 4414 select 18:52 pts/1 00:00:00 vi edgecore.yaml 0 S root 203747 85398 0 80 0 - 1418 pipe_r 18:58 pts/0 00:00:00 grep --color=auto edgecore root@cloud:~/cmd# keadm reset keadm: command not found root@cloud:~/cmd# ls edgecore edgecore.log keadm keadm.txt root@cloud:~/cmd# ./keadm reset [reset] WARNING: Changes made to this host by 'keadm init' or 'keadm join' will be reverted. [reset] Are you sure you want to proceed? [y/N]: y edgecore is stopped root@cloud:~/cmd# ps -elf | grep edgecore 0 S root 203204 86080 0 80 0 - 4414 select 18:52 pts/1 00:00:00 vi edgecore.yaml 0 S root 204010 85398 0 80 0 - 1418 pipe_r 19:01 pts/0 00:00:00 grep --color=auto edgecore
root@ubuntu:/etc/kubeedge# openssl x509 -in ./certs/edge.crt -noout -text Certificate: Data: Version: 1 (0x0) Serial Number: 7e:cb:e7:46:ab:a6:93:16:66:8d:c0:35:83:b2:d5:54:69:13:b3:00 Signature Algorithm: sha256WithRSAEncryption Issuer: C = CN, ST = Zhejiang, L = Hangzhou, O = KubeEdge, CN = kubeedge.io Validity Not Before: Apr 12 07:22:08 2021 GMT Not After : Apr 12 07:22:08 2022 GMT Subject: C = CN, ST = Zhejiang, L = Hangzhou, O = KubeEdge, CN = 10.0.16.83
Generate Certificates RootCA certificate and a cert/key pair is required to have a setup for KubeEdge. Same cert/key pair can be used in both cloud and edge. # Generete Root Key openssl genrsa -des3 -out rootCA.key 4096 # Generate Root Certificate openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.crt # Generate Key openssl genrsa -out kubeedge.key 2048 # Generate csr, Fill required details after running the command openssl req -new -key kubeedge.key -out kubeedge.csr # Generate Certificate openssl x509 -req -in kubeedge.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out kubeedge.crt -days 500 -sha256
F0413 11:09:01.528568 214355 certmanager.go:91] Error: failed to validate CA certificate. tokenCAhash: 57803e2a5572d38cb27216149ff17151ab20ef5f2e09b6c18ad7f5f031f37c82, CAhash: 21d7a6a637cf1ab7c4899ec225d0f13ea0e08778faf4e6f17b0009841179f76d ~ root
没有token
解决方法
root@ubuntu:/opt/gopath/src/github.com/kubeedge/kubeedge/build/tools# cat extfile.cnf subjectAltName = IP:10.10.10.6.8
genCert() { local name=$1 openssl x509 -req -in ${certPath}/${name}.csr -CA ${caPath}/rootCA.crt -CAkey ${caPath}/rootCA.key \ -CAcreateserial -extfile extfile.cnf -passin pass:kubeedge.io -out ${certPath}/${name}.crt -days 365 -sha256 }
echo "subjectAltName = IP:172.x.x.x" > extfile.cnf
gettoken
./certgen.sh 运行一次会有不同的token
root@ubuntu:~/cmd# ./keadm gettoken de626ee089a041d82f804e9163bdac77254e72398c92efa31df6b60aca24756e.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MTg0NTI5MTF9.4at2obIEKNtFeCNk4sZkQ8kQgJnjRVFBlPD1IW_65AQ root@ubuntu:~/cmd# ./keadm gettoken e2c33de33253464c4c08b48bb0f81061ea5e621669a3ec263fa3ce1925df31a5.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MTg0NTQ0NTF9.14eSrKgAuhdJeTBH_X-JL8ClWYmAEMX9UHNFgS8RDkg root@ubuntu:~/cmd#
port
edgecore通过10000和10004连接edgecore
cloudcore nat
cloudcore port
edgecore 组件
edgecore docker0
mosquitto
mosquitto -d -p 1883 //边缘端开启mosquitto，用于传输消息
kubeedge-temperature-demo
创建device model和device --cloud
创建device model --cloud
root@cloud:~/kubeedge-example/examples/temperature-demo/crds# kubectl apply -f model.yaml devicemodel.devices.kubeedge.io/temperature-model created root@cloud:~/kubeedge-example/examples/temperature-demo/crds# ls instance.yaml model.yaml root@cloud:~/kubeedge-example/examples/temperature-demo/crds#
修改device.yaml文件的边缘节点名称
部署设备--cloud
root@cloud:~/kubeedge-example/examples/temperature-demo/crds# kubectl apply -f instance.yaml device.devices.kubeedge.io/temperature created root@cloud:~/kubeedge-example/examples/temperature-demo/crds#
root@cloud:~/kubeedge-example/examples/temperature-demo/crds# cat instance.yaml apiVersion: devices.kubeedge.io/v1alpha2 kind: Device metadata: name: temperature labels: description: 'temperature' manufacturer: 'test' spec: deviceModelRef: name: temperature-model nodeSelector: nodeSelectorTerms: - matchExpressions: - key: 'name' operator: In values: - edge01-vm status: twins: - propertyName: temperature-status desired: metadata: type: string value: ''
temperature-mapper代码的修改
修改temperature-mapper/main.go中的代码，主要修改以下三部分：
1）注释硬件相关的代码
2）增加温度生成的代码（通过随机函数生成）
3）配置MQTT服务器的地址
具体修改部分如下图所示：
func connectToMqtt() *client.Client { cli := client.New(&client.Options{ // Define the processing of the error handler. ErrorHandler: func(err error) { fmt.Println(err) }, }) defer cli.Terminate() // Connect to the MQTT Server. err := cli.Connect(&client.ConnectOptions{ Network: "tcp", Address: "12.0.0.128:1883", ClientID: []byte("receive-client"), }) if err != nil { panic(err) } return cli }
import ( "context" "encoding/json" "fmt" "os" "strconv" "syscall" "time" "math/rand" "github.com/d2r2/go-dht" "github.com/d2r2/go-shell" "github.com/yosssi/gmq/mqtt" "github.com/yosssi/gmq/mqtt/client" logger "github.com/d2r2/go-logger" )
sensorType := dht.DHT11 // sensorType := dht.AM2302 //sensorType := dht.DHT12 //pin := 11 totalRetried := 0 totalMeasured := 0 totalFailed := 0 term := false // connect to Mqtt broker cli := connectToMqtt() rand.Seed(time.Now().Unix()) for { // Read DHT11 sensor data from specific pin, retrying 10 times in case of failure. //temperature, humidity, retried, err := // dht.ReadDHTxxWithContextAndRetry(ctx, sensorType, pin, false, 10) temperature :=float32(rand.Intn(100)) humidity := 0 retried :=1 var err error =nil totalMeasured++ totalRetried += retried if err != nil && ctx.Err() == nil { totalFailed++ lg.Error(err) continue } // print temperature and humidity if ctx.Err() == nil { lg.Infof("Sensor = %v: Temperature = %v*C, Humidity = %v%% (retried %d times)", sensorType, temperature, humidity, retried) }
构建temperature-mapper镜像
拷贝temperature-mapper镜像到边缘节点
root@cloud:~/kubeedge-example/examples/temperature-demo# docker save -o kubeedge-temperature-mapper.tar kubeedge-temperature-mapper:test-0629
scp -P4498 kubeedge-temperature-mapper.tar root@210.212.212.150:~
load temperature-mapper镜像（edge节点）
root@edge01-vm:~# docker load -i kubeedge-temperature-mapper.tar 1aed0a4fa040: Loading layer [==================================================>] 5.598MB/5.598MB f2c832c0a239: Loading layer [==================================================>] 819.7kB/819.7kB cd9b03c31d86: Loading layer [==================================================>] 2.56kB/2.56kB 4b77ab5e7fca: Loading layer [==================================================>] 342.2MB/342.2MB 127ae7e4ba5f: Loading layer [==================================================>] 3.072kB/3.072kB e0f116d55b95: Loading layer [==================================================>] 88.7MB/88.7MB b8ea063de466: Loading layer [==================================================>] 3.072kB/3.072kB 55c313862e76: Loading layer [==================================================>] 4.5MB/4.5MB 2a9bc75fc9df: Loading layer [==================================================>] 7.094MB/7.094MB Loaded image: kubeedge-temperature-mapper:test-0629 root@edge01-vm:~#
部署temperature mapper（master节点）
修改deployment.yaml文件
根据你的实际情况修改如下两个地方：
a. 边缘节点名称
b. 镜像名称
部署temperature-mapper
root@cloud:~/kubeedge-example/examples/temperature-demo# kubectl create -f deployment.yaml deployment.apps/temperature-mapper created root@cloud:~/kubeedge-example/examples/temperature-demo#
6、观察temperature的变化情况（master节点）
多次执行以下命令，将会看到temperature的变化：
root@cloud:~/kubeedge-example/examples/temperature-demo# kubectl get device temperature -o yaml apiVersion: devices.kubeedge.io/v1alpha2 kind: Device metadata:
deployment.yaml有问题
kubectl version Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.6", GitCommit:"dff82dc0de47299ab66c83c626e08b245ab19037", GitTreeState:"clean", BuildDate:"2020-07-15T16:58:53Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/arm64"} Server Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.6", GitCommit:"dff82dc0de47299ab66c83c626e08b245ab19037", GitTreeState:"clean", BuildDate:"2020-07-15T16:51:04Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/arm64"}
18.6版本改成这样
root@cloud:~/kubeedge-example/examples/temperature-demo# cat deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: temperature-mapper labels: app: temperature spec: replicas: 1 selector: matchLabels: app: temperature template: metadata: labels: app: temperature spec: hostNetwork: true nodeName: edge01-vm containers: - name: temperature image: kubeedge-temperature-mapper:test-0629 imagePullPolicy: IfNotPresent securityContext: privileged: true
edge 运行 temperature-mapper
资源使用情况
certmanager.go:91] Error: failed to get edge certificate from the cloudcore, error: Get "https://xxxx:10002/edge.crt": x509: certificate signed by unknown authority
pause
mosquitto-clients
使用订阅命令订阅，订阅后如果有消息发送，就会收到消息，如下所示。其中#表示订阅任何主题，可以替换为指定的主题，如/aaa、/bbb等。
root@edge01-vm:~# mosquitto_sub -t '#' -d Command 'mosquitto_sub' not found, but can be installed with: snap install mosquitto # version 1.6.12, or apt install mosquitto-clients See 'snap info mosquitto' for additional versions. root@edge01-vm:~# apt install mosquitto-clients -y Reading package lists... Done Building dependency tree Reading state information... Done The following additional packages will be installed: libc-ares2 libmosquitto1 The following NEW packages will be installed: libc-ares2 libmosquitto1 mosquitto-clients 0 upgraded, 3 newly installed, 0 to remove and 214 not upgraded. Need to get 91.6 kB of archives. After this operation, 333 kB of additional disk space will be used. Get:1 http://us.ports.ubuntu.com/ubuntu-ports bionic/main arm64 libc-ares2 arm64 1.14.0-1 [33.0 kB] Get:2 http://us.ports.ubuntu.com/ubuntu-ports bionic-updates/universe arm64 libmosquitto1 arm64 1.4.15-2ubuntu0.18.04.3 [28.6 kB] Get:3 http://us.ports.ubuntu.com/ubuntu-ports bionic-updates/universe arm64 mosquitto-clients arm64 1.4.15-2ubuntu0.18.04.3 [29.9 kB] Fetched 91.6 kB in 7s (13.3 kB/s) Selecting previously unselected package libc-ares2:arm64. (Reading database ... 67139 files and directories currently installed.) Preparing to unpack .../libc-ares2_1.14.0-1_arm64.deb ... Unpacking libc-ares2:arm64 (1.14.0-1) ... Selecting previously unselected package libmosquitto1:arm64. Preparing to unpack .../libmosquitto1_1.4.15-2ubuntu0.18.04.3_arm64.deb ... Unpacking libmosquitto1:arm64 (1.4.15-2ubuntu0.18.04.3) ... Selecting previously unselected package mosquitto-clients. Preparing to unpack .../mosquitto-clients_1.4.15-2ubuntu0.18.04.3_arm64.deb ... Unpacking mosquitto-clients (1.4.15-2ubuntu0.18.04.3) ... Processing triggers for libc-bin (2.27-3ubuntu1) ... Processing triggers for man-db (2.8.3-2ubuntu0.1) ... Setting up libc-ares2:arm64 (1.14.0-1) ... Setting up libmosquitto1:arm64 (1.4.15-2ubuntu0.18.04.3) ... Setting up mosquitto-clients (1.4.15-2ubuntu0.18.04.3) ... Processing triggers for libc-bin (2.27-3ubuntu1) ... root@edge01-vm:~# mosquitto_sub -t '#' -d Client mosqsub|9973-edge01-vm sending CONNECT Client mosqsub|9973-edge01-vm received CONNACK Client mosqsub|9973-edge01-vm sending SUBSCRIBE (Mid: 1, Topic: #, QoS: 0) Client mosqsub|9973-edge01-vm received SUBACK Subscribed (mid: 1): 0
nohup ./edgecore > edgecore.log 2>&1 &
https://blog.csdn.net/MSSC_/article/details/114866906?spm=1001.2014.3001.5501
边缘节点注册
https://seeseeu.top/see/45656/
https://www.dogfei.cn/archives/kubeedge
https://codeleading.com/article/86244737707/
kubeedge-temperature-demo 示例
浙公网安备 33010602011771号