Linux之jq命令

1.　　jq简介　　　　

　　　　jq是一款命令行下处理JSON数据的工具。

　　　　其可以接受标准输入，命令管道或者文件中的JSON数据，经过一系列的过滤器(filters)和表达式的转后形成我们需要的数据结构并将结果输出到标准输出中。

　　　　jq的这种特性使我们可以很容易地在Shell脚本中调用它。

2.　　基本用法

　　　　1.　　格式化

　　　　　　　　1.　　将一行数据使用json格式输出

[root@k8s-master ~]# echo -n '{"id":1, "name":"zhangsan", "score":[75, 85, 90]}'|jq .
{
  "id": 1,
  "name": "zhangsan",
  "score": [
    75,
    85,
    90
  ]
}

　　　　　　　　2.　　将json格式输出到一行

[root@k8s-master ~]# jq -c . <<eof
> {
>   "id": 1,
>   "name": "zhangsan",
>   "score": [
>     75,
>     85,
>     90
>   ]
> }
> eof
{"id":1,"name":"zhangsan","score":[75,85,90]}　　　　　　　　　　　　　　　　　　

　　　　2.　　属性提取

　　　　　　　　1.　　取id的值

[root@k8s-master ~]# echo  '{"id":1, "name":"zhangsan", "score":[75, 85, 90]}' | jq '.id'
1

　　　　　　　　2.　　取name的值

[root@k8s-master ~]# echo  '{"id":1, "name":"zhangsan", "score":[75, 85, 90]}' | jq '.name'
"zhangsan"

　　　　　　　　3.　　取name的值，去掉双引号

[root@k8s-master ~]# echo  '{"id":1, "name":"zhangsan", "score":[75, 85, 90]}' | jq -r '.name'
zhangsan

　　　　　　　　4.　　取多层值

[root@k8s-master ~]# echo -n '{"id":1, "name":"zhangsan", "attr":{"height":1.78,"weight":"60kg"}}'|jq '.attr.height'
1.78

　　　　　　　　5.　　取数组值

$ echo -n '{"id":1, "name":"zhangsan", "score":[75, 85, 90]}'|jq -r '.score[0]'
75

$ echo -n '[75, 85, 90]'|jq -r '.[0]'
75

# 数组截取
$ echo -n '[75, 85, 90]'|jq -r '.[1:3]'
[
  85,
  90
]

# []展开数组
$ echo -n '[75, 85, 90]'|jq '.[]'
75
85
90

　　　　　　　　6.　　展开所有结构

[root@k8s-master ~]# echo  '{"id":1, "name":"zhangsan", "score":[75, 85, 90]}' | jq -c '..'
{"id":1,"name":"zhangsan","score":[75,85,90]}
1
"zhangsan"
[75,85,90]
75
85
90

　　　　　　　　7.　　其他情况

# 从非对象类型中提取字段，会报错
$ echo -n '{"id":1, "name":"zhangsan", "attr":{"height":1.78,"weight":"60kg"}}'|jq '.name.alias'
jq: error (at <stdin>:0): Cannot index string with string "alias"

# 使用?号可以避免这种报错
$ echo -n '{"id":1, "name":"zhangsan", "attr":{"height":1.78,"weight":"60kg"}}'|jq '.name.alias?'

# //符号用于，当前面的表达式取不到值时，执行后面的表达式
$ echo -n '{"id":1, "name":"zhangsan", "attr":{"height":1.78,"weight":"60kg"}}'|jq '.alias//.name'
"zhangsan"

　　　　3.　　管道，逗号与括号

　　　　　　　　1.　　管道

　　　　　　　　　　　　1.　　管道可以将值从前一个命令传送到后一个命令

[root@k8s-master ~]# echo -n '{"id":1, "name":"zhangsan", "attr":{"height":1.78,"weight":"60kg"}}'|jq '.attr|.height'
1.78

　　　　　　　　　　　　2.　　可以做一些简单的运算

$ echo -n '{"id":1, "name":"zhangsan", "attr":{"height":1.78,"weight":"60kg"}}'|jq '.attr|.height*100|tostring + "cm"'
"178cm"

　　　　　　　　2.　　逗号

　　　　　　　　　　　　1.　　使用逗号可以输出多个结果

[root@k8s-master ~]# echo -n '{"id":1, "name":"zhangsan", "attr":{"height":1.78,"weight":"60kg"}}'|jq -r '.attr.height,.attr.weight'
1.78
60kg

　　　　4.　　将读取的元素，都当作字符串，使用-R参数

[root@k8s-master ~]# seq 4 | jq -R '.'
"1"
"2"
"3"
"4"

　　　　5.　　将json元素，变成json数组元素

[root@k8s-master ~]# echo -n '{"id":1, "name":"zhangsan"}' | jq -s '.'
[
  {
    "id": 1,
    "name": "zhangsan"
  }
]

　　　　6.　　数据构造

$ cat data.txt
id  name      age  score
1   zhangsan  17   75
2   lisi      16   80
3   wangwu    18   85
4   zhaoliu   18   90

# 每行分割成数组，[]构造新的数组输出
$ tail -n+2 data.txt|jq -R '[splits("\\s+")]' -c
["1","zhangsan","17","75"]
["2","lisi","16","80"]
["3","wangwu","18","85"]
["4","zhaoliu","18","90"]

$ jq -n '{id:1, name:"zhangsan"}' -c
{"id":1,"name":"zhangsan"}

# 每行转换为对象，{}构造新的对象格式输出
$ tail -n+2 data.txt|jq -R '[splits("\\s+")] | {id:.[0]|tonumber, name:.[1], age:.[2], score:.[3]}' -c
{"id":1,"name":"zhangsan","age":"17","score":"75"}
{"id":2,"name":"lisi","age":"16","score":"80"}
{"id":3,"name":"wangwu","age":"18","score":"85"}
{"id":4,"name":"zhaoliu","age":"18","score":"90"}

# \()字符串占位变量替换
$ cat data.json
{"id":1,"name":"zhangsan","age":"17","score":"75"}
{"id":2,"name":"lisi","age":"16","score":"80"}
{"id":3,"name":"wangwu","age":"18","score":"85"}
{"id":4,"name":"zhaoliu","age":"18","score":"90"}

$ cat data.json |jq '"id:\(.id),name:\(.name),age:\(.age),score:\(.score)"' -r
id:1,name:zhangsan,age:17,score:75
id:2,name:lisi,age:16,score:80
id:3,name:wangwu,age:18,score:85
id:4,name:zhaoliu,age:18,score:90　

　　　　7.　　基础函数

# has函数，检测对象是否包含key
$ echo -n '{"id":1,"name":"zhangsan","age":"17","score":"75"}'|jq 'has("id")'
true

# del函数，删除某个属性
$ echo -n '{"id":1,"name":"zhangsan","age":"17","score":"75"}'|jq 'del(.id)' -c
{"name":"zhangsan","age":"17","score":"75"}

# map函数，对数组中每个元素执行表达式计算，计算结果组织成新数组
$ seq 4|jq -s 'map(. * 2)' -c
[2,4,6,8]

# 上面map函数写法，其实等价于这个写法
$ seq 4|jq -s '[.[]|.*2]' -c
[2,4,6,8]

# keys函数，列出对象属性
$ echo -n '{"id":1,"name":"zhangsan","age":"17","score":"75"}'|jq 'keys' -c
["age","id","name","score"]

# to_entries函数，列出对象键值对
$ echo -n '{"id":1,"name":"zhangsan","age":"17","score":"75"}'|jq 'to_entries' -c
[{"key":"id","value":1},{"key":"name","value":"zhangsan"},{"key":"age","value":"17"},{"key":"score","value":"75"}]

# length函数，计算数组或字符串长度
$ jq -n '[1,2,3,4]|length'
4

# add函数，计算数组中数值之和
$ seq 4|jq -s 'add'
10

# tostring与tonumber，类型转换
$ seq 4|jq 'tostring|tonumber'
1
2
3
4

# type函数，获取元素类型
$ jq 'type' <<eof
1
"zhangsan"
true
null
{"id":1}
[75, 80, 85]
eof

"number"
"string"
"boolean"
"null"
"object"
"array"

　　　　8.　　过滤，排序，分组函数

$ cat data.json
{"id":1,"name":"zhangsan","sex": 0, "age":"17","score":"75"}
{"id":2,"name":"lisi","sex": 1, "age":"16","score":"80"}
{"id":3,"name":"wangwu","sex": 0, "age":"18","score":"85"}
{"id":4,"name":"zhaoliu","sex": 0, "age":"18","score":"90"}

# select函数用于过滤，类似SQL中的where
$ cat data.json |jq 'select( (.id>1) and (.age|IN("16","17","18")) and (.name != "lisi") or (has("attr")|not) and (.score|tonumber >= 90) )' -c
{"id":3,"name":"wangwu","sex":0,"age":"18","score":"85"}
{"id":4,"name":"zhaoliu","sex":0,"age":"18","score":"90"}

# 有一些简化的过滤函数，如arrays, objects, iterables, booleans, numbers, normals, finites, strings, nulls, values, scalars
# 它们根据类型过滤，如objects过滤出对象，values过滤出非null值等
$ jq -c 'objects' <<eof
1
"zhangsan"
true
null
{"id":1}
[75, 80, 85]
eof

{"id":1}

$ jq -c 'values' <<eof
1
"zhangsan"
true
null
{"id":1}
[75, 80, 85]
eof

1
"zhangsan"
true
{"id":1}
[75,80,85]

# 选择出id与name字段，类似SQL中的select id,name
$ cat data.json|jq -s 'map({id,name})[]' -c
{"id":1,"name":"zhangsan"}
{"id":2,"name":"lisi"}
{"id":3,"name":"wangwu"}
{"id":4,"name":"zhaoliu"}

# 提取前2行，类似SQL中的limit 2
$ cat data.json|jq -s 'limit(2; map({id,name})[])' -c
{"id":1,"name":"zhangsan"}
{"id":2,"name":"lisi"}

# 按照age、id排序，类似SQL中的order by age,id
$ cat data.json|jq -s 'sort_by((.age|tonumber), .id)[]' -c
{"id":2,"name":"lisi","sex":1,"age":"16","score":"80"}
{"id":1,"name":"zhangsan","sex":0,"age":"17","score":"75"}
{"id":3,"name":"wangwu","sex":0,"age":"18","score":"85"}
{"id":4,"name":"zhaoliu","sex":0,"age":"18","score":"90"}


# 根据sex与age分组，并每组聚合计算count(*)、avg(score)、max(id)
$ cat data.json |jq -s 'group_by(.sex, .age)[]' -c
[{"id":1,"name":"zhangsan","sex":0,"age":"17","score":"75"}]
[{"id":3,"name":"wangwu","sex":0,"age":"18","score":"85"},{"id":4,"name":"zhaoliu","sex":0,"age":"18","score":"90"}]
[{"id":2,"name":"lisi","sex":1,"age":"16","score":"80"}]

$ cat data.json |jq -s 'group_by(.sex, .age)[]|{sex:.[0].sex, age:.[0].age, count:length, avg_score:map(.score|tonumber)|(add/length), scores:map(.score)|join(","), max_id:map(.id)|max }' -c                 
{"sex":0,"age":"17","count":1,"avg_score":75,"scores":"75","max_id":1}
{"sex":0,"age":"18","count":2,"avg_score":87.5,"scores":"85,90","max_id":4}
{"sex":1,"age":"16","count":1,"avg_score":80,"scores":"80","max_id":2}

　　　　9.　　字符串操作函数

# contains函数，判断是否包含，实际也可用于判断数组是否包含某个元素
$ echo hello | jq -R 'contains("he")'
true

# 判断是否以he开头
$ echo hello | jq -R 'startswith("he")'
true

# 判断是否以llo结尾
$ echo hello | jq -R 'endswith("llo")'
true

# 去掉起始空格
$ echo ' hello '|jq -R 'ltrimstr(" ")|rtrimstr(" ")'
"hello"

# 大小写转换
$ echo hello|jq -R 'ascii_upcase'
"HELLO"

$ echo HELLO|jq -R 'ascii_downcase'
"hello"

# 字符串数组，通过逗号拼接成一个字符串
$ seq 4|jq -s 'map(tostring)|join(",")'
"1,2,3,4"

# json字符串转换为json对象
$ echo -n '{"id":1,"name":"zhangsan","age":"17","attr":"{\"weight\":56,\"height\":178}"}'|jq '.attr = (.attr|fromjson)' -c
{"id":1,"name":"zhangsan","age":"17","attr":{"weight":56,"height":178}}

# json对象转换为json字符串
$ echo -n '{"id":1,"name":"zhangsan","age":"17","attr":{"weight":56,"height":178}}'|jq '.attr = (.attr|tojson)'
{
  "id": 1,
  "name": "zhangsan",
  "age": "17",
  "attr": "{\"weight\":56,\"height\":178}"
}

$ cat data.txt
id:1,name:zhangsan,age:17,score:75
id:2,name:lisi,age:16,score:80
id:3,name:wangwu,age:18,score:85
id:4,name:zhaoliu,age:18,score:90

# 正则表达式过滤，jq使用的是PCRE
$ cat data.txt|jq -R 'select(test("id:\\d+,name:\\w+,age:\\d+,score:8\\d+"))' -r
id:2,name:lisi,age:16,score:80
id:3,name:wangwu,age:18,score:85

# 正则拆分字符串
$ cat data.txt|jq -R '[splits(",")]' -cr
["id:1","name:zhangsan","age:17","score:75"]
["id:2","name:lisi","age:16","score:80"]
["id:3","name:wangwu","age:18","score:85"]
["id:4","name:zhaoliu","age:18","score:90"]

# 正则替换字符串
$ cat data.txt |jq -R 'gsub("name"; "nick")' -r
id:1,nick:zhangsan,age:17,score:75
id:2,nick:lisi,age:16,score:80
id:3,nick:wangwu,age:18,score:85
id:4,nick:zhaoliu,age:18,score:90

# 正则表达式捕获数据
$ cat data.txt|jq -R 'match("id:(?<id>\\d+),name:(?<name>\\w+),age:\\d+,score:8\\d+")' -cr
{"offset":0,"length":30,"string":"id:2,name:lisi,age:16,score:80","captures":[{"offset":3,"length":1,"string":"2","name":"id"},{"offset":10,"length":4,"string":"lisi","name":"name"}]}
{"offset":0,"length":32,"string":"id:3,name:wangwu,age:18,score:85","captures":[{"offset":3,"length":1,"string":"3","name":"id"},{"offset":10,"length":6,"string":"wangwu","name":"name"}]}

# capture命名捕获，生成key是捕获组名称，value是捕获值的对象
$ cat data.txt|jq -R 'capture("id:(?<id>\\d+),name:(?<name>\\w+),age:\\d+,score:8\\d+")' -rc
{"id":"2","name":"lisi"}
{"id":"3","name":"wangwu"}

# 正则扫描输入字符串
$ cat data.txt|jq -R '[scan("\\w+:\\w+")]' -rc
["id:1","name:zhangsan","age:17","score:75"]
["id:2","name:lisi","age:16","score:80"]
["id:3","name:wangwu","age:18","score:85"]
["id:4","name:zhaoliu","age:18","score:90"]

　　　　10.　　日期函数

# 当前时间缀
$ jq -n 'now'
1653820640.939947

# 将时间缀转换为0时区的分解时间(broken down time)，形式为 年 月 日 时 分 秒 dayOfWeek dayOfYear
$ jq -n 'now|gmtime' -c
[2022,4,29,10,45,5.466768980026245,0,148]

# 将时间缀转换为本地时区的分解时间(broken down time)
$ jq -n 'now|localtime' -c
[2022,4,29,18,46,5.386353015899658,0,148]

# 分解时间转换为时间串
$ jq -n 'now|localtime|strftime("%Y-%m-%dT%H:%M:%S")' -c
"2022-05-29T18:50:33"

# 与上面等效
$ jq -n 'now|strflocaltime("%Y-%m-%dT%H:%M:%SZ")'
"2022-05-29T19:00:40Z"

# 时间串解析为分解时间
$ date +%FT%T|jq -R 'strptime("%Y-%m-%dT%H:%M:%S")' -c
[2022,4,29,18,51,27,0,148]

# 分解时间转换为时间缀
$ date +%FT%T|jq -R 'strptime("%Y-%m-%dT%H:%M:%S")|mktime'
1653850310　　

　　　　11.　　高级用法

$ cat data.json
{"id":1,"name":"zhangsan","sex": 0, "age":"17","score":"75"}
{"id":2,"name":"lisi","sex": 1, "age":"16","score":"80"}
{"id":3,"name":"wangwu","sex": 0, "age":"18","score":"85"}
{"id":4,"name":"zhaoliu","sex": 0, "age":"18","score":"90"}

# 单变量定义
$ cat data.json| jq '.id as $id|$id'
1
2
3
4

# 对象展开式变量定义
$ cat data.json |jq '. as {id:$id,name:$name}|"id:\($id),name:\($name)"'
"id:1,name:zhangsan"
"id:2,name:lisi"
"id:3,name:wangwu"
"id:4,name:zhaoliu"

$ cat data.json
["1","zhangsan","17","75"]
["2","lisi","16","80"]
["3","wangwu","18","85"]
["4","zhaoliu","18","90"]

# 数组展开式变量定义
$ cat data.json|jq '. as [$id,$name]|"id:\($id),name:\($name)"'
"id:1,name:zhangsan"
"id:2,name:lisi"
"id:3,name:wangwu"
"id:4,name:zhaoliu"

# 分支结构
$ cat data.json|jq '. as [$id,$name]|if ($id>"1") then "id:\($id),name:\($name)" else empty end'
"id:2,name:lisi"
"id:3,name:wangwu"
"id:4,name:zhaoliu"

# 循环结构，第一个表达式条件满足时，执行只每二个表达式
# 循环结构除了while，还有until、recurse等
$ echo 1|jq 'while(.<100; .*2)'
1
2
4
8
16
32
64

# 自定义计算3次方的函数
$ echo 2|jq 'def cube: .*.*. ; cube'
8