强大的jq - SRE回忆录 - 云野生SRE回忆录

共计 9578 个字符，预计需要花费 24 分钟才能阅读完成。

以往在linux中使用awk/sed来处理临时数据采集的工作，但在处理json这种结构数据时显得有点力不从心，但是有个jq命令却可以方便对json格式数据做处理，以前没怎么用，最近数据采集处理多了特此记录下

jq使用

jq安装

[root@k8s-master ~]# yum install jq -y
[root@k8s-master ~]# echo '{"foo": 0}' | jq .
{
  "foo": 0
}

jq基本使用

样例数据

[root@k8s-master ~]# kubectl get ns monitoring -o json
{
    "apiVersion": "v1",
    "kind": "Namespace",
    "metadata": {
        "annotations": {
            "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n"
        },
        "creationTimestamp": "2023-03-16T15:58:48Z",
        "labels": {
            "kubernetes.io/metadata.name": "monitoring"
        },
        "name": "monitoring",
        "resourceVersion": "49152",
        "uid": "fc5c167b-9e1e-49dd-aa00-47c3e30d475f"
    },
    "spec": {
        "finalizers": [
            "kubernetes"
        ]
    },
    "status": {
        "phase": "Active"
    }
}

jq默认格式化后缩进为2个空格

[root@k8s-master ~]# kubectl get ns monitoring -o json | jq .
{
  "apiVersion": "v1",
  "kind": "Namespace",
  "metadata": {
    "annotations": {
      "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n"
    },
    "creationTimestamp": "2023-03-16T15:58:48Z",
    "labels": {
      "kubernetes.io/metadata.name": "monitoring"
    },
    "name": "monitoring",
    "resourceVersion": "49152",
    "uid": "fc5c167b-9e1e-49dd-aa00-47c3e30d475f"
  },
  "spec": {
    "finalizers": [
      "kubernetes"
    ]
  },
  "status": {
    "phase": "Active"
  }
}

jq获取指定字段

[root@k8s-master ~]# kubectl get ns monitoring -o json | jq .metadata
{
  "annotations": {
    "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n"
  },
  "creationTimestamp": "2023-03-16T15:58:48Z",
  "labels": {
    "kubernetes.io/metadata.name": "monitoring"
  },
  "name": "monitoring",
  "resourceVersion": "49152",
  "uid": "fc5c167b-9e1e-49dd-aa00-47c3e30d475f"
}
[root@k8s-master ~]# kubectl get ns monitoring -o json | jq .metadata.name
"monitoring"

jq获取多个字段

[root@k8s-master ~]# kubectl get ns monitoring -o json | jq ".metadata.name,.status"
"monitoring"
{
  "phase": "Active"
}

jq迭代处理

jq处理数组或列表

[root@k8s-master ~]# echo '["a","b","c","d","e"]' | jq
[
  "a",
  "b",
  "c",
  "d",
  "e"
]

[root@k8s-master ~]# echo '["a","b","c","d","e"]' | jq '.[2:3]'
[
  "c"
]
[root@k8s-master ~]# echo '["a","b","c","d","e"]' | jq '.[2:4]'
[
  "c",
  "d"
]
[root@k8s-master ~]# echo '["a","b","c","d","e"]' | jq '.[2:]'
[
  "c",
  "d",
  "e"
]
[root@k8s-master ~]# echo '["a","b","c","d","e"]' | jq '.[:4]'
[
  "a",
  "b",
  "c",
  "d"
]
[root@k8s-master ~]# echo '["a","b","c","d","e"]' | jq '.[-2:]'
[
  "d",
  "e"
]

[root@k8s-master ~]# echo '["a","b","c","d","e"]' | jq '.[2,4]'
"c"
"e"


[root@k8s-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq
[
  {
    "name": "JSON",
    "good": true
  },
  {
    "name": "XML",
    "good": false
  }
]

[root@k8s-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq ".[].name"
"JSON"
"XML"

[root@k8s-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq ".[0].name"
"JSON"
[root@k8s-master ~]#
[root@k8s-master ~]# echo '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' | jq ".[1].name"
"XML"

# 获取ns信息
[root@k8s-master ~]# kubectl get ns -o json | jq '.items[].metadata.name'
"calico-apiserver"
"calico-system"
"default"
"kong"
"kube-node-lease"
"kube-public"
"kube-system"
"monitoring"
"tigera-operator"

[root@k8s-master ~]# kubectl get ns -o json | jq -r '.items[].metadata.name'
calico-apiserver
calico-system
default
kong
kube-node-lease
kube-public
kube-system
monitoring
tigera-operator

jq管道

管道将命令的输出给后面命令作为输入，和系统命令行的管道类似

[root@k8s-master ~]# kubectl get ns -o json | jq -r '.items[] | .metadata.name'
calico-apiserver
calico-system
default
kong
kube-node-lease
kube-public
kube-system
monitoring
tigera-operator
[root@k8s-master ~]# kubectl get ns -o json | jq -r '.items[] | .metadata.name,.metadata.creationTimestamp'
calico-apiserver
2023-03-14T18:58:51Z
calico-system
2023-03-14T18:55:48Z
default
2023-03-14T18:53:41Z
kong
2023-03-15T15:24:29Z
kube-node-lease
2023-03-14T18:53:39Z
kube-public
2023-03-14T18:53:39Z
kube-system
2023-03-14T18:53:39Z
monitoring
2023-03-16T15:58:48Z
tigera-operator
2023-03-14T18:55:07Z

jq运算

[root@k8s-master ~]# echo '[1,2,3,4,5]' | jq '.[] | (. + 2) * 5'
15
20
25
30
35
[root@k8s-master ~]# echo '{"a": [1,2], "b": [3,4]}' | jq '.a + .b'
[
  1,
  2,
  3,
  4
]
[root@k8s-master ~]# echo '["xml", "yaml", "json"]' | jq '. - ["xml", "yaml"]'
[
  "json"
]

jq输出新对象

[root@k8s-master ~]# kubectl get ns kong  -o json | jq "{metadata, test: .status
{
  "metadata": {
    "creationTimestamp": "2023-03-15T15:24:29Z",
    "labels": {
      "kubernetes.io/metadata.name": "kong"
    },
    "name": "kong",
    "resourceVersion": "20554",
    "uid": "64776950-e34f-4be0-94cc-3ff56ca0010f"
  },
  "test": {
    "phase": "Active"
  }
}

jq一些内置函数

length获取长度

[root@k8s-master ~]# kubectl get ns -o json | jq '.items | length'
9

key获取键

[root@k8s-master ~]# kubectl get ns kong  -o json | jq 'keys'
[
  "apiVersion",
  "kind",
  "metadata",
  "spec",
  "status"
]
[root@k8s-master ~]# echo '[4,3,2,1]' | jq 'keys'
[
  0,
  1,
  2,
  3
]

select判断

[root@k8s-master ~]# echo '[1,5,3,0,7]' | jq 'map(select(. >= 2))'
[
5,
3,
7
]

[root@k8s-master ~]# kubectl get pods -A -o json | jq '.items[] | select(.status.qosClass == "BestEffort") | .metadata.name'
"calico-apiserver-868f4c547b-pw452"
"calico-apiserver-868f4c547b-zn96k"
"calico-kube-controllers-77b99b58ff-g8qz6"
"calico-node-srsmd"
"calico-typha-6fb7f79d46-nkqd2"
"echo-5fc5b5bc84-fjkcb"
"echo-5fc5b5bc84-h5smd"
"echo-5fc5b5bc84-hdpdl"
"echo-5fc5b5bc84-rv9jk"
"echo-5fc5b5bc84-sgsrj"
"kong-kong-7959784879-2zlrm"
"kong-kong-init-migrations--1-8r4td"
"kong-kong-pre-upgrade-migrations--1-t5b8d"
"konga-7c477b867b-wfj4j"
"kube-proxy-5pk9h"
"nfs-client-provisioner-6949899dd7-gfvnc"
"tigera-operator-66f6445597-7zgfp"

add相加

[root@k8s-master ~]# echo '["a","b","c"]' | jq 'add'
"abc"
[root@k8s-master ~]# echo '[2,3,4]' | jq 'add'
9

split分割

[root@k8s-master ~]# echo '"a, b,c,d, e, "' | jq -c ' split(", ")'
["a","b,c,d","e",""]

[root@k8s-master ~]# echo '"a-b-c-d-e-"' | jq -c ' split("-")'
["a","b","c","d","e",""]

join联合

[root@k8s-master ~]# echo '["a","b,c,d","e"]' | jq 'join(",")'
"a,b,c,d,e"
[root@k8s-master ~]# echo '["a","b,c,d","e"]' | jq 'join(",") | split(",")'
[
  "a",
  "b",
  "c",
  "d",
  "e"
]

json转换

[root@k8s-master ~]# echo '[1, "foo", ["foo"]]' | jq '[.[]|tostring]'
[
  "1",
  "foo",
  "[\"foo\"]"
]
[root@k8s-master ~]# echo '[1, "foo", ["foo"]]' | jq '[.[]|tojson]'
[
  "1",
  "\"foo\"",
  "[\"foo\"]"
]

[root@k8s-master ~]# kubectl get ns monitoring -o json | jq '.metadata.annotations | ."kubectl.kubernetes.io/last-applied-configuration"'
"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"name\":\"monitoring\"}}\n"
[root@k8s-master ~]# kubectl get ns monitoring -o json | jq '.metadata.annotations | ."kubectl.kubernetes.io/last-applied-configuration" | fromjson'
{
  "apiVersion": "v1",
  "kind": "Namespace",
  "metadata": {
    "annotations": {},
    "name": "monitoring"
  }
}

jq示例

获取容器的resources

[root@k8s-master ~]# kubectl get pods -n monitoring -o json | jq -r '.items[] | .metadata.namespace as $ns | .metadata.name as $pod | .spec.containers[] | [$ns, $pod, .name, ((.resources.requests.cpu // "0m")| if endswith("m") then sub("m$"; "") else .| tonumber * 1000  end), ((.resources.limits.cpu // "0m") | if endswith("m") then sub("m$"; "") else .| tonumber * 1000  end), ((.resources.requests.memory // "0Mi")| if endswith("Mi") then sub("Mi$"; "") elif endswith("Gi") then sub("Gi$"; "") | tonumber * 1024 else . end), ((.resources.limits.memory // "0Mi") |if endswith("Mi") then sub("Mi$"; "") elif endswith("Gi") then sub("Gi$"; "") | tonumber * 1024 else . end )] | @tsv' | column -t
monitoring  alertmanager-main-0                   alertmanager               4    100  100  100
monitoring  alertmanager-main-0                   config-reloader            100  100  50   50
monitoring  alertmanager-main-1                   alertmanager               4    100  100  100
monitoring  alertmanager-main-1                   config-reloader            100  100  50   50
monitoring  alertmanager-main-2                   alertmanager               4    100  100  100
monitoring  alertmanager-main-2                   config-reloader            100  100  50   50
monitoring  blackbox-exporter-5c545d55d6-v6fnw    blackbox-exporter          10   20   20   40
monitoring  blackbox-exporter-5c545d55d6-v6fnw    module-configmap-reloader  10   20   20   40
monitoring  blackbox-exporter-5c545d55d6-v6fnw    kube-rbac-proxy            10   20   20   40
monitoring  grafana-7945df75ff-lgg59              grafana                    100  200  100  200
monitoring  kube-state-metrics-54bd6b479c-9vtv2   kube-state-metrics         10   100  190  250
monitoring  kube-state-metrics-54bd6b479c-9vtv2   kube-rbac-proxy-main       20   40   20   40
monitoring  kube-state-metrics-54bd6b479c-9vtv2   kube-rbac-proxy-self       10   20   20   40
monitoring  node-exporter-npcpd                   node-exporter              102  250  180  180
monitoring  node-exporter-npcpd                   kube-rbac-proxy            10   20   20   40
monitoring  prometheus-adapter-7bf7ff5b67-gn66n   prometheus-adapter         102  250  180  180
monitoring  prometheus-adapter-7bf7ff5b67-k8gzg   prometheus-adapter         102  250  180  180
monitoring  prometheus-k8s-0                      prometheus                 0    0    400  0
monitoring  prometheus-k8s-0                      config-reloader            100  100  50   50
monitoring  prometheus-k8s-1                      prometheus                 0    0    400  0
monitoring  prometheus-k8s-1                      config-reloader            100  100  50   50
monitoring  prometheus-operator-54dd69bbf6-6kfql  prometheus-operator        100  200  100  200
monitoring  prometheus-operator-54dd69bbf6-6kfql  kube-rbac-proxy            10   20   20   40

获取容器

[root@k8s-master ~]# kubectl get pods -n monitoring -o json | jq -r '.items[] | .metadata.namespace as $ns | .metadata.name as $pod | .spec.containers[].name | [$ns, $pod, .] | @tsv' | column -t
monitoring  alertmanager-main-0                   alertmanager
monitoring  alertmanager-main-0                   config-reloader
monitoring  alertmanager-main-1                   alertmanager
monitoring  alertmanager-main-1                   config-reloader
monitoring  alertmanager-main-2                   alertmanager
monitoring  alertmanager-main-2                   config-reloader
monitoring  blackbox-exporter-5c545d55d6-v6fnw    blackbox-exporter
monitoring  blackbox-exporter-5c545d55d6-v6fnw    module-configmap-reloader
monitoring  blackbox-exporter-5c545d55d6-v6fnw    kube-rbac-proxy
monitoring  grafana-7945df75ff-lgg59              grafana
monitoring  kube-state-metrics-54bd6b479c-9vtv2   kube-state-metrics
monitoring  kube-state-metrics-54bd6b479c-9vtv2   kube-rbac-proxy-main
monitoring  kube-state-metrics-54bd6b479c-9vtv2   kube-rbac-proxy-self
monitoring  node-exporter-npcpd                   node-exporter
monitoring  node-exporter-npcpd                   kube-rbac-proxy
monitoring  prometheus-adapter-7bf7ff5b67-gn66n   prometheus-adapter
monitoring  prometheus-adapter-7bf7ff5b67-k8gzg   prometheus-adapter
monitoring  prometheus-k8s-0                      prometheus
monitoring  prometheus-k8s-0                      config-reloader
monitoring  prometheus-k8s-1                      prometheus
monitoring  prometheus-k8s-1                      config-reloader
monitoring  prometheus-operator-54dd69bbf6-6kfql  prometheus-operator
monitoring  prometheus-operator-54dd69bbf6-6kfql  kube-rbac-proxy