linux 下强大的 JSON 解析命令 jq

2020-03-16

介绍

jq is like sed for JSON data - you can use it to slice and filter and map and transform structured data with the same ease that sed, awk, grep and friends let you play with text.

jq 可以对 JSON 数据进行切片、过滤、映射和转换，和sed, awk, grep 命令一样简单好用。

jq is written in portable C, and it has zero runtime dependencies. You can download a single binary, scp it to a far away machine of the same type, and expect it to work.

jq 是用 C语言编写的，没有运行时依赖。独立二进制文件，可以使用 scp 复制到其他服务器直接运行。

安装

linux

# 下载 https://stedolan.github.io/jq/download/
$ wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /usr/local/bin/jq

# 执行权限
$ chmod +x /usr/local/bin/jq

OS X

$ brew install jq

使用

以 njmon 的 json 输出来演示 jq 的常用方式，高级用法见官方手册。

jq "."

以 json格式化输出。

$ njmon -s 1 -c 1 | jq '.'
{
  "timestamp": {
    "datetime": "2020-03-14T19:59:22",
    "UTC": "2020-03-14T11:59:22",
    "snapshot_seconds": 1,
    "snapshot_maxloops": 1,
    "snapshot_loop": 0
  },
 ..... 省略其他输出
}

指定 key 查询

$ njmon -s 1 -c 1 | jq '.timestamp'
{
  "datetime": "2020-03-14T20:14:00",
  "UTC": "2020-03-14T12:14:00",
  "snapshot_seconds": 1,
  "snapshot_maxloops": 1,
  "snapshot_loop": 0
}

$ njmon -s 1 -c 1 | jq '.timestamp.datetime'
"2020-03-14T20:14:43"

--raw-output/-r

标准输出，即不格式化为带引号的json 字符串

$ njmon -s 1 -c 1 | jq '.timestamp.datetime'
"2020-03-14T20:14:43"

$ njmon -s 1 -c 1 | jq -r '.timestamp.datetime'
2020-03-14T20:14:43

@csv：格式化输出

还有其他格式方式： @html,@sh,@base64,@base64d等。

$ njmon -s 1 -c 1 | jq -r 'keys' | jq -r '@csv'
"cpu_total","cpuinfo","cpus","disks","filesystems","identity","lscpu","networks","os_release","proc_meminfo","proc_version","proc_vmstat","stat_counters","timestamp","uptime"

数组

$ njmon -s 1 -c 1 | jq '.cpus'
{
  "cpu0": {
    "user": 0.999,
    "nice": 0,
    "sys": 0.999,
    "idle": 99.87,
    "iowait": 0,
    "hardirq": 0,
    "softirq": 0,
    "steal": 0,
    "guest": 0,
    "guestnice": 0
  },
  "cpu1": {
    "user": 2.996,
    "nice": 0,
    "sys": 0.999,
    "idle": 97.873,
    "iowait": 0,
    "hardirq": 0,
    "softirq": 0,
    "steal": 0,
    "guest": 0,
    "guestnice": 0
  }
}

$ njmon -s 1 -c 1 | jq '.cpus[]'
{
  "user": 0.999,
  "nice": 0,
  "sys": 0.999,
  "idle": 99.92,
  "iowait": 0,
  "hardirq": 0,
  "softirq": 0,
  "steal": 0,
  "guest": 0,
  "guestnice": 0
}
{
  "user": 2.998,
  "nice": 0,
  "sys": 0.999,
  "idle": 97.922,
  "iowait": 0,
  "hardirq": 0,
  "softirq": 0,
  "steal": 0,
  "guest": 0,
  "guestnice": 0
}

函数

keys：获取JSON中的key信息

$ njmon -s 1 -c 1 | jq 'keys'
[
  "cpu_total",
  "cpuinfo",
  "cpus",
  "disks",
  "filesystems",
  "identity",
  "lscpu",
  "networks",
  "os_release",
  "proc_meminfo",
  "proc_version",
  "proc_vmstat",
  "stat_counters",
  "timestamp",
  "uptime"
]

select(boolean_expression)：搜索指定条件内容

$ njmon -s 1 -c 1 | jq '.filesystems[] | select(.fs_dir == "/home")'
{
  "fs_dir": "/home",
  "fs_type": "ext4",
  "fs_opts": "rw,seclabel,relatime,data=ordered",
  "fs_freqs": 0,
  "fs_passno": 0,
  "fs_bsize": 4096,
  "fs_size_mb": 11502,
  "fs_free_mb": 11448,
  "fs_used_mb": 54,
  "fs_full_percent": 0.463,
  "fs_avail": 10842,
  "fs_files": 757392,
  "fs_files_free": 757110,
  "fs_namelength": 255
}

has(key)：判断是否存在某个key

$ njmon -s 1 -c 1 | jq 'has("cpus")'
true

$ njmon -s 1 -c 1 | jq 'has("cpus2")'
false

length：查看输出长度或个数

# 5 个字段
$ njmon -s 1 -c 1 | jq '.timestamp | length'
5

# 长度 19
$ njmon -s 1 -c 1 | jq '.timestamp.datetime | length'
19

实际场景

nacos 工具脚本

在做自动部署脚本时，需要写个 Nacos 工具脚本，实现操作 Nacos 实例状态查看、下线、上线等。针对 Nacos 的 json 结构数据，使用 jq 来进行处理。脚本如下：

nacosUtils.sh

#!/bin/bash
# Nacos工具脚本
# 使用方法
usage() {
   echo "Usage: sh $0 {instance|instances|offline|online} URL NAMESPACE_ID SERVICE_NAME IP PORT [FORCE]"
   echo "n"
   echo "t {instance|instances|offline|online} 方法名称"
   echo "tt -instance：查询某个健康实例状态，true表示在线状态，false表示下线状态, 返回空表示实例不存在"
   echo "tt -instances：查询健康实例总数"
   echo "t SERVICE_NAME 系统名称"
   echo "t IP 实例IP"
   echo "t PORT 实例启动端口号"
   echo "t FORCE 方法为offline时，true表示强制下线，非true表示校验是否多于2个健康实例"
   echo "n"
   echo "示例："
   echo "t 启动：sh $0 instance http://127.0.0.1/nacos ****** serviceName x.x.x.x 8090"
}

# 判断参数
if [ $# -lt 4 ];
then
  usage
  exit 1
fi
    
# nacos url
URL=$2
# nacos 命名空间
NAMESPACE_ID=$3
# 系统名称
SERVICE_NAME=$4
# 实例 IP地址
IP=$5
# 端口
PORT=$6
# 强制下线标识
FORCE=$7
    
# 查看实例url
instanceListUrl="$URL/v1/ns/catalog/instances?serviceName=$SERVICE_NAME&clusterName=DEFAULT&pageSize=10&pageNo=1&namespaceId=$NAMESPACE_ID"
# 更新实例
instanceUrl="$URL/v1/ns/instance?serviceName=$SERVICE_NAME&ip=$IP&port=$PORT&namespaceId=$NAMESPACE_ID"
    
# 查询健康且上线的实例个数
function instances() {
  echo `curl -X GET -s $instanceListUrl | jq -r ".list[] | select(.healthy == true) | select(.enabled == true) | .ip" | wc -l`
}

# 查询健康实例状态  在线：true；下线：false, 没有查询到: 空
function instance() {
  echo `curl -X GET -s $instanceListUrl | jq -r ".list[] | select(.ip == "$IP") | select(.port == $PORT) | select(.healthy == true) | .enabled"`
}

# 下线  健康实例总数为1时，不可下线唯一的服务
function offline() {
  if [ "X$FORCE" != "Xtrue" ];then
    counts=`instances`
    if [ $counts -lt 2 ];then
      curl -X GET -s $instanceListUrl | jq .
      echo "$SERVICE_NAME 实例总数小于2个，不可下线唯一的服务！！！"
      exit 1
    fi
  fi
  result=`curl -X PUT -s "$instanceUrl&enabled=false"`
  echo $result
}

# 上线
function online() {
  enabledFlag=`instance`
  if [ "X$enabledFlag" == "Xtrue" ]; then
    curl -X GET -s $instanceListUrl | jq .
    echo "$SERVICE_NAME 实例为上线状态，无需执行上线！！！"
  else
    result=`curl -X PUT -s "$instanceUrl&enabled=true"`
  fi
  echo $result
}

case "$1" in
        'instance')
            instance
            ;;
        'instances')
            instances
            ;;
        'offline')
            offline
            ;;
        'online')
            online
            ;;
        *)
            usage
            exit 1
            ;;
esac

exit 0

njmon 结果格式化

njmon与nmon类似，但输出为 JSON 格式，用于服务器性能统计。

$ njmon -s 1 -c 1 | jq .
{
  "timestamp": {
    "datetime": "2020-03-14T19:59:22",
    "UTC": "2020-03-14T11:59:22",
    "snapshot_seconds": 1,
    "snapshot_maxloops": 1,
    "snapshot_loop": 0
  },
 ..... 省略其他输出
}

扩展阅读

jq 官网：https://stedolan.github.io/jq

jq 在线运行：https://jqplay.org/