Saturday, May 17, 2014

json output in Shell script

Parsing JSON output in Shell script

jq is a useful utility to parse JSON output in a shell script

jq can be downloaded as a binary and scp'ed to any host and it works just fine.

For more info on jq refer http://stedolan.github.io/jq/

Let us put that to work. The following curl command throws up a JSON output

# curl -s http://localhost:12345/metrics
{"SOURCE.scribeSrc":{"OpenConnectionCount":"0","AppendBatchAcceptedCount":"0","AppendBatchReceivedCount":"0","Type":"SOURCE","EventAcceptedCount":"1850929558","AppendReceivedCount":"0","StopTime":"0","EventReceivedCount":"1850929558","StartTime":"1399527923731","AppendAcceptedCount":"0"},"SINK.avroSink3":{"BatchCompleteCount":"1786","ConnectionFailedCount":"8","EventDrainAttemptCount":"463854128","ConnectionCreatedCount":"1135","BatchEmptyCount":"251379","Type":"SINK","ConnectionClosedCount":"1134","EventDrainSuccessCount":"463852531","StopTime":"0","StartTime":"1399527920721","BatchUnderflowCount":"1222487"},"SINK.avroSink2":{"BatchCompleteCount":"1826","ConnectionFailedCount":"8","EventDrainAttemptCount":"464081974","ConnectionCreatedCount":"882","BatchEmptyCount":"251234","Type":"SINK","ConnectionClosedCount":"881","EventDrainSuccessCount":"464080652","StopTime":"0","StartTime":"1399527920721","BatchUnderflowCount":"1226221"},"SINK.avroSink1":{"BatchCompleteCount":"1830","ConnectionFailedCount":"8","EventDrainAttemptCount":"462363915","ConnectionCreatedCount":"882","BatchEmptyCount":"250488","Type":"SINK","ConnectionClosedCount":"881","EventDrainSuccessCount":"462362687","StopTime":"0","StartTime":"1399527920720","BatchUnderflowCount":"1219047"},"SINK.avroSink":{"BatchCompleteCount":"1881","ConnectionFailedCount":"8","EventDrainAttemptCount":"460631266","ConnectionCreatedCount":"882","BatchEmptyCount":"250796","Type":"SINK","ConnectionClosedCount":"881","EventDrainSuccessCount":"460630534","StopTime":"0","StartTime":"1399527920720","BatchUnderflowCount":"1219204"},"CHANNEL.fileChannel":{"EventPutSuccessCount":"1850929558","ChannelFillPercentage":"0.003154","Type":"CHANNEL","StopTime":"0","EventPutAttemptCount":"1850929558","ChannelSize":"3154","StartTime":"1399527920713","EventTakeSuccessCount":"1850926404","ChannelCapacity":"100000000","EventTakeAttemptCount":"1856822139"}}[12:46:23]

To make it readable, let us use jq command

# curl -s http://localhost:12345/metrics | /tmp/jq '.'
{
  "CHANNEL.fileChannel": {
    "EventTakeAttemptCount": "1857737035",
    "ChannelCapacity": "100000000",
    "EventPutSuccessCount": "1851841723",
    "ChannelFillPercentage": "0.001698",
    "Type": "CHANNEL",
    "StopTime": "0",
    "EventPutAttemptCount": "1851841723",
    "ChannelSize": "1698",
    "StartTime": "1399527920713",
    "EventTakeSuccessCount": "1851840025"
  },
  "SINK.avroSink": {
    "BatchUnderflowCount": "1219720",
    "StartTime": "1399527920720",
    "StopTime": "0",
    "BatchCompleteCount": "1881",
    "ConnectionFailedCount": "8",
    "EventDrainAttemptCount": "460864030",
    "ConnectionCreatedCount": "882",
    "BatchEmptyCount": "250852",
    "Type": "SINK",
    "ConnectionClosedCount": "881",
    "EventDrainSuccessCount": "460863849"
  },
  "SINK.avroSink1": {
    "BatchUnderflowCount": "1219539",
    "StartTime": "1399527920720",
    "StopTime": "0",
    "BatchCompleteCount": "1830",
    "ConnectionFailedCount": "8",
    "EventDrainAttemptCount": "462602212",
    "ConnectionCreatedCount": "882",
    "BatchEmptyCount": "250540",
    "Type": "SINK",
    "ConnectionClosedCount": "881",
    "EventDrainSuccessCount": "462601149"
  },
  "SINK.avroSink2": {
    "BatchUnderflowCount": "1226686",
    "StartTime": "1399527920721",
    "StopTime": "0",
    "BatchCompleteCount": "1826",
    "ConnectionFailedCount": "8",
    "EventDrainAttemptCount": "464310084",
    "ConnectionCreatedCount": "882",
    "BatchEmptyCount": "251284",
    "Type": "SINK",
    "ConnectionClosedCount": "881",
    "EventDrainSuccessCount": "464309471"
  },
  "SINK.avroSink3": {
    "BatchUnderflowCount": "1222934",
    "StartTime": "1399527920721",
    "StopTime": "0",
    "BatchCompleteCount": "1786",
    "ConnectionFailedCount": "8",
    "EventDrainAttemptCount": "464067716",
    "ConnectionCreatedCount": "1135",
    "BatchEmptyCount": "251438",
    "Type": "SINK",
    "ConnectionClosedCount": "1134",
    "EventDrainSuccessCount": "464065556"
  },
  "SOURCE.scribeSrc": {
    "AppendAcceptedCount": "0",
    "StartTime": "1399527923731",
    "OpenConnectionCount": "0",
    "AppendBatchAcceptedCount": "0",
    "AppendBatchReceivedCount": "0",
    "Type": "SOURCE",
    "EventAcceptedCount": "1851841723",
    "AppendReceivedCount": "0",
    "StopTime": "0",
    "EventReceivedCount": "1851841723"
  }
}



No comments:

Post a Comment