Hadoop：yarn的Rust API接口

今天头一次接触了yarn的Rust API接口，在本地搭建了集群，能够得到每个任务的详细信息。

(一)得到所有任务的所有信息命令：

默认是json格式，也可以指定xml的格式，如(curl --compressed -H "Accept: application/xml" -X)

注意：这里的端口号是集群的resourceManager所在的节点

[root@node141 hadoop]# curl -X GET "http://node142:8088/ws/v1/cluster/apps"

由于返回的数据过多，这里截个图：

(二)得到指定任务的所有信息命令：

[root@node141 hadoop]# curl --compressed -H "Accept: application/json" -X GET "http://node142:8088/ws/v1/cluster/apps/application_1696942734063_0004"

或者
curl -X GET http://node142:8088/ws/v1/cluster/apps/application_1696942734063_0004

返回结果：

{
    "app": {
        "id": "application_1696942734063_0004",
        "user": "root",
        "name": "word count",
        "queue": "default",
        "state": "FINISHED",
        "finalStatus": "SUCCEEDED",
        "progress": 100.0,
        "trackingUI": "History",
        "trackingUrl": "http://node142:8088/proxy/application_1696942734063_0004/",
        "diagnostics": "",
        "clusterId": 1696942734063,
        "applicationType": "MAPREDUCE",
        "applicationTags": "",
        "priority": 0,
        "startedTime": 1696943373204,
        "launchTime": 1696943373820,
        "finishedTime": 1696943387557,
        "elapsedTime": 14353,
        "amContainerLogs": "http://node141:8042/node/containerlogs/container_1696942734063_0004_01_000001/root",
        "amHostHttpAddress": "node141:8042",
        "amRPCAddress": "node141:34142",
        "masterNodeId": "node141:37268",
        "allocatedMB": -1,
        "allocatedVCores": -1,
        "reservedMB": -1,
        "reservedVCores": -1,
        "runningContainers": -1,
        "memorySeconds": 48979,
        "vcoreSeconds": 26,
        "queueUsagePercentage": 0.0,
        "clusterUsagePercentage": 0.0,
        "resourceSecondsMap": {
            "entry": {
                "key": "memory-mb",
                "value": "48979"
            },
            "entry": {
                "key": "vcores",
                "value": "26"
            }
        },
        "preemptedResourceMB": 0,
        "preemptedResourceVCores": 0,
        "numNonAMContainerPreempted": 0,
        "numAMContainerPreempted": 0,
        "preemptedMemorySeconds": 0,
        "preemptedVcoreSeconds": 0,
        "preemptedResourceSecondsMap": {
            
        },
        "logAggregationStatus": "SUCCEEDED",
        "unmanagedApplication": false,
        "amNodeLabelExpression": "",
        "timeouts": {
            "timeout": [
                {
                    "type": "LIFETIME",
                    "expiryTime": "UNLIMITED",
                    "remainingTimeInSeconds": -1
                }
            ]
        }
    }
}

(三)过滤出state == "FINISHED"的json——失败

curl http://node142:8088/ws/v1/cluster/apps | jq '.apps.app[] | select(.state == "FINISHED")' > /opt/data/running_apps.json

[root@node141 ~]# cat /opt/data/running_apps.json 
{
  "id": "application_1696942734063_0001",
  "user": "root",
  "name": "word count",
  "queue": "default",
  "state": "FINISHED",
 ......
}
{
  "id": "application_1696942734063_0002",
  "user": "root",
  "name": "word count",
  "queue": "default",
  "state": "FINISHED",
 ......
}
{
  "id": "application_1696942734063_0003",
  "user": "root",
  "name": "word count",
  "queue": "default",
  "state": "FINISHED",
  ......
}
{
  "id": "application_1696942734063_0004",
  "user": "root",
  "name": "word count",
  "queue": "default",
  "state": "FINISHED",
  ......
}