AppAgent 源码 (xml 解析)
1. 数据准备
adb shell uiautomator dump /sdcard/output.xml # 获取手机ui界面的xml文件
adb pull /sdcard/output.xml output.xml # 将手机上的xml文件拉取到电脑上
具体的xml文件：
<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="0" text="" resource-id="" class="android.widget.LinearLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="0" text="" resource-id="android:id/content" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="0" text="" resource-id="com.miui.home:id/shortcut_menu_layer" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="0" text="" resource-id="com.miui.home:id/drag_layer_background" class="android.view.View" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" />
<node index="1" text="" resource-id="com.miui.home:id/drag_layer" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="0" text="" resource-id="com.miui.home:id/shortcut_menu_blur_view" class="android.view.View" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" />
<node index="1" text="" resource-id="com.miui.home:id/drag_view_container" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" />
<node index="2" text="" resource-id="com.miui.home:id/screen" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="1" text="" resource-id="com.miui.home:id/scrim_view" class="android.view.View" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" />
<node index="2" text="" resource-id="com.miui.home:id/screen_content" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]">
<node index="0" text="" resource-id="com.miui.home:id/workspace" class="com.miui.home.launcher.ScreenView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2660]">
<node index="1" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,1959][1220,2133]">
<node index="0" text="" resource-id="" class="com.miui.home.launcher.ScreenView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[0,1959][1220,2133]">
<node index="0" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="App vault" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[451,2019][501,2069]" />
<node index="1" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="1 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[504,2019][557,2072]" />
<node index="2" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="2 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="true" bounds="[557,2019][610,2072]" />
<node index="3" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="3 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[610,2019][663,2072]" />
<node index="4" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="4 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[663,2019][716,2072]" />
<node index="5" text="" resource-id="" class="android.widget.ImageView" package="com.miui.home" content-desc="5 screen" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[716,2019][769,2072]" />
</node>
</node>
<node index="2" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,243][1220,2660]">
<node index="0" text="" resource-id="" class="android.view.ViewGroup" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,243][1220,2660]">
<node index="0" text="Amap" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Amap" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,1101][324,1387]" />
<node index="1" text="Clock" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Clock" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,1387][324,1673]" />
<node index="2" text="Notes" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Notes" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[896,1387][1182,1673]" />
<node index="3" text="Calculator" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Calculator" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,1673][324,1959]" />
<node index="4" text="Settings" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Settings" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[324,1673][610,1959]" />
<node index="5" text="Calendar" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Calendar" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[610,1673][896,1959]" />
<node index="6" text="Weather" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Weather" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[896,1673][1182,1959]" />
</node>
</node>
</node>
<node index="1" text="" resource-id="com.miui.home:id/hotseat" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,2133][1220,2419]">
<node index="0" text="" resource-id="com.miui.home:id/hotseat_content_screen" class="com.miui.home.launcher.ScreenView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[12,2133][1208,2419]">
<node index="0" text="Phone" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Phone" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[38,2133][324,2419]" />
<node index="1" text="Messaging" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Messaging" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[324,2133][610,2419]" />
<node index="2" text="Camera" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="Camera" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[610,2133][896,2419]" />
<node index="3" text="夸克" resource-id="" class="android.widget.TextView" package="com.miui.home" content-desc="夸克" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[896,2133][1182,2419]" />
</node>
</node>
</node>
<node index="3" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,2712]" />
<node index="4" text="" resource-id="com.miui.home:id/search_bar_container" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,2418][1220,2712]">
<node index="0" text="" resource-id="com.miui.home:id/search_bar" class="android.widget.FrameLayout" package="com.miui.home" content-desc="Search bar" checkable="false" checked="false" clickable="false" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[91,2418][1129,2611]">
<node index="0" text="" resource-id="com.miui.home:id/search_bar_desktop_layout" class="android.widget.RelativeLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="true" password="false" selected="false" bounds="[91,2446][1129,2583]">
<node index="0" text="" resource-id="com.miui.home:id/search_bar_extra_layout" class="android.widget.ViewAnimator" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[91,2446][1129,2583]">
<node index="0" text="" resource-id="com.miui.home:id/search_bar_xiaoai_layout" class="android.widget.FrameLayout" package="com.miui.home" content-desc="Mi AI" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[976,2446][1129,2583]">
<node index="0" text="" resource-id="com.miui.home:id/xiaoai_button" class="android.widget.ImageView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[1015,2446][1090,2583]" />
</node>
</node>
<node index="1" text="" resource-id="com.miui.home:id/search_bar_content_icon_layout" class="android.widget.FrameLayout" package="com.miui.home" content-desc="Search" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[91,2446][244,2583]">
<node index="0" text="" resource-id="com.miui.home:id/search_bar_content_icon" class="android.widget.ImageView" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[130,2446][205,2583]" />
</node>
</node>
</node>
</node>
</node>
<node index="3" text="" resource-id="" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1,2712]">
<node index="0" text="" resource-id="com.miui.home:id/default_position" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="true" focused="true" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,90][1,2712]" />
</node>
<node index="4" text="" resource-id="com.miui.home:id/drop_target_bar" class="android.widget.FrameLayout" package="com.miui.home" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1220,985]" />
</node>
</node>
</node>
</node>
</node>
</node>
</hierarchy>
原始的xml文件没有格式化，可以通过这个网址将其格式化，方便阅读：
https://tool.12333si.com/xmlformat/
2. xml 数据解析
1. 导入必要的库
首先，我们需要导入 xml.etree.ElementTree 模块来解析 XML 文件，并使用 dataclasses 模块来定义数据结构。
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
2. 定义 AndroidElement 类
原始的代码为：
class AndroidElement:
def __init__(self, uid, bbox, attrib):
self.uid = uid
self.bbox = bbox
self.attrib = attrib
但是这样不方便打印输出。
我们使用 dataclass 装饰器定义一个 AndroidElement 类，用于存储每个元素的唯一标识符 (uid)、边界框 (bbox) 和属性 (attrib)。
@dataclass
class AndroidElement:
uid: str
bbox: tuple
attrib: dict = field(default_factory=dict)
3. 配置参数
我们定义了一个全局配置字典 configs，其中包含一个最小距离参数 MIN_DIST，用于判断两个元素是否过于接近。
configs = {"MIN_DIST": 30}
4. 获取元素的唯一标识符
get_id_from_element 函数用于生成每个元素的唯一标识符。它根据元素的 resource-id、class、content-desc 以及元素的宽度和高度来生成标识符。
def get_id_from_element(elem):
bounds = elem.attrib["bounds"][1:-1].split("][")
x1, y1 = map(int, bounds[0].split(","))
x2, y2 = map(int, bounds[1].split(","))
elem_w, elem_h = x2 - x1, y2 - y1
if "resource-id" in elem.attrib and elem.attrib["resource-id"]:
elem_id = elem.attrib["resource-id"].replace(":", ".").replace("/", "_")
else:
elem_id = f"{elem.attrib['class']}_{elem_w}_{elem_h}"
if (
"content-desc" in elem.attrib
and elem.attrib["content-desc"]
and len(elem.attrib["content-desc"]) < 20
):
content_desc = (
elem.attrib["content-desc"]
.replace("/", "_")
.replace(" ", "")
.replace(":", "_")
)
elem_id += f"_{content_desc}"
return elem_id
5. 遍历 XML 树
traverse_tree 函数用于遍历 XML 树，并提取符合条件的元素。它会检查每个元素的属性，并根据配置的最小距离参数判断是否将其添加到列表中。
def traverse_tree(xml_path, elem_list, attrib, add_index=False):
path = []
for event, elem in ET.iterparse(xml_path, ["start", "end"]):
if event == "start":
path.append(elem)
if attrib in elem.attrib and elem.attrib[attrib] == "true":
parent_prefix = ""
if len(path) > 1:
parent_prefix = get_id_from_element(path[-2])
bounds = elem.attrib["bounds"][1:-1].split("][")
x1, y1 = map(int, bounds[0].split(","))
x2, y2 = map(int, bounds[1].split(","))
center = (x1 + x2) // 2, (y1 + y2) // 2
elem_id = get_id_from_element(elem)
if parent_prefix:
elem_id = parent_prefix + "_" + elem_id
if add_index:
elem_id += f"_{elem.attrib['index']}"
close = False
for e in elem_list:
bbox = e.bbox
center_ = (bbox[0][0] + bbox[1][0]) // 2, (
bbox[0][1] + bbox[1][1]
) // 2
dist = (
abs(center[0] - center_[0]) ** 2
+ abs(center[1] - center_[1]) ** 2
) ** 0.5
if dist <= configs["MIN_DIST"]:
close = True
break
if not close:
elem_list.append(
AndroidElement(elem_id, ((x1, y1), (x2, y2)), attrib)
)
if event == "end":
path.pop()
6. 解析 XML 文件并输出结果
最后，我们指定 XML 文件的路径，并调用 traverse_tree 函数来解析文件。解析完成后，我们输出提取的元素列表及其长度。
xml_path = "../output.xml"
clickable_list = []
traverse_tree(xml_path, clickable_list, "clickable", True)
print(clickable_list)
print(len(clickable_list))
7. 输出结果
运行上述代码后，程序将输出所有可点击元素的列表及其数量。例如：
[AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_50_50_Appvault_0', bbox=((451, 2019), (501, 2069)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_1screen_1', bbox=((504, 2019), (557, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_2screen_2', bbox=((557, 2019), (610, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_3screen_3', bbox=((610, 2019), (663, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_4screen_4', bbox=((663, 2019), (716, 2072)), attrib='clickable'), AndroidElement(uid='com.miui.home.launcher.ScreenView_1220_174_android.widget.ImageView_53_53_5screen_5', bbox=((716, 2019), (769, 2072)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Amap_0', bbox=((38, 1101), (324, 1387)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Clock_1', bbox=((38, 1387), (324, 1673)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Notes_2', bbox=((896, 1387), (1182, 1673)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Calculator_3', bbox=((38, 1673), (324, 1959)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Settings_4', bbox=((324, 1673), (610, 1959)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Calendar_5', bbox=((610, 1673), (896, 1959)), attrib='clickable'), AndroidElement(uid='android.view.ViewGroup_1220_2417_android.widget.TextView_286_286_Weather_6', bbox=((896, 1673), (1182, 1959)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_Phone_0', bbox=((38, 2133), (324, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_Messaging_1', bbox=((324, 2133), (610, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_Camera_2', bbox=((610, 2133), (896, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_hotseat_content_screen_android.widget.TextView_286_286_夸克_3', bbox=((896, 2133), (1182, 2419)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_search_bar_Searchbar_com.miui.home.id_search_bar_desktop_layout_0', bbox=((91, 2446), (1129, 2583)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_search_bar_extra_layout_com.miui.home.id_search_bar_xiaoai_layout_MiAI_0', bbox=((976, 2446), (1129, 2583)), attrib='clickable'), AndroidElement(uid='com.miui.home.id_search_bar_desktop_layout_com.miui.home.id_search_bar_content_icon_layout_Search_1', bbox=((91, 2446), (244, 2583)), attrib='clickable')]
20