From 4f3ea93278d7d1c0a539dc74e70135ed90a57ddb Mon Sep 17 00:00:00 2001 From: haozhi shao Date: Fri, 16 Oct 2026 12:55:55 +0800 Subject: [PATCH 01/10] iOS Simulator: sandboxed helper, Desktop zero-copy view, macOS test fixes Sandbox the acecode-ios-sim helper with an ACECode-authored Seatbelt profile (no IP networking, no reading personal files, writes only to simulator data and temp folders). CFBundle must be able to list the helper's ancestor directories and the profile follows the user-record home, as CoreSimulator does. When sandbox-exec cannot apply the profile the helper runs unconfined and status/settings say why. Draw the simulator in ACECode Desktop on macOS natively: the simulator's IOSurface becomes the contents of a click-through layer over the panel, so frames skip readback and JPEG encoding. Desktop loads only the system CoreSimulator through a static library shared with the helper. The panel stops the JPEG stream with a new WebSocket frames message while the native view renders and falls back when it cannot. Move the dtuhidd message builders into a header shared by the helper and unit tests. Live verification on Xcode 27 is still outstanding. Fix the macOS-only unit failures: the remote Web proxy could not bind its 127.0.0.2 source address on macOS (remote Web never worked on Macs), and several tests depended on Linux/Windows behaviour (serialized system(), Crow shutdown race, autofs /home, browser tools, label width, a dangling pointer). Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 11 +- CMakeLists.txt | 12 +- cmake/acecode_desktop.cmake | 14 +- cmake/acecode_layer_contract.cmake | 2 +- cmake/acecode_layer_libraries.cmake | 2 + cmake/acecode_source_paths.cmake | 29 +- docs/daemon-api.md | 12 +- docs/ios-simulator.md | 7 +- openspec/changes/add-ios-simulator/design.md | 12 +- .../specs/ios-simulator/spec.md | 14 + openspec/changes/add-ios-simulator/tasks.md | 6 +- .../changes/add-ios-simulator/verification.md | 13 +- .../tool/ios_simulator/dtuhid_messages.hpp | 133 +++++++ .../tool/ios_simulator/helper/helper_main.mm | 1 + .../tool/ios_simulator/helper/sim_ax.mm | 10 +- .../tool/ios_simulator/helper/sim_core.h | 18 +- .../tool/ios_simulator/helper/sim_core.mm | 37 +- .../tool/ios_simulator/helper/sim_display.mm | 133 ++----- .../ios_simulator/helper/sim_framebuffer.h | 39 ++ .../ios_simulator/helper/sim_framebuffer.mm | 173 +++++++++ .../tool/ios_simulator/helper/sim_hid.mm | 73 +--- .../tool/ios_simulator/helper/sim_output.h | 16 + .../tool/ios_simulator/helper/sim_output.mm | 45 +++ .../tool/ios_simulator/helper_client.cpp | 7 +- .../tool/ios_simulator/helper_client.hpp | 8 +- .../tool/ios_simulator/helper_sandbox.cpp | 175 +++++++++ .../tool/ios_simulator/helper_sandbox.hpp | 55 +++ src/adapters/tool/ios_simulator/runtime.cpp | 46 ++- src/adapters/tool/ios_simulator/runtime.hpp | 3 + src/apps/desktop/ios_simulator_view.cpp | 30 ++ src/apps/desktop/ios_simulator_view.hpp | 58 +++ .../desktop/ios_simulator_view_bridge.cpp | 54 +++ .../desktop/ios_simulator_view_layout.cpp | 130 +++++++ .../desktop/ios_simulator_view_layout.hpp | 68 ++++ src/apps/desktop/ios_simulator_view_mac.mm | 339 ++++++++++++++++++ src/apps/desktop/main.cpp | 6 +- src/apps/tui/settings/settings_center.cpp | 8 +- src/apps/web/remote_web_proxy.cpp | 70 +++- src/apps/web/routes/routes_ios_simulator.cpp | 98 +++-- tests/CMakeLists.txt | 10 + tests/channels/core/state_log_test.cpp | 4 +- .../helpers/ios_simulator_view_mac_smoke.mm | 151 ++++++++ .../ios_simulator_view_layout_test.cpp | 126 +++++++ tests/network/websocket_client_test.cpp | 4 + tests/sandbox/sandbox_policy_test.cpp | 10 +- tests/tool/builtin_tool_registry_test.cpp | 5 +- .../ios_simulator/dtuhid_messages_test.cpp | 121 +++++++ .../ios_simulator/helper_sandbox_test.cpp | 142 ++++++++ .../ios_simulator/ios_simulator_e2e_test.cpp | 6 + tests/utils/state_file_test.cpp | 30 +- tests/web/remote_web_proxy_test.cpp | 8 + web/scripts/i18n-en-overrides.mjs | 1 + web/src/components/IosSimulatorPanel.jsx | 144 +++++++- web/src/i18n/sourceCatalog.generated.js | 2 + web/src/lib/iosSimulator.js | 11 +- web/src/lib/iosSimulator.test.js | 5 + web/src/lib/iosSimulatorNativeView.js | 105 ++++++ web/src/lib/iosSimulatorNativeView.test.js | 135 +++++++ web/src/lib/iosSimulatorSettings.js | 5 + web/src/lib/iosSimulatorSettings.test.js | 7 + web/src/lib/runTests.js | 1 + 61 files changed, 2686 insertions(+), 314 deletions(-) create mode 100644 src/adapters/tool/ios_simulator/dtuhid_messages.hpp create mode 100644 src/adapters/tool/ios_simulator/helper/sim_framebuffer.h create mode 100644 src/adapters/tool/ios_simulator/helper/sim_framebuffer.mm create mode 100644 src/adapters/tool/ios_simulator/helper/sim_output.h create mode 100644 src/adapters/tool/ios_simulator/helper/sim_output.mm create mode 100644 src/adapters/tool/ios_simulator/helper_sandbox.cpp create mode 100644 src/adapters/tool/ios_simulator/helper_sandbox.hpp create mode 100644 src/apps/desktop/ios_simulator_view.cpp create mode 100644 src/apps/desktop/ios_simulator_view.hpp create mode 100644 src/apps/desktop/ios_simulator_view_bridge.cpp create mode 100644 src/apps/desktop/ios_simulator_view_layout.cpp create mode 100644 src/apps/desktop/ios_simulator_view_layout.hpp create mode 100644 src/apps/desktop/ios_simulator_view_mac.mm create mode 100644 tests/desktop/helpers/ios_simulator_view_mac_smoke.mm create mode 100644 tests/desktop/ios_simulator_view_layout_test.cpp create mode 100644 tests/tool/ios_simulator/dtuhid_messages_test.cpp create mode 100644 tests/tool/ios_simulator/helper_sandbox_test.cpp create mode 100644 web/src/lib/iosSimulatorNativeView.js create mode 100644 web/src/lib/iosSimulatorNativeView.test.js diff --git a/CLAUDE.md b/CLAUDE.md index a6ce78dbd..aa26af838 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -353,6 +353,8 @@ Daemon startup writes pid, port, guid, token, and heartbeat files under ` `DEVELOPER_DIR` > xcode-select(必须是完整 Xcode)> /Applications、~/Applications、Spotlight 里最新的。simctl / xcodebuild 直接执行 `/usr/bin/…`,它们按自身位置解析 Xcode,所以**不要**改成 `xcrun`(那会回到 xcode-select)。SimulatorKit 在 Xcode 27 挪到 `Contents/SharedFrameworks`,两处都要找。 - **HID**:CoreSimulator < 1155.4 走 Indigo(`SimDeviceLegacyHIDClient` + `IndigoHIDMessageFor*`;单指触摸要手工重封成 0x140 字节两 payload 消息,第二个 payload 在 0xB0、field1=1/field2=2);≥ 1155.4(Xcode 27)走 dtuhidd 数字化器 XPC(`xpc_endpoint_create_mach_port_4sim`),guest 不再可靠接收 Indigo。手势/按键规划在 daemon(`input_plan.cpp`,纯函数有单测),时序在 helper 里执行。非 ASCII 文本走 `simctl pbcopy` + Cmd+V。 - **坐标 = 界面方向下的 point**:给模型的截图 1 像素 = 1 point 且转正,触摸在 helper 里反旋转。朝向来源:Xcode 27 runtime 用 guest 的 CoreDevice display-info(`com.apple.coredevice.feature.getdisplayinfo`);更老的 runtime 没有这个服务(405),退回 Simulator.app 偏好 `DevicePreferences..SimulatorWindowOrientation`。无障碍坐标本来就是界面方向。 -- **帧缓冲读取必须同步加锁**:`kIOSurfaceLockAvoidSync` 返回 `kIOReturnCannotLock`,无锁读到的是陈旧 CPU 副本(实测连续帧逐字节相同)。Intel Mac 上同步锁会等模拟器 GPU,动画期间能卡数秒 —— `simctl io screenshot` 同样卡,是平台特性;所以截图/推流在每设备独立队列,和输入互不阻塞;缩放旋转用 vImage(CPU),Core Image / VTPixelTransfer 同样被 GPU 同步拖住。原生零拷贝视图(IOSurface 进 CALayer)是后续项。 -- **无障碍**:必须把 bridge token 代理装在 `AXPTranslator.sharedInstance`(装在 `sharediOSInstance` 上元素读属性时收不到回调,只剩根节点);属性用私有 `accessibilityMultipleAttributes:` 批量读(一元素一次往返);开机/启动 App 后头几次 frontmost 请求常为空,重试 3 次。 +- **帧缓冲读取必须同步加锁**:`kIOSurfaceLockAvoidSync` 返回 `kIOReturnCannotLock`,无锁读到的是陈旧 CPU 副本(实测连续帧逐字节相同)。Intel Mac 上同步锁会等模拟器 GPU,动画期间能卡数秒 —— `simctl io screenshot` 同样卡,是平台特性;所以截图/推流在每设备独立队列,和输入互不阻塞;缩放旋转用 vImage(CPU),Core Image / VTPixelTransfer 同样被 GPU 同步拖住。Desktop 里改用下面的原生视图,不读回。 +- **无障碍**:必须把 bridge token 代理装在 `AXPTranslator.sharedInstance`(装在 `sharediOSInstance` 上元素读属性时收不到回调,只剩根节点);属性用私有 `accessibilityMultipleAttributes:` 批量读(一元素一次往返);开机/启动 App 后头几秒 frontmost 请求常为空,按 0.4/0.7/1/1.5/2 秒退避重试(刚 `simctl boot` 完 1.4 秒不够)。 - **权限**:`ios_simulator` 进 `PermissionManager::should_auto_allow` 免确认名单(同 Agent Browser),但每台模拟器首次使用经 `ctx.ask_user_questions` 征求同意,记在 state.json `ios_simulator.device_consent`,超时/取消/未选一律算拒绝,yolo 隐式授权,设置页可撤销;`ios_simulator_build` 会执行工程脚本,走普通权限确认。plan 模式只允许观察类动作。 - **面板**:`WS /ws/ios-simulator/` 推 JPEG 二进制帧(带 JSON 头),收触摸/按键/文本;同一设备多个观看者共用一路 helper 流;订阅在后台线程里做,别放回 Crow IO 线程(首次要做 Xcode 发现 + 起 helper)。用户在面板里的操作不需要设备授权(授权约束的是 AI)。 -- **验证现状**:本机 Xcode 26.6(绿色安装于 ~/Downloads)+ iOS 26.5 实测通过 describe / 截图(竖横屏)/ 点击 / 滑动 / 键盘 / 无障碍 / 推流;DTUHID 通道待 Xcode 27 机器验证。回归:`tests/tool/ios_simulator/*`、打包脚本测试(`tests/scripts/*` 中的 acecode-ios-sim 断言)。 +- **helper 跑在 Seatbelt 里**(`helper_sandbox.cpp`,ACECode 自写的 profile,路径一律走 `sandbox-exec -D` 参数不拼进 SBPL):allow default + 禁 IP 网络 + 禁读 /Users、/Volumes、家目录下的文件内容(放行 helper 所在 .app 包、所选 Xcode、`~/Library/Developer/CoreSimulator|CoreDevice`、`~/Library/Logs/CoreSimulator`、三个模拟器偏好 plist、DARWIN 临时 / 缓存目录)+ 只许写 /dev、模拟器数据 / 日志与临时目录。两个坑:(1) CFBundle 定位主包要**列出可执行文件往上两级目录**(猜 X.app/Contents/MacOS 结构),不放行时主包为 NULL,CoreSimulator 的 `NSUserDefaults simulatorDefaults` 直接段错误 —— `listable_dirs` 只给列名,不给读内容;(2) 家目录取 `getpwuid`,CoreSimulator 不认 `$HOME`(假 HOME 下照样用真实设备集),按 `$HOME` 放行会把真实设备数据挡掉。sandbox-exec 应用失败(ACECode 自己已在沙盒里)时 hello 报 HELPER_EXITED,runtime 退回无沙盒启动并在 status 的 `helper.sandboxed/sandbox_note` 与日志里说明;`ACECODE_IOS_SIM_SANDBOX=0` 手动关闭。 +- **Desktop 原生零拷贝视图**(`src/apps/desktop/ios_simulator_view*`):把模拟器 IOSurface 直接设成 CALayer contents,放在网页面板屏幕区域上方,每帧回调 `-setContentsChanged`(私有但 WebKit / Simulator 同用,缺失时重设 contents)。helper 与 Desktop 共用 `acecode_ios_sim_native` 静态库(`sim_core` / `sim_guest` / `sim_framebuffer`;stdout 写帧的 `sim_output` 只在 helper)。Desktop **只加载系统路径的 CoreSimulator**:屏幕描述符是 CoreSimulatorService 的 ROCK 代理,不需要 SimulatorKit,网页传来的 developer_dir 只作为字符串交给 CoreSimulator,绝不据此加载代码。视图 `hitTest` 恒 nil,输入照旧走网页 canvas → WS → helper;遮挡矩形与 Agent Browser 同一套(`nativeSurfaceOverlayGeometryByDocument`),macOS 上 CSS px = point。翻转视图的 backing layer `geometryFlipped=YES`,所以正角度 = 顺时针,与 helper 截图的顺时针旋转一致(CARenderer 离屏渲染验证过)。面板逻辑在 `web/src/lib/iosSimulatorNativeView.js`:原生 `rendering` 为真才对该 socket 发 `{type:"frames",enabled:false}` 停 JPEG(省掉编码),失败 / 被对话框挡住即恢复;socket 在原生视图上线后才连上时要在 `onopen` 补发,否则 JPEG 一直推。三个踩过的坑:ObjC block 捕获 C++ **引用参数时按引用捕获**,异步 block 里用之前先拷成局部值(否则读到已析构的 layout 字符串,json dump 报非法 UTF-8);`FramebufferSource` 构造函数返回前帧回调就可能到,回调只能经加锁的 `current_surface()` 访问 source;同一 udid 只 attach 一次,失败后靠页面「重新连接」重挂,别在每次布局时重试。回归:`ios_simulator_view_mac_smoke`(opt-in,真机模拟器上查 IOSurface contents / 几何 / 遮罩 / 显隐 / 卸载)、`tests/desktop/ios_simulator_view_layout_test.cpp`、前端 `iosSimulatorNativeView.test.js`。 +- **DTUHID 报文**在 header-only 的 `dtuhid_messages.hpp`(helper 用 ARC、单测用纯 C++ 各自编译),schema / 相位 / 归一化 / 存活探测都有单测;本机没有 Xcode 27,真实 dtuhidd 链路仍未实测。 +- **验证现状**:本机 Xcode 26.6(绿色安装于 ~/Downloads)+ iOS 26.5:沙盒内 helper 全部操作、E2E(断言 sandboxed)、原生视图 smoke、浏览器里注入假桥接验证面板切换 / 回退 / 输入映射均通过。回归:`tests/tool/ios_simulator/*`、打包脚本测试(`tests/scripts/*` 中的 acecode-ios-sim 断言)。**别在这台 Mac 上跑 lldb**:开发者模式关闭,debugserver 会弹 SecurityAgent 密码框,弹框挂着期间所有非 Apple 签名的可执行文件(ninja、测试二进制、node)exec 都卡在 U 状态。 ### Web UI: 控制台停靠区(ConsoleDock) diff --git a/CMakeLists.txt b/CMakeLists.txt index 633a5517a..bca9ea9be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -243,14 +243,18 @@ endif() if(APPLE) # iOS Simulator helper. It is a macOS-only payload: other platforms never # build or package it. + add_library(acecode_ios_sim_native STATIC ${ACECODE_IOS_SIM_NATIVE_SOURCES}) + set_target_properties(acecode_ios_sim_native PROPERTIES OBJCXX_STANDARD 17 OBJCXX_STANDARD_REQUIRED YES) + target_compile_options(acecode_ios_sim_native PRIVATE -fobjc-arc -mmacosx-version-min=14.0) + target_include_directories(acecode_ios_sim_native PUBLIC ${CMAKE_SOURCE_DIR}/src/adapters) + target_link_libraries(acecode_ios_sim_native PUBLIC nlohmann_json::nlohmann_json + "-framework Foundation" "-framework IOSurface") add_executable(acecode-ios-sim ${ACECODE_IOS_SIM_HELPER_SOURCES}) set_target_properties(acecode-ios-sim PROPERTIES OBJCXX_STANDARD 17 OBJCXX_STANDARD_REQUIRED YES) target_compile_options(acecode-ios-sim PRIVATE -fobjc-arc -mmacosx-version-min=14.0) target_link_options(acecode-ios-sim PRIVATE -mmacosx-version-min=14.0) - target_include_directories(acecode-ios-sim PRIVATE ${CMAKE_SOURCE_DIR}/src/adapters) - target_link_libraries(acecode-ios-sim PRIVATE nlohmann_json::nlohmann_json - "-framework Foundation" "-framework CoreGraphics" "-framework Accelerate" - "-framework ImageIO" "-framework IOSurface") + target_link_libraries(acecode-ios-sim PRIVATE acecode_ios_sim_native + "-framework CoreGraphics" "-framework Accelerate" "-framework ImageIO") install(TARGETS acecode-ios-sim RUNTIME DESTINATION . COMPONENT ios_simulator_runtime) endif() diff --git a/cmake/acecode_desktop.cmake b/cmake/acecode_desktop.cmake index fdb8b3714..6cbf8f24a 100644 --- a/cmake/acecode_desktop.cmake +++ b/cmake/acecode_desktop.cmake @@ -71,14 +71,18 @@ FetchContent_MakeAvailable(webview) # acecode_testable 把 agent/TUI/web assets 全部拖入桌面壳。 if(APPLE) set(ACECODE_AGENT_BROWSER_HOST_SOURCE ${ACECODE_AGENT_BROWSER_HOST_MAC_SOURCE}) - # The macOS office host is the only ARC translation unit in this target. + # The macOS office host and the iOS Simulator view are the ARC + # translation units in this target. set(ACECODE_DESKTOP_PET_SOURCE ${ACECODE_DESKTOP_PET_MAC_SOURCE}) - set_source_files_properties(${ACECODE_DESKTOP_PET_MAC_SOURCE} + set(ACECODE_IOS_SIMULATOR_VIEW_SOURCE ${ACECODE_IOS_SIMULATOR_VIEW_MAC_SOURCE}) + set_source_files_properties(${ACECODE_DESKTOP_PET_MAC_SOURCE} ${ACECODE_IOS_SIMULATOR_VIEW_MAC_SOURCE} PROPERTIES COMPILE_OPTIONS "-fobjc-arc") endif() set(ACECODE_DESKTOP_SOURCES ${ACECODE_AGENT_BROWSER_HOST_SOURCE} + ${ACECODE_IOS_SIMULATOR_VIEW_SOURCE} + ${ACECODE_IOS_SIMULATOR_VIEW_BRIDGE_SOURCE} ${ACECODE_DESKTOP_MAIN_SOURCE} ${ACECODE_DESKTOP_SPLASH_SOURCE} ${ACECODE_DESKTOP_WEB_HOST_SOURCE} @@ -115,6 +119,12 @@ target_link_libraries(acecode-desktop PRIVATE nlohmann_json::nlohmann_json ) +if(APPLE) + # Zero-copy simulator screen: shares the helper's CoreSimulator access. + target_link_libraries(acecode-desktop PRIVATE acecode_ios_sim_native + "-framework QuartzCore" "-framework IOSurface") +endif() + if(ACECODE_DEEPIN) # DTK package configs can set directory-wide definitions/includes. Keep # their discovery in a child directory so they cannot affect the daemon, diff --git a/cmake/acecode_layer_contract.cmake b/cmake/acecode_layer_contract.cmake index 1813574bf..b0be14395 100644 --- a/cmake/acecode_layer_contract.cmake +++ b/cmake/acecode_layer_contract.cmake @@ -26,7 +26,7 @@ if(TARGET acecode_computer_use_native) list(APPEND _primary_targets acecode_computer_use_native acecode-computer-use) endif() if(TARGET acecode-ios-sim) - list(APPEND _primary_targets acecode-ios-sim) + list(APPEND _primary_targets acecode_ios_sim_native acecode-ios-sim) endif() if(TARGET winpty_static) list(APPEND _primary_targets winpty_static) diff --git a/cmake/acecode_layer_libraries.cmake b/cmake/acecode_layer_libraries.cmake index 1500fa923..a12536505 100644 --- a/cmake/acecode_layer_libraries.cmake +++ b/cmake/acecode_layer_libraries.cmake @@ -169,6 +169,8 @@ set(ACECODE_DESKTOP_BINARY_ONLY_SOURCES ${ACECODE_MACOS_CLI_LAUNCHER_SOURCE} ${ACECODE_DEEPIN_WINDOW_EFFECTS_SOURCE} ${ACECODE_DESKTOP_LINUX_SOURCE} ${ACECODE_AGENT_BROWSER_HOST_SOURCE} ${ACECODE_AGENT_BROWSER_HOST_MAC_SOURCE} + ${ACECODE_IOS_SIMULATOR_VIEW_SOURCE} ${ACECODE_IOS_SIMULATOR_VIEW_MAC_SOURCE} + ${ACECODE_IOS_SIMULATOR_VIEW_BRIDGE_SOURCE} ${ACECODE_DESKTOP_MAIN_SOURCE} ${ACECODE_DESKTOP_SPLASH_SOURCE} ${ACECODE_DESKTOP_WEB_HOST_SOURCE} ${ACECODE_DESKTOP_PET_SOURCE} ${ACECODE_DESKTOP_PET_MAC_SOURCE}) diff --git a/cmake/acecode_source_paths.cmake b/cmake/acecode_source_paths.cmake index 34386aab2..b732cf46f 100644 --- a/cmake/acecode_source_paths.cmake +++ b/cmake/acecode_source_paths.cmake @@ -13,6 +13,9 @@ set(ACECODE_DESKTOP_PET_MAC_SOURCE "${CMAKE_SOURCE_DIR}/src/apps/desktop/desktop set(ACECODE_DESKTOP_LINUX_SOURCE "${CMAKE_SOURCE_DIR}/src/apps/desktop/linux_desktop.cpp") set(ACECODE_AGENT_BROWSER_HOST_SOURCE "${CMAKE_SOURCE_DIR}/src/apps/desktop/agent_browser_host.cpp") set(ACECODE_AGENT_BROWSER_HOST_MAC_SOURCE "${CMAKE_SOURCE_DIR}/src/apps/desktop/agent_browser_host_mac.mm") +set(ACECODE_IOS_SIMULATOR_VIEW_SOURCE "${CMAKE_SOURCE_DIR}/src/apps/desktop/ios_simulator_view.cpp") +set(ACECODE_IOS_SIMULATOR_VIEW_MAC_SOURCE "${CMAKE_SOURCE_DIR}/src/apps/desktop/ios_simulator_view_mac.mm") +set(ACECODE_IOS_SIMULATOR_VIEW_BRIDGE_SOURCE "${CMAKE_SOURCE_DIR}/src/apps/desktop/ios_simulator_view_bridge.cpp") set(ACECODE_BROWSER_CDP_SOURCE "${CMAKE_SOURCE_DIR}/src/adapters/tool/agent_browser/cdp_client.cpp") set(ACECODE_BROWSER_POINTER_SOURCE "${CMAKE_SOURCE_DIR}/src/adapters/tool/agent_browser/browser_pointer_overlay.cpp") set(ACECODE_PROCESS_RESOURCES_SOURCE "${CMAKE_SOURCE_DIR}/src/base/platform/process/unique_resources.cpp") @@ -32,15 +35,20 @@ set(ACECODE_COMPUTER_USE_MAC_SOURCES ${CMAKE_SOURCE_DIR}/src/adapters/computer_use/macos_capture.mm ${CMAKE_SOURCE_DIR}/src/adapters/computer_use/macos_input.mm ${CMAKE_SOURCE_DIR}/src/adapters/computer_use/macos_pointer.mm) -# iOS Simulator helper (macOS only): Objective-C++ over Xcode's private -# CoreSimulator / SimulatorKit, loaded at runtime. +# iOS Simulator (macOS only): Objective-C++ over Xcode's private +# CoreSimulator / SimulatorKit, loaded at runtime. The native library is the +# simulator access shared by the acecode-ios-sim helper and the Desktop +# shell's zero-copy view; the rest is helper-only. +set(ACECODE_IOS_SIM_NATIVE_SOURCES + ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_core.mm + ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_guest.mm + ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_framebuffer.mm) set(ACECODE_IOS_SIM_HELPER_SOURCES ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/helper_main.mm - ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_core.mm + ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_output.mm ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_hid.mm ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_display.mm - ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_ax.mm - ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_guest.mm) + ${CMAKE_SOURCE_DIR}/src/adapters/tool/ios_simulator/helper/sim_ax.mm) set(ACECODE_NATIVE_BRIDGE_MAC_SOURCES ${CMAKE_SOURCE_DIR}/src/base/platform/native_ui/folder_picker_mac.mm ${CMAKE_SOURCE_DIR}/src/base/platform/native_ui/notifications_mac.mm) @@ -86,6 +94,9 @@ set(ACECODE_AGENT_BROWSER_HOST_MAC_SMOKE_SOURCES ${ACECODE_BROWSER_CDP_SOURCE}) set(ACECODE_MACOS_NOTIFICATION_SMOKE_SOURCES ${CMAKE_SOURCE_DIR}/tests/desktop/helpers/macos_notification_smoke.mm) +set(ACECODE_IOS_SIMULATOR_VIEW_MAC_SMOKE_SOURCES + ${CMAKE_SOURCE_DIR}/tests/desktop/helpers/ios_simulator_view_mac_smoke.mm + ${ACECODE_IOS_SIMULATOR_VIEW_MAC_SOURCE}) # 不按当前平台跳过校验：在 Windows configure 时也能发现 .mm 路径拼错。 acecode_require_sources("shared target source paths" @@ -102,6 +113,9 @@ acecode_require_sources("shared target source paths" ${ACECODE_DESKTOP_LINUX_SOURCE} ${ACECODE_AGENT_BROWSER_HOST_SOURCE} ${ACECODE_AGENT_BROWSER_HOST_MAC_SOURCE} + ${ACECODE_IOS_SIMULATOR_VIEW_SOURCE} + ${ACECODE_IOS_SIMULATOR_VIEW_MAC_SOURCE} + ${ACECODE_IOS_SIMULATOR_VIEW_BRIDGE_SOURCE} ${ACECODE_BROWSER_CDP_SOURCE} ${ACECODE_BROWSER_POINTER_SOURCE} ${ACECODE_COMPUTER_USE_RUNTIME_SOURCE} @@ -111,6 +125,8 @@ acecode_require_sources("shared target source paths" ${ACECODE_COMPUTER_USE_MAC_MAIN_SOURCE} ${ACECODE_COMPUTER_USE_WINDOWS_SOURCES} ${ACECODE_COMPUTER_USE_MAC_SOURCES} + ${ACECODE_IOS_SIM_NATIVE_SOURCES} + ${ACECODE_IOS_SIM_HELPER_SOURCES} ${ACECODE_NATIVE_BRIDGE_MAC_SOURCES} ${ACECODE_DESKTOP_OBJCXX_SOURCES} ${ACECODE_UPGRADE_MAC_SOURCE}) @@ -124,5 +140,6 @@ if(BUILD_TESTING) ${ACECODE_AGENT_BROWSER_HOST_SMOKE_SOURCES} ${ACECODE_AGENT_BROWSER_POINTER_DEMO_SOURCES} ${ACECODE_AGENT_BROWSER_HOST_MAC_SMOKE_SOURCES} - ${ACECODE_MACOS_NOTIFICATION_SMOKE_SOURCES}) + ${ACECODE_MACOS_NOTIFICATION_SMOKE_SOURCES} + ${ACECODE_IOS_SIMULATOR_VIEW_MAC_SMOKE_SOURCES}) endif() diff --git a/docs/daemon-api.md b/docs/daemon-api.md index 89ca5c5b3..318911bc4 100644 --- a/docs/daemon-api.md +++ b/docs/daemon-api.md @@ -70,7 +70,7 @@ Remote Web access uses a distinct supervised proxy process configured by | Bind | Token required? | |---|---| | Loopback (`127.0.0.1`, `localhost`, `::1`) | Optional for same-origin loopback requests | -| Proxy-originated loopback (`127.0.0.2`) | Explicit token required | +| Proxy-originated (`127.0.0.2`; on macOS, where only 127.0.0.1 exists on lo0, one of the machine's own non-loopback IPv4 addresses) | Explicit token required | | Non-loopback remote client | Explicit token required | Token locations: @@ -5754,7 +5754,7 @@ offset). A corrupt file blocks that platform with state `error`; it is never reset silently. WhatsApp (section 18) is unaffected and is not managed here. All routes accept only direct local clients. Requests forwarded by the remote -Web proxy (source `127.0.0.2`) or from any other address return +Web proxy (source `127.0.0.2`, or a local non-loopback address on macOS) or from any other address return `403 {"error":"LOCAL_ONLY"}` even with a valid token. Errors use `{"error": CODE, "message": text}`: `400 CHANNEL_ERROR` or `BAD_REQUEST` (messages are user-facing Chinese text without credentials), @@ -5941,7 +5941,7 @@ Backs the `ios_simulator` / `ios_simulator_build` tools, Settings > Tools > iOS | Route | Purpose | |---|---| -| `GET /api/ios-simulator/status[?refresh=1]` | `{supported, enabled, config:{xcode_path, stream_fps, stream_scale, stream_quality}, helper:{installed, running, developer_dir}, sessions:{:}, xcode:{installs[], selected, selected_source, configured_path, configured_error, error, xcode_select_path, core_simulator_version}, device_consent:{:{name, granted_at}}}`. `refresh=1` reruns discovery including Spotlight. | +| `GET /api/ios-simulator/status[?refresh=1]` | `{supported, enabled, config:{xcode_path, stream_fps, stream_scale, stream_quality}, helper:{installed, running, developer_dir, sandboxed, sandbox_note}, sessions:{:}, xcode:{installs[], selected, selected_source, configured_path, configured_error, error, xcode_select_path, core_simulator_version}, device_consent:{:{name, granted_at}}}`. `refresh=1` reruns discovery including Spotlight. | | `PUT /api/config/ios-simulator` | Body: any of `enabled`, `xcode_path` (`""` = automatic), `stream_fps` (5–60), `stream_scale` (0.25–1), `stream_quality` (0.3–0.95). Saves config.json (sparse), applies it and (un)registers the tools; returns the status. `400 {error:"XCODE_INVALID", message}` for an unusable Xcode path. | | `POST /api/ios-simulator/xcode/inspect` | `{path}` → `{ok:true, install}` or `{ok:false, message}`. Accepts `…/Xcode.app` or `…/Xcode.app/Contents/Developer`. | | `GET /api/ios-simulator/devices[?session=]` | `{devices:[{udid, name, state, booted, runtime, runtime_version, platform, device_type, available, allowed, sessions[]}], error, session_device?}`. | @@ -5949,10 +5949,12 @@ Backs the `ios_simulator` / `ios_simulator_build` tools, Settings > Tools > iOS | `POST /api/ios-simulator/devices//consent` | `{allowed, name}` grants or revokes the agent's permission to control that simulator. | | `GET /api/ios-simulator/devices//screenshot` | Full-resolution PNG, turned to the interface orientation. | -`WS /ws/ios-simulator/?token=…` (token rules as `/ws/sessions`): +`WS /ws/ios-simulator/?token=…[&frames=0]` (token rules as `/ws/sessions`). `frames=0` opens the socket for input and status only (no JPEG stream); the Desktop shell uses it while it draws the screen natively. - Server → client text: `{type:"device", device:{udid, name, state, runtime, device_type, product_family, point_width, point_height, scale, rotation, hid_transport}}`, `{type:"ended", reason}`, `{type:"error", message}`. - Server → client binary: `u32le json_length | {type:"frame", seq, width, height, rotation} | JPEG`. Frames are sent when the screen content changes, at most `stream_fps` per second, already turned to the interface orientation. All viewers of one device share one helper stream. -- Client → server text: `{type:"touch", phase:"down"|"move"|"up", x, y, edge?}` (device points in the frame's orientation), `{type:"button", name}` (`home`, `lock`, `side_button`, `siri`, `volume_up`, `volume_down`), `{type:"key", name}` (e.g. `return`, `cmd+a`), `{type:"type", text}` (keyboard), `{type:"text", text}` (paste; non-ASCII goes through the simulator pasteboard). Panel input does not require device consent: consent gates the agent, not the user. +- Client → server text: `{type:"touch", phase:"down"|"move"|"up", x, y, edge?}` (device points in the frame's orientation), `{type:"button", name}` (`home`, `lock`, `side_button`, `siri`, `volume_up`, `volume_down`), `{type:"key", name}` (e.g. `return`, `cmd+a`), `{type:"type", text}` (keyboard), `{type:"text", text}` (paste; non-ASCII goes through the simulator pasteboard), `{type:"frames", enabled}` (start / stop this socket's JPEG stream; while stopped, touches are bounded by the larger screen side and the helper turns them to the current orientation). Panel input does not require device consent: consent gates the agent, not the user. + +`helper.sandboxed` is true while the helper runs inside its Seatbelt profile; `sandbox_note` explains an unconfined helper (`ACECODE_IOS_SIM_SANDBOX=0`, no `sandbox-exec`, or the profile could not be applied). Tool results of `ios_simulator` carry `metadata.ios_simulator = {action, udid, name, runtime, point_width, point_height, rotation}`; the web UI opens the panel on `attach` and closes it on `detach`. diff --git a/docs/ios-simulator.md b/docs/ios-simulator.md index 5a49f5ffd..6e62cc714 100644 --- a/docs/ios-simulator.md +++ b/docs/ios-simulator.md @@ -24,9 +24,12 @@ The model tool `ios_simulator` accepts actions such as `list_devices`, `boot`, ` `attach` opens a **Simulator** tab in the side panel; you can also open it manually. Click or drag on the screen to touch, use the scroll wheel to scroll, click the screen to give it keyboard focus and type or paste (Chinese and other non-ASCII text is pasted through the simulator pasteboard). The toolbar has Home, Lock and Save screenshot. Stream frame rate, quality and scale are configurable in Settings. +In ACECode Desktop on macOS the screen is drawn natively: the simulator's own framebuffer is shown in a layer over the panel, at full resolution and frame rate, without JPEG encoding or copies (and without the Intel readback stall described below). Touch and keyboard input still go through the panel. A web browser (or a remote Web UI) uses the JPEG stream, and Desktop falls back to it automatically when the native view cannot attach; **Reconnect** retries the native view. The native view keeps the CoreSimulator session of the first Xcode it used; restart Desktop after switching Xcode for it to follow. + ## Notes and limits - The tool uses Apple's private CoreSimulator / SimulatorKit frameworks from your Xcode at runtime (as idb and Xcode's own tools do). Xcode updates can change them; ACECode reports precise errors when something is missing. -- Xcode 27 (CoreSimulator 1155.4+) delivers input through `dtuhidd`; earlier versions use the legacy Indigo channel. Set `ACECODE_IOS_SIM_HID=indigo|dtuhid` to override for diagnosis. -- On Intel Macs, reading the framebuffer waits for the simulator's GPU work, so during long animations the live view and screenshots can lag (Apple's `simctl io screenshot` behaves the same). Input is not affected. +- The helper runs inside a macOS Seatbelt profile: no IP networking, no reading of files under /Users, /Volumes and your home folder except the selected Xcode, the ACECode bundle, simulator data and logs and the simulator preference files, and writes only to simulator data, logs and the temporary/cache folders. If `sandbox-exec` cannot apply the profile (for example when ACECode itself runs inside a sandbox) the helper runs unconfined; Settings shows a warning and the daemon log says why. `ACECODE_IOS_SIM_SANDBOX=0` turns it off for diagnosis. +- Xcode 27 (CoreSimulator 1155.4+) delivers input through `dtuhidd`; earlier versions use the legacy Indigo channel. Set `ACECODE_IOS_SIM_HID=indigo|dtuhid` to override for diagnosis. The `dtuhidd` messages are covered by unit tests but have not yet been tried against an Xcode 27 simulator. +- On Intel Macs, reading the framebuffer waits for the simulator's GPU work, so during long animations the JPEG stream and screenshots can lag (Apple's `simctl io screenshot` behaves the same). Input and the Desktop native view are not affected. - Landscape apps are shown upright; on runtimes older than Xcode 27 the orientation comes from Simulator.app's device window setting. diff --git a/openspec/changes/add-ios-simulator/design.md b/openspec/changes/add-ios-simulator/design.md index 50217d1e2..616b8b80e 100644 --- a/openspec/changes/add-ios-simulator/design.md +++ b/openspec/changes/add-ios-simulator/design.md @@ -6,7 +6,7 @@ Reverse engineering Claude Desktop's iOS Simulator feature showed: a Swift helpe **Goals:** feature parity for driving simulators (observe, touch, type, buttons, apps, logs, accessibility, builds), a live interactive panel, robust Xcode discovery including green installs, macOS-only packaging, no build-time dependency on Xcode private frameworks. -**Non-Goals:** physical devices, Android emulators, tvOS trackpad / iPhone Duo hinge, vendoring FBSimulatorControl (needs xcodegen/xcodebuild and would dominate the build), a Seatbelt sandbox for the helper (follow-up), the native zero-copy desktop view (follow-up; see Decision 6). +**Non-Goals:** physical devices, Android emulators, tvOS trackpad / iPhone Duo hinge, vendoring FBSimulatorControl (needs xcodegen/xcodebuild and would dominate the build), an H.264 stream (the Desktop native view replaces it; browsers keep JPEG). ## Decisions @@ -15,13 +15,17 @@ Reverse engineering Claude Desktop's iOS Simulator feature showed: a Swift helpe 3. **Xcode selection.** configured path > DEVELOPER_DIR > xcode-select (only if it is a full Xcode) > newest of /Applications, ~/Applications and Spotlight (`kMDItemCFBundleIdentifier == 'com.apple.dt.Xcode'`). simctl and xcodebuild are executed from `/usr/bin`, which resolves its own Xcode. SimulatorKit is loaded from `Contents/SharedFrameworks` (Xcode 27) or `Contents/Developer/Library/PrivateFrameworks` (≤ 26). 4. **HID transport.** Indigo on CoreSimulator < 1155.4; dtuhidd digitizer XPC (via `xpc_endpoint_create_mach_port_4sim`) from 1155.4, where the guest drops Indigo input. `ACECODE_IOS_SIM_HID` overrides for diagnosis. Gestures are planned in the daemon (pure, tested: interpolation, edge gestures, two-finger paths, US key map) and executed in the helper so timing is not subject to IPC jitter. Non-ASCII text uses `simctl pbcopy` + Cmd+V. 5. **Coordinates = interface-orientation points.** Screenshots for the model are rendered at 1 px = 1 pt and turned to the interface orientation; touches are unrotated in the helper. Rotation comes from the CoreDevice display-info provider (Xcode 27 runtimes) or, on older runtimes, from Simulator.app's per-device window orientation preference. -6. **Framebuffer reads.** A synchronizing IOSurface lock is required (AvoidSync returns stale CPU memory). On Intel Macs the lock waits for the simulator's GPU work and can stall for the length of an animation — `simctl io screenshot` behaves the same — so captures/streams run on per-device queues separate from input. Scaling/rotation use vImage on the private copy; JPEG via ImageIO. A native IOSurface CALayer view in Desktop would avoid readback entirely and is the planned follow-up. +6. **Framebuffer reads.** A synchronizing IOSurface lock is required (AvoidSync returns stale CPU memory). On Intel Macs the lock waits for the simulator's GPU work and can stall for the length of an animation — `simctl io screenshot` behaves the same — so captures/streams run on per-device queues separate from input. Scaling/rotation use vImage on the private copy; JPEG via ImageIO. The Desktop native view (Decision 10) avoids readback entirely. 7. **Consent and permissions.** `ios_simulator` is auto-approved (like browser tools) and gated by a per-UDID consent question (remembered in state.json, revocable in Settings, timeouts never count as consent, yolo grants implicitly). `ios_simulator_build` runs project scripts and keeps the ordinary permission prompt. State-changing actions are refused in plan mode. 8. **Panel transport.** `WS /ws/ios-simulator/` (daemon token auth like the session WS) carries binary JPEG frames with a small JSON header and accepts touch/button/key/text messages; stream start runs off Crow's IO thread. One helper stream per device is shared by all viewers. 9. **Distribution.** CMake builds `acecode-ios-sim` only `if(APPLE)`; Desktop copies it into `ACECode.app/Contents/MacOS`, standalone packages install component `ios_simulator_runtime`; pkg/update/portable/npm/CI checks require it on macOS only. +10. **Desktop native view.** The shell loads the system CoreSimulator (fixed path) in-process: the screen descriptors are CoreSimulatorService proxies, so SimulatorKit is not needed and nothing is loaded from the web-supplied developer directory. A layer-backed, flipped, click-through NSView over the panel's canvas gets the IOSurface as layer contents, `-setContentsChanged` on frame callbacks, the interface rotation as a clockwise transform and the DOM occlusion rectangles as an even-odd mask (the Agent Browser contract). The panel keeps the WebSocket for input and switches its JPEG stream off with `{type:"frames"}` while the native view renders. Device access code (`sim_core`, `sim_guest`, `sim_framebuffer`) is a static library shared by the helper and the shell. +11. **Seatbelt.** `sandbox-exec -p -D …` with an ACECode-authored allow-default profile: IP networking denied, file contents under /Users, /Volumes and the user-record home denied except the helper bundle, Xcode, simulator data/logs and preference files; writes allowed only to /dev, simulator data/logs and the Darwin temp/cache folders. CFBundle needs to list the executable's ancestors to find the main bundle; without it CoreSimulator crashes on a NULL bundle. If the profile cannot be applied the helper is restarted unconfined and the status reports it. + ## Risks / Trade-offs - Private APIs change with Xcode releases → runtime lookups, explicit errors, version-gated transport; DTUHID path is untested on this machine (Xcode 26.6) and must be verified on Xcode 27. -- Intel readback stalls → acceptable degradation; follow-up native view. -- Accessibility translator warm-up: first request after boot may be empty → three retries. +- Intel readback stalls → affect only the JPEG stream and screenshots; the Desktop native view does not read back. +- Accessibility translator warm-up: requests right after boot may be empty for a few seconds → retries with backoff up to about 5.6 s. +- `-setContentsChanged` is private CALayer API (used by WebKit and Simulator); when missing the view reassigns the contents. diff --git a/openspec/changes/add-ios-simulator/specs/ios-simulator/spec.md b/openspec/changes/add-ios-simulator/specs/ios-simulator/spec.md index 26629acce..f2828fc82 100644 --- a/openspec/changes/add-ios-simulator/specs/ios-simulator/spec.md +++ b/openspec/changes/add-ios-simulator/specs/ios-simulator/spec.md @@ -28,6 +28,20 @@ The system SHALL show a live view of the attached simulator in the side panel, o - **WHEN** the user clicks the panel image - **THEN** the click is delivered as a touch at the corresponding device point +### Requirement: Confined helper +The helper SHALL run inside a macOS Seatbelt profile that denies IP networking, denies reading file contents under /Users, /Volumes and the home directory except the helper bundle, the selected Xcode, simulator data and logs and the simulator preference files, and denies writes outside simulator data, logs and the per-user temporary and cache folders. When the profile cannot be applied the helper SHALL run unconfined, and the status SHALL say so with the reason. + +#### Scenario: Compromised helper reads personal files +- **WHEN** code inside the helper tries to read a file in the user's home folder or open a network connection +- **THEN** the sandbox refuses it while every simulator operation keeps working + +### Requirement: Native view in the macOS Desktop shell +On macOS the Desktop shell SHALL draw the simulator screen from the simulator's own framebuffer surface without copying or encoding it, placed over the panel's screen area and passing input through to the panel. The panel SHALL stop the JPEG stream while the native view renders and SHALL fall back to it when the native view cannot render. The shell SHALL load only the system CoreSimulator framework for this and SHALL NOT load code from a path supplied by the web page. + +#### Scenario: Desktop shows the simulator +- **WHEN** the user opens the simulator panel in ACECode Desktop on a Mac +- **THEN** the screen is composited from the simulator's surface at full frame rate, touches still reach the simulator through the panel, and no JPEG frames are sent to the panel + ### Requirement: macOS-only distribution The helper SHALL be built and packaged only for macOS (app bundle, portable package, installer, update archive, npm darwin packages). Other platforms SHALL neither build nor ship it, and the tools SHALL not be registered there. diff --git a/openspec/changes/add-ios-simulator/tasks.md b/openspec/changes/add-ios-simulator/tasks.md index 424fb7948..70738d0be 100644 --- a/openspec/changes/add-ios-simulator/tasks.md +++ b/openspec/changes/add-ios-simulator/tasks.md @@ -24,5 +24,7 @@ ## 4. Validation - [x] 4.1 Unit tests and frontend tests/build pass. - [x] 4.2 End-to-end on a green-installed Xcode 26.6 with booted simulators: describe, screenshot (portrait and landscape), tap, swipe, keyboard, accessibility, stream. -- [ ] 4.3 Verify the DTUHID path on Xcode 27 / CoreSimulator ≥ 1155.4. -- [ ] 4.4 Follow-ups: Seatbelt profile for the helper; native zero-copy simulator view in Desktop. +- [ ] 4.3 Verify the DTUHID path live on Xcode 27 / CoreSimulator ≥ 1155.4 (message builders extracted to `dtuhid_messages.hpp` and unit tested; no Xcode 27 toolchain on the development machine). +- [x] 4.4 Seatbelt profile for the helper (`helper_sandbox.cpp`, fallback with status note, `ACECODE_IOS_SIM_SANDBOX=0`); unit tests, all helper operations and E2E run sandboxed, negative checks (home read, network, home write) refused. +- [x] 4.5 Native zero-copy simulator view in Desktop (`acecode_ios_sim_native` shared with the helper, `ios_simulator_view_mac.mm`, bridges, panel `frames` toggle and fallback); layout unit tests, opt-in `ios_simulator_view_mac_smoke`, panel checked in a browser with a stubbed bridge. +- [x] 4.6 macOS-only unit test failures fixed (remote Web proxy source address on macOS, cross-process claim workers, Crow shutdown race, autofs `/home`, macOS browser tools, settings label width, channel state log). diff --git a/openspec/changes/add-ios-simulator/verification.md b/openspec/changes/add-ios-simulator/verification.md index 2545ee371..416ab89e0 100644 --- a/openspec/changes/add-ios-simulator/verification.md +++ b/openspec/changes/add-ios-simulator/verification.md @@ -6,13 +6,22 @@ Environment: Xcode 26.6 (17F113) green-installed at `~/Downloads/Xcode.app`, `xc - Helper protocol driven directly: hello, list_devices, describe, screenshot (portrait 390×844 and landscape 874×402 at 1 px = 1 pt), display_info, Home button, swipe between home-screen pages, tap launching Settings/Photos and opening "通用", keyboard events, accessibility tree (Settings, SpringBoard, a landscape game) and point hit test, JPEG stream start/stop. - `acecode_unit_tests --gtest_filter='IosSimulator*:XcodeLocator*:SimctlParse*'`: 30 passed; `IosSimulatorE2E.*` against the booted iPhone 17e (attach with consent, screenshot attachment, home, inspect, swipe, cmd+a, out-of-screen rejection, stream frames): passed. -- Full unit suite: 6281 passed; 11 failures and one crash (`RemoteWebTcpProxy`) are in remote-web proxy, WebSocket client, cross-process state lock, sandbox path, TUI settings rendering, channel state log and macOS browser-tool registration tests that do not touch this change (macOS host only; CI runs unit tests on Linux). +- Full unit suite (first pass): 6281 passed; 11 failures and one crash, all macOS-only. Follow-up pass: fixed (remote Web proxy could not bind 127.0.0.2 on macOS — a product bug that broke remote Web on Macs; plus test-environment issues: macOS serializes `system()`, Crow stop drops the close frame, autofs `/home`, stale browser-tool expectations, long temp path truncating a settings label, dangling pointer in a channel log test). Full suite then: 6326 passed, 0 failed. - Daemon on an isolated port: `/api/ios-simulator/status` (Xcode found via Spotlight), `/devices`, `/xcode/inspect`, `/devices//screenshot` (PNG), PUT `/api/config/ios-simulator` (invalid path rejected, valid path saved sparsely) and `WS /ws/ios-simulator/` (device message, JPEG frames, panel Home button). - Web UI in a browser against that daemon: Settings > Tools > iOS 模拟器 (installs, sources, devices with consent switches and boot/shutdown), header button opens the simulator tab, device chooser, live view, drag swipes pages, toolbar Home works. `pnpm test` and `pnpm build` pass. - CI lint set (layers, size, ownership --final, includes, doc paths, map) with the new files staged: no findings. macOS packaging script tests (pkg, update zip, portable zip, release scripts, npm, verify-package) pass. +## Follow-up pass (sandbox, native view) + +- Seatbelt: every helper op (hello, list_devices, describe, display_info, screenshot, ui_describe tree/point, button, touch, keys, start/stop stream, hardware keyboard, release) under the generated profile; the same profile refuses reading `~/.zshrc`, `curl https://example.com` and writing into the home folder. `IosSimulatorE2E.*` now asserts `helper.sandboxed`; passes. Daemon status reports `sandboxed:true`. +- Native view: offscreen CARenderer render of the IOSurface layer for 0/90/180/270° in flipped and unflipped geometry (orientation and clockwise sign); `ios_simulator_view_mac_smoke` on the booted iPhone 17e: IOSurface layer contents (zero copy), DOM frame, rounded corners, occlusion mask, click-through, hide/show without re-attach, unmount, DEVICE_NOT_FOUND for an unknown UDID — SMOKE_OK. +- WebSocket `frames` toggle against a daemon: `frames=0` delivers device info and no JPEG, enabling delivers frames, disabling stops them while touches still work. +- Panel in a browser with a stubbed macOS bridge: layout carries the canvas rect, corner radius and developer dir; `frames:false` sent once the native view renders (including when the socket connects late); taps map to device points from the native framebuffer; a native failure re-enables frames and JPEG resumes; leaving the session unmounts the view. +- DTUHID message builders unit tested (`dtuhid_messages_test.cpp`). + ## Not verified -- DTUHID input on Xcode 27 / CoreSimulator ≥ 1155.4 (no such toolchain on this machine). +- DTUHID input on a live Xcode 27 / CoreSimulator ≥ 1155.4 simulator (no such toolchain on this machine; the iOS 26.5 runtime has no dtuhidd). +- The native view inside the full Desktop app on screen (no Screen Recording permission for this session); covered by the smoke, the CARenderer render and the stubbed-bridge panel test. - Signed/notarized packages (CI). - Automatic panel opening from a real model's `attach` result (covered by frontend unit tests). diff --git a/src/adapters/tool/ios_simulator/dtuhid_messages.hpp b/src/adapters/tool/ios_simulator/dtuhid_messages.hpp new file mode 100644 index 000000000..17e2d51ca --- /dev/null +++ b/src/adapters/tool/ios_simulator/dtuhid_messages.hpp @@ -0,0 +1,133 @@ +#pragma once + +// Messages for the dtuhidd digitizer service that CoreSimulator 1155.4+ +// (Xcode 27) exposes inside a booted simulator. Header-only so the +// acecode-ios-sim helper (Objective-C++, ARC) and the unit tests (plain C++) +// build the same XPC dictionaries. Each builder returns a new dictionary: ARC +// releases it in the helper, plain C++ callers own it (xpc_release). +// +// Wire shape, all keys required: +// envelope {messageType: string, isBarrier: bool, +// featureIdentifier: kDigitizerService, payload: dictionary} +// IndigoDigitizerEvent {pointOne: {x, y}, [pointTwo: {x, y}], +// eventType: 0 began | 1 moved | 2 ended, +// edge: 0 none | 1 top | 2 left | 3 bottom | 4 right, +// target: 0} +// IndigoButtonEvent {usagePage, usageCode, state: 1 down | 2 up} +// IndigoKeyboardButtonEvent {usageCode, state: 1 down | 2 up} +// Points are fractions of the device's portrait point size (0…1). + +#include + +#ifdef __APPLE__ +#include +#endif + +namespace acecode::ios_simulator::dtuhid { + +inline constexpr const char* kDigitizerService = "com.apple.coredevice.feature.remote.hid.digitizer"; +inline constexpr const char* kFirstCoreSimulatorVersion = "1155.4"; +inline constexpr const char* kDigitizerEvent = "IndigoDigitizerEvent"; +inline constexpr const char* kButtonEvent = "IndigoButtonEvent"; +inline constexpr const char* kKeyboardEvent = "IndigoKeyboardButtonEvent"; + +enum class Phase : std::uint64_t { Began = 0, Moved = 1, Ended = 2 }; + +// Down events start a contact and continue it; Up ends it. One tracker per +// contact kind (one finger, two fingers). +class ContactTracker { +public: + Phase next(bool up) { + if (up) { + active_ = false; + return Phase::Ended; + } + if (active_) return Phase::Moved; + active_ = true; + return Phase::Began; + } + bool active() const { return active_; } + +private: + bool active_ = false; +}; + +struct UnitPoint { + double x = 0; + double y = 0; +}; + +inline UnitPoint unit_point(double x, double y, double point_width, double point_height) { + return {point_width > 0 ? x / point_width : 0, point_height > 0 ? y / point_height : 0}; +} + +inline std::uint64_t button_state(bool down) { return down ? 1 : 2; } + +#ifdef __APPLE__ + +inline xpc_object_t point_dictionary(UnitPoint point) { + xpc_object_t value = xpc_dictionary_create(nullptr, nullptr, 0); + xpc_dictionary_set_double(value, "x", point.x); + xpc_dictionary_set_double(value, "y", point.y); + return value; +} + +// Takes the payload by reference: the envelope retains it. +inline xpc_object_t envelope(const char* message_type, xpc_object_t payload, bool barrier) { + xpc_object_t message = xpc_dictionary_create(nullptr, nullptr, 0); + xpc_dictionary_set_string(message, "messageType", message_type); + xpc_dictionary_set_bool(message, "isBarrier", barrier); + xpc_dictionary_set_string(message, "featureIdentifier", kDigitizerService); + xpc_dictionary_set_value(message, "payload", payload); + return message; +} + +inline xpc_object_t digitizer_payload(UnitPoint one, const UnitPoint* two, Phase phase, unsigned edge) { + xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); + xpc_object_t first = point_dictionary(one); + xpc_dictionary_set_value(payload, "pointOne", first); +#if !__has_feature(objc_arc) + xpc_release(first); +#endif + if (two) { + xpc_object_t second = point_dictionary(*two); + xpc_dictionary_set_value(payload, "pointTwo", second); +#if !__has_feature(objc_arc) + xpc_release(second); +#endif + } + xpc_dictionary_set_uint64(payload, "eventType", static_cast(phase)); + xpc_dictionary_set_uint64(payload, "edge", two ? 0 : edge); + xpc_dictionary_set_uint64(payload, "target", 0); + return payload; +} + +inline xpc_object_t button_payload(std::uint16_t usage_page, std::uint16_t usage, bool down) { + xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); + xpc_dictionary_set_uint64(payload, "usagePage", usage_page); + xpc_dictionary_set_uint64(payload, "usageCode", usage); + xpc_dictionary_set_uint64(payload, "state", button_state(down)); + return payload; +} + +inline xpc_object_t keyboard_payload(std::uint32_t usage, bool down) { + xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); + xpc_dictionary_set_uint64(payload, "usageCode", usage); + xpc_dictionary_set_uint64(payload, "state", button_state(down)); + return payload; +} + +// A key-up of usage 0 sent as a barrier: harmless, and dtuhidd replies to it, +// which proves the endpoint is live before real input is sent. +inline xpc_object_t liveness_probe() { + xpc_object_t payload = keyboard_payload(0, false); + xpc_object_t message = envelope(kKeyboardEvent, payload, true); +#if !__has_feature(objc_arc) + xpc_release(payload); +#endif + return message; +} + +#endif // __APPLE__ + +} // namespace acecode::ios_simulator::dtuhid diff --git a/src/adapters/tool/ios_simulator/helper/helper_main.mm b/src/adapters/tool/ios_simulator/helper/helper_main.mm index 542ca481b..8f8146e0b 100644 --- a/src/adapters/tool/ios_simulator/helper/helper_main.mm +++ b/src/adapters/tool/ios_simulator/helper/helper_main.mm @@ -6,6 +6,7 @@ #import "sim_ax.h" #import "sim_core.h" +#import "sim_output.h" #import "sim_display.h" #import "sim_hid.h" #import "private_api.h" diff --git a/src/adapters/tool/ios_simulator/helper/sim_ax.mm b/src/adapters/tool/ios_simulator/helper/sim_ax.mm index 50ac59b7c..774a63056 100644 --- a/src/adapters/tool/ios_simulator/helper/sim_ax.mm +++ b/src/adapters/tool/ios_simulator/helper/sim_ax.mm @@ -1,5 +1,6 @@ #import "sim_ax.h" #import "sim_core.h" +#import "sim_output.h" #import "private_api.h" #include @@ -228,9 +229,12 @@ json describe_ui(id device, const std::string& udid, const UiQuery& query) { ax.accessibilityEnabled = YES; id translation = nil; // The guest accessibility server starts on the first request; the - // first answers after a boot or app launch are often empty. - for (int attempt = 0; attempt < 3 && !translation; ++attempt) { - if (attempt) usleep(700 * 1000); + // first answers after a boot or app launch are often empty for a few + // seconds (observed right after `simctl boot`). + constexpr useconds_t kRetryDelaysMs[] = {0, 400, 700, 1000, 1500, 2000}; + for (useconds_t delay_ms : kRetryDelaysMs) { + if (translation) break; + if (delay_ms) usleep(delay_ms * 1000); translation = query.mode == "point" ? [ax objectAtPoint:CGPointMake(query.x, query.y) displayId:0 bridgeDelegateToken:token] : [ax frontmostApplicationWithDisplayId:0 bridgeDelegateToken:token]; diff --git a/src/adapters/tool/ios_simulator/helper/sim_core.h b/src/adapters/tool/ios_simulator/helper/sim_core.h index 99b24a616..6c239e6c3 100644 --- a/src/adapters/tool/ios_simulator/helper/sim_core.h +++ b/src/adapters/tool/ios_simulator/helper/sim_core.h @@ -1,5 +1,6 @@ -// Shared state for the acecode-ios-sim helper: private framework loading, -// device lookup, ObjC exception containment and the framed stdout writer. +// Simulator access shared by the acecode-ios-sim helper and the Desktop +// shell: private framework loading, device lookup, guest services, display +// rotation and ObjC exception containment. #pragma once #import @@ -37,9 +38,11 @@ struct Environment { std::string load_error; }; -// Loads CoreSimulator (required) and SimulatorKit (input) once. Throws SimError -// when CoreSimulator cannot be loaded. -const Environment& load_frameworks(const std::string& developer_dir); +// Loads CoreSimulator (required, fixed system path) once and, when +// simulator_kit is set, SimulatorKit from the selected Xcode (input +// injection). The first call wins for the life of the process. Throws +// SimError when CoreSimulator cannot be loaded. +const Environment& load_frameworks(const std::string& developer_dir, bool simulator_kit = true); const Environment& environment(); void* simulator_kit_symbol(const char* name); bool load_accessibility_framework(std::string& error); @@ -62,9 +65,4 @@ int display_rotation(id device, const std::string& udid); // Uncached display-info probe with the raw provider reply, for diagnostics. json display_info(id device, const std::string& udid); -// Framed writer (serialized, safe from any queue). Frames with drop_if_busy -// are skipped while more than a couple of droppable frames are pending. -void emit(const json& header, const std::string& binary = {}, bool drop_if_busy = false); -void emit_log(const std::string& level, const std::string& message); - } // namespace acecode::ios_simulator::helper diff --git a/src/adapters/tool/ios_simulator/helper/sim_core.mm b/src/adapters/tool/ios_simulator/helper/sim_core.mm index 7d6866f3b..f7aaae676 100644 --- a/src/adapters/tool/ios_simulator/helper/sim_core.mm +++ b/src/adapters/tool/ios_simulator/helper/sim_core.mm @@ -1,8 +1,6 @@ #import "sim_core.h" #import "private_api.h" -#include "tool/ios_simulator/wire.hpp" - #include #include #include @@ -19,9 +17,6 @@ id g_service_context = nil; id g_device_set = nil; -std::mutex g_write_mu; -std::atomic g_droppable_pending{0}; - NSString* const kCoreSimulatorPath = @"/Library/Developer/PrivateFrameworks/CoreSimulator.framework"; NSString* const kAccessibilityPath = @"/System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework"; @@ -41,17 +36,6 @@ bool load_bundle(NSString* path, std::string& error) { return true; } -void write_all(const std::string& bytes) { - std::size_t sent = 0; - while (sent < bytes.size()) { - const auto count = ::write(STDOUT_FILENO, bytes.data() + sent, bytes.size() - sent); - if (count > 0) { sent += static_cast(count); continue; } - if (count < 0 && errno == EINTR) continue; - // The daemon is gone; there is nobody left to serve. - _exit(0); - } -} - id service_device_set() { if (g_device_set) return g_device_set; Class factory = NSClassFromString(@"SimServiceContext"); @@ -112,7 +96,7 @@ void guarded(const char* code, void (^body)(void)) { } } -const Environment& load_frameworks(const std::string& developer_dir) { +const Environment& load_frameworks(const std::string& developer_dir, bool simulator_kit) { std::lock_guard lock(g_env_mu); if (g_loaded) return g_env; g_env.developer_dir = developer_dir; @@ -125,6 +109,8 @@ throw SimError("CORESIMULATOR_UNAVAILABLE", error + g_env.core_simulator_version = utf8([NSBundle bundleForClass:device_class].infoDictionary[@"CFBundleVersion"]); + g_loaded = true; + if (!simulator_kit) return g_env; NSString* developer = ns(developer_dir); NSArray* candidates = @[ [[developer stringByAppendingPathComponent:@"../SharedFrameworks/SimulatorKit.framework"] @@ -143,7 +129,6 @@ throw SimError("CORESIMULATOR_UNAVAILABLE", error + } if (!g_env.simulator_kit_loaded && g_env.load_error.empty()) g_env.load_error = "SimulatorKit.framework was not found in the selected Xcode."; - g_loaded = true; return g_env; } @@ -233,20 +218,4 @@ json list_devices() { return out; } -void emit(const json& header, const std::string& binary, bool drop_if_busy) { - if (drop_if_busy && g_droppable_pending.load() > 2) return; - std::string frame; - try { frame = wire::encode(header, binary); } catch (const std::exception&) { return; } - if (drop_if_busy) g_droppable_pending.fetch_add(1); - { - std::lock_guard lock(g_write_mu); - write_all(frame); - } - if (drop_if_busy) g_droppable_pending.fetch_sub(1); -} - -void emit_log(const std::string& level, const std::string& message) { - emit({{"event", "log"}, {"level", level}, {"message", message}}); -} - } // namespace acecode::ios_simulator::helper diff --git a/src/adapters/tool/ios_simulator/helper/sim_display.mm b/src/adapters/tool/ios_simulator/helper/sim_display.mm index c941dff16..d2295141b 100644 --- a/src/adapters/tool/ios_simulator/helper/sim_display.mm +++ b/src/adapters/tool/ios_simulator/helper/sim_display.mm @@ -1,6 +1,7 @@ #import "sim_display.h" #import "sim_core.h" -#import "private_api.h" +#import "sim_framebuffer.h" +#import "sim_output.h" #import #import @@ -135,57 +136,25 @@ CGImageRef copy_surface_image(IOSurfaceRef surface, double scale, int rotation, class FramebufferDisplay final : public Display, public std::enable_shared_from_this { public: - FramebufferDisplay(id device, id renderable, std::string udid) - : device_(device), renderable_(renderable), udid_(std::move(udid)) { - token_ = [NSUUID UUID]; - callback_queue_ = dispatch_queue_create("dev.acecode.ios-sim.display.callbacks", DISPATCH_QUEUE_SERIAL); + FramebufferDisplay(id device, std::string udid) : device_(device), udid_(std::move(udid)) { encode_queue_ = dispatch_queue_create("dev.acecode.ios-sim.display.encode", DISPATCH_QUEUE_SERIAL); } - ~FramebufferDisplay() override { unregister(); } - - void register_callbacks() { - std::weak_ptr weak = weak_from_this(); - id renderable = renderable_; - NSUUID* token = token_; - auto changed = ^(id surface) { - if (auto self = weak.lock()) self->surface_changed(surface); - }; - auto frame = ^{ - if (auto self = weak.lock()) self->frame_rendered(); - }; - bool registered = false; - if ([renderable respondsToSelector:@selector(registerScreenCallbacksWithUUID:callbackQueue:frameCallback: - surfacesChangedCallback:propertiesChangedCallback:)]) { - @try { - [(id)renderable registerScreenCallbacksWithUUID:token callbackQueue:callback_queue_ - frameCallback:frame surfacesChangedCallback:^(id primary, id masked) { (void)masked; changed(primary); } - propertiesChangedCallback:^(id properties) { (void)properties; }]; - registered = true; - screen_callbacks_ = true; - } @catch (NSException*) { - @try { [(id)renderable unregisterScreenCallbacksWithUUID:token]; } @catch (NSException*) {} - } - } - if (!registered) { - bool surface_ok = false; - @try { - [(id)renderable registerCallbackWithUUID:token ioSurfacesChangeCallback:changed]; - surface_ok = true; - } @catch (NSException*) {} - @try { - [(id)renderable registerCallbackWithUUID:token ioSurfaceChangeCallback:changed]; - surface_ok = true; - } @catch (NSException*) {} - @try { - [(id)renderable registerCallbackWithUUID:token - damageRectanglesCallback:^(NSArray* rects) { (void)rects; frame(); }]; - } @catch (NSException*) { - if (!surface_ok) throw SimError("DISPLAY_UNAVAILABLE", "Could not subscribe to framebuffer updates."); - } - } + void start() { + std::weak_ptr surface_owner = weak_from_this(); + std::weak_ptr frame_owner = surface_owner; + // Callbacks can fire before the constructor returns; they reach the + // source only through current_surface(), which waits for this store. + auto source = std::make_unique( + device_, + [surface_owner](id surface) { + if (auto self = surface_owner.lock()) self->surface_changed(surface); + }, + [frame_owner]() { + if (auto self = frame_owner.lock()) self->schedule(0); + }); std::lock_guard lock(mu_); - surface_ = initial_surface(); + source_ = std::move(source); } EncodedImage capture(const std::string& format, double scale, double quality) override { @@ -241,30 +210,14 @@ int pixel_height() override { } private: - id initial_surface() { - id surface = nil; - @try { surface = [(id)renderable_ framebufferSurface]; } @catch (NSException*) {} - if (!surface) { - @try { surface = [(id)renderable_ ioSurface]; } @catch (NSException*) {} - } - return surface; - } - id current_surface() { - { - std::lock_guard lock(mu_); - if (surface_) return surface_; - } - id surface = initial_surface(); std::lock_guard lock(mu_); - if (!surface_) surface_ = surface; - return surface_; + return source_ ? source_->surface() : nil; } void surface_changed(id surface) { { std::lock_guard lock(mu_); - surface_ = surface; for (auto& [stream, state] : streams_) state.have_seed = false; } if (!surface) { @@ -280,8 +233,6 @@ void surface_changed(id surface) { schedule(0); } - void frame_rendered() { schedule(0); } - void schedule(double delay_ms) { bool expected = false; if (!scheduled_.compare_exchange_strong(expected, true)) return; @@ -296,11 +247,10 @@ void schedule(double delay_ms) { // elapsed and whose surface content changed; reschedules the rest. void pump() { scheduled_.store(false); - id surface = nil; + id surface = current_surface(); std::map snapshot; { std::lock_guard lock(mu_); - surface = surface_; snapshot = streams_; } if (!surface || snapshot.empty()) return; @@ -352,59 +302,22 @@ void pump() { if (next_delay >= 0) schedule(next_delay); } - void unregister() { - id renderable = renderable_; - NSUUID* token = token_; - if (screen_callbacks_) { - @try { [(id)renderable unregisterScreenCallbacksWithUUID:token]; } @catch (NSException*) {} - } - @try { [(id)renderable unregisterIOSurfacesChangeCallbackWithUUID:token]; } @catch (NSException*) {} - @try { [(id)renderable unregisterIOSurfaceChangeCallbackWithUUID:token]; } @catch (NSException*) {} - @try { [(id)renderable unregisterDamageRectanglesCallbackWithUUID:token]; } @catch (NSException*) {} - } - id device_; - id renderable_; std::string udid_; int last_rotation_ = 0; // encode_queue_ only - NSUUID* token_ = nil; - dispatch_queue_t callback_queue_ = nil; dispatch_queue_t encode_queue_ = nil; - bool screen_callbacks_ = false; std::atomic scheduled_{false}; std::mutex mu_; - id surface_ = nil; std::map streams_; + // Last: destroyed (callbacks unregistered and drained) before the rest. + std::unique_ptr source_; }; -id find_main_renderable(id device) { - __block id found = nil; - __block id fallback = nil; - guarded("DISPLAY_UNAVAILABLE", ^{ - id io = [(id)device io]; - NSArray* ports = io ? [(id)io ioPorts] : nil; - for (id port in ports) { - id descriptor = [(id)port descriptor]; - if (![descriptor respondsToSelector:@selector(framebufferSurface)] && - ![descriptor respondsToSelector:@selector(ioSurface)]) continue; - if (![descriptor respondsToSelector:@selector(state)]) continue; - id state = [(id)descriptor state]; - if (![state respondsToSelector:@selector(displayClass)]) continue; - if ([(id)state displayClass] == 0) { found = descriptor; break; } - if (!fallback) fallback = descriptor; - } - }); - return found ?: fallback; -} - } // namespace std::shared_ptr open_display(id device, const std::string& udid) { - if (!device_booted(device)) throw SimError("DEVICE_NOT_BOOTED", "The simulator is not booted."); - id renderable = find_main_renderable(device); - if (!renderable) throw SimError("DISPLAY_UNAVAILABLE", "Could not find the simulator's main display."); - auto display = std::make_shared(device, renderable, udid); - display->register_callbacks(); + auto display = std::make_shared(device, udid); + display->start(); return display; } diff --git a/src/adapters/tool/ios_simulator/helper/sim_framebuffer.h b/src/adapters/tool/ios_simulator/helper/sim_framebuffer.h new file mode 100644 index 000000000..fab9216a9 --- /dev/null +++ b/src/adapters/tool/ios_simulator/helper/sim_framebuffer.h @@ -0,0 +1,39 @@ +// Subscription to a booted simulator's main-display IOSurface. Shared by the +// helper (JPEG captures and streams) and the Desktop shell (zero-copy layer): +// only CoreSimulator is needed, since the screen descriptors are remote +// proxies served by CoreSimulatorService. +#pragma once + +#import + +#include +#include + +namespace acecode::ios_simulator::helper { + +// The main screen's renderable (display class 0), or the first renderable +// display when none reports class 0. nil when the device exposes none. +id find_main_renderable(id device); + +class FramebufferSource { +public: + // surface is nil when the display went away. Both handlers run on a + // private serial queue and are never called after the destructor returns. + using SurfaceHandler = std::function; + using FrameHandler = std::function; + + // Throws SimError (DEVICE_NOT_BOOTED, DISPLAY_UNAVAILABLE). + FramebufferSource(id device, SurfaceHandler on_surface, FrameHandler on_frame); + ~FramebufferSource(); + FramebufferSource(const FramebufferSource&) = delete; + FramebufferSource& operator=(const FramebufferSource&) = delete; + + // The latest surface (nil until the simulator rendered one). + id surface(); + +private: + struct State; + std::shared_ptr state_; +}; + +} // namespace acecode::ios_simulator::helper diff --git a/src/adapters/tool/ios_simulator/helper/sim_framebuffer.mm b/src/adapters/tool/ios_simulator/helper/sim_framebuffer.mm new file mode 100644 index 000000000..9b2fd4bfe --- /dev/null +++ b/src/adapters/tool/ios_simulator/helper/sim_framebuffer.mm @@ -0,0 +1,173 @@ +#import "sim_framebuffer.h" +#import "sim_core.h" +#import "private_api.h" + +#include + +namespace acecode::ios_simulator::helper { + +struct FramebufferSource::State { + std::mutex mu; + bool closed = false; + SurfaceHandler on_surface; + FrameHandler on_frame; + id surface = nil; + id renderable = nil; + NSUUID* token = nil; + dispatch_queue_t queue = nil; + bool screen_callbacks = false; +}; + +namespace { + +const void* const kQueueKey = &kQueueKey; + +id initial_surface(id renderable) { + id surface = nil; + @try { surface = [(id)renderable framebufferSurface]; } @catch (NSException*) {} + if (!surface) { + @try { surface = [(id)renderable ioSurface]; } @catch (NSException*) {} + } + return surface; +} + +void unregister_callbacks(id renderable, NSUUID* token, bool screen_callbacks) { + if (screen_callbacks) { + @try { [(id)renderable unregisterScreenCallbacksWithUUID:token]; } @catch (NSException*) {} + } + @try { [(id)renderable unregisterIOSurfacesChangeCallbackWithUUID:token]; } @catch (NSException*) {} + @try { [(id)renderable unregisterIOSurfaceChangeCallbackWithUUID:token]; } @catch (NSException*) {} + @try { [(id)renderable unregisterDamageRectanglesCallbackWithUUID:token]; } @catch (NSException*) {} +} + +} // namespace + +id find_main_renderable(id device) { + __block id found = nil; + __block id fallback = nil; + guarded("DISPLAY_UNAVAILABLE", ^{ + id io = [(id)device io]; + NSArray* ports = io ? [(id)io ioPorts] : nil; + for (id port in ports) { + id descriptor = [(id)port descriptor]; + if (![descriptor respondsToSelector:@selector(framebufferSurface)] && + ![descriptor respondsToSelector:@selector(ioSurface)]) continue; + if (![descriptor respondsToSelector:@selector(state)]) continue; + id state = [(id)descriptor state]; + if (![state respondsToSelector:@selector(displayClass)]) continue; + if ([(id)state displayClass] == 0) { found = descriptor; break; } + if (!fallback) fallback = descriptor; + } + }); + return found ?: fallback; +} + +FramebufferSource::FramebufferSource(id device, SurfaceHandler on_surface, FrameHandler on_frame) + : state_(std::make_shared()) { + if (!device_booted(device)) throw SimError("DEVICE_NOT_BOOTED", "The simulator is not booted."); + id renderable = find_main_renderable(device); + if (!renderable) throw SimError("DISPLAY_UNAVAILABLE", "Could not find the simulator's main display."); + state_->on_surface = std::move(on_surface); + state_->on_frame = std::move(on_frame); + state_->renderable = renderable; + state_->token = [NSUUID UUID]; + state_->queue = dispatch_queue_create("dev.acecode.ios-sim.framebuffer", DISPATCH_QUEUE_SERIAL); + dispatch_queue_set_specific(state_->queue, kQueueKey, const_cast(kQueueKey), nullptr); + + // The blocks hold the state weakly: CoreSimulator may keep them after + // unregistering, and they must never keep a closed source alive. + std::weak_ptr weak = state_; + void (^changed)(id) = ^(id surface) { + auto state = weak.lock(); + if (!state) return; + SurfaceHandler handler; + { + std::lock_guard lock(state->mu); + if (state->closed) return; + state->surface = surface; + handler = state->on_surface; + } + if (handler) handler(surface); + }; + void (^frame)(void) = ^{ + auto state = weak.lock(); + if (!state) return; + FrameHandler handler; + { + std::lock_guard lock(state->mu); + if (state->closed) return; + handler = state->on_frame; + } + if (handler) handler(); + }; + + NSUUID* token = state_->token; + bool registered = false; + if ([renderable respondsToSelector:@selector(registerScreenCallbacksWithUUID:callbackQueue:frameCallback: + surfacesChangedCallback:propertiesChangedCallback:)]) { + @try { + [(id)renderable registerScreenCallbacksWithUUID:token callbackQueue:state_->queue + frameCallback:frame + surfacesChangedCallback:^(id primary, id masked) { (void)masked; changed(primary); } + propertiesChangedCallback:^(id properties) { (void)properties; }]; + registered = true; + state_->screen_callbacks = true; + } @catch (NSException*) { + @try { [(id)renderable unregisterScreenCallbacksWithUUID:token]; } @catch (NSException*) {} + } + } + if (!registered) { + // Legacy callbacks arrive on CoreSimulator's queue; hop to ours so the + // handlers stay serialized and drainable. + dispatch_queue_t queue = state_->queue; + void (^changed_on_queue)(id) = ^(id surface) { dispatch_async(queue, ^{ changed(surface); }); }; + bool surface_ok = false; + @try { + [(id)renderable registerCallbackWithUUID:token + ioSurfacesChangeCallback:changed_on_queue]; + surface_ok = true; + } @catch (NSException*) {} + @try { + [(id)renderable registerCallbackWithUUID:token + ioSurfaceChangeCallback:changed_on_queue]; + surface_ok = true; + } @catch (NSException*) {} + @try { + [(id)renderable registerCallbackWithUUID:token + damageRectanglesCallback:^(NSArray* rects) { (void)rects; dispatch_async(queue, frame); }]; + } @catch (NSException*) { + if (!surface_ok) { + unregister_callbacks(renderable, token, false); + throw SimError("DISPLAY_UNAVAILABLE", "Could not subscribe to framebuffer updates."); + } + } + } + id surface = initial_surface(renderable); + std::lock_guard lock(state_->mu); + if (!state_->surface) state_->surface = surface; +} + +FramebufferSource::~FramebufferSource() { + { + std::lock_guard lock(state_->mu); + state_->closed = true; + state_->on_surface = nullptr; + state_->on_frame = nullptr; + } + unregister_callbacks(state_->renderable, state_->token, state_->screen_callbacks); + // Wait for a handler that is already running (unless it is this call). + if (dispatch_get_specific(kQueueKey) != kQueueKey) dispatch_sync(state_->queue, ^{}); +} + +id FramebufferSource::surface() { + { + std::lock_guard lock(state_->mu); + if (state_->surface) return state_->surface; + } + id surface = initial_surface(state_->renderable); + std::lock_guard lock(state_->mu); + if (!state_->surface) state_->surface = surface; + return state_->surface; +} + +} // namespace acecode::ios_simulator::helper diff --git a/src/adapters/tool/ios_simulator/helper/sim_hid.mm b/src/adapters/tool/ios_simulator/helper/sim_hid.mm index 52c3a914d..a53f35eb2 100644 --- a/src/adapters/tool/ios_simulator/helper/sim_hid.mm +++ b/src/adapters/tool/ios_simulator/helper/sim_hid.mm @@ -2,6 +2,8 @@ #import "sim_core.h" #import "private_api.h" +#include "tool/ios_simulator/dtuhid_messages.hpp" + #include #include #include @@ -15,8 +17,6 @@ namespace acecode::ios_simulator::helper { namespace { -constexpr const char* kDigitizerService = "com.apple.coredevice.feature.remote.hid.digitizer"; -constexpr const char* kFirstDTUHIDVersion = "1155.4"; bool version_at_least(const std::string& version, const char* minimum) { if (version.empty()) return false; @@ -184,7 +184,7 @@ void send(unsigned char* bytes) { std::string last_error; for (int attempt = 1; attempt <= 5; ++attempt) { bool unsupported = false; - id connection = connect_guest_service(device, kDigitizerService, last_error, &unsupported); + id connection = connect_guest_service(device, dtuhid::kDigitizerService, last_error, &unsupported); if (unsupported) throw SimError("HID_UNAVAILABLE", "This simulator runtime does not provide dtuhidd."); if (connection) { xpc_connection_t xpc = (xpc_connection_t)connection; @@ -207,40 +207,26 @@ void send(unsigned char* bytes) { std::string transport() const override { return "dtuhid"; } void touch(Direction direction, double x, double y, unsigned edge) override { - xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); - xpc_dictionary_set_value(payload, "pointOne", point(x, y)); - xpc_dictionary_set_uint64(payload, "eventType", contact_phase(contact_active_, direction)); - xpc_dictionary_set_uint64(payload, "edge", edge); - xpc_dictionary_set_uint64(payload, "target", 0); - send("IndigoDigitizerEvent", payload); + const auto phase = contact_.next(direction == Direction::Up); + send(dtuhid::kDigitizerEvent, dtuhid::digitizer_payload(unit(x, y), nullptr, phase, edge)); } void touch2(Direction direction, double x1, double y1, double x2, double y2) override { - xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); - xpc_dictionary_set_value(payload, "pointOne", point(x1, y1)); - xpc_dictionary_set_value(payload, "pointTwo", point(x2, y2)); - xpc_dictionary_set_uint64(payload, "eventType", contact_phase(two_finger_active_, direction)); - xpc_dictionary_set_uint64(payload, "edge", 0); - xpc_dictionary_set_uint64(payload, "target", 0); - send("IndigoDigitizerEvent", payload); + const auto phase = two_fingers_.next(direction == Direction::Up); + const auto second = unit(x2, y2); + send(dtuhid::kDigitizerEvent, dtuhid::digitizer_payload(unit(x1, y1), &second, phase, 0)); } void button(const HidButton& button, Direction direction) override { if (button.usage_page == 0) throw SimError("HID_UNSUPPORTED", "Apple Pay is a double side-button press on this transport; " "press side_button twice instead."); - xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); - xpc_dictionary_set_uint64(payload, "usagePage", button.usage_page); - xpc_dictionary_set_uint64(payload, "usageCode", button.usage); - xpc_dictionary_set_uint64(payload, "state", direction == Direction::Down ? 1 : 2); - send("IndigoButtonEvent", payload); + send(dtuhid::kButtonEvent, + dtuhid::button_payload(button.usage_page, button.usage, direction == Direction::Down)); } void key(std::uint32_t usage, Direction direction) override { - xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); - xpc_dictionary_set_uint64(payload, "usageCode", usage); - xpc_dictionary_set_uint64(payload, "state", direction == Direction::Down ? 1 : 2); - send("IndigoKeyboardButtonEvent", payload); + send(dtuhid::kKeyboardEvent, dtuhid::keyboard_payload(usage, direction == Direction::Down)); } void flush() override { @@ -251,23 +237,11 @@ void flush() override { } private: - static xpc_object_t envelope(const char* type, xpc_object_t payload, bool barrier) { - xpc_object_t message = xpc_dictionary_create(nullptr, nullptr, 0); - xpc_dictionary_set_string(message, "messageType", type); - xpc_dictionary_set_bool(message, "isBarrier", barrier); - xpc_dictionary_set_string(message, "featureIdentifier", kDigitizerService); - xpc_dictionary_set_value(message, "payload", payload); - return message; - } - static bool confirm_liveness(xpc_connection_t connection, std::string& error) { - xpc_object_t payload = xpc_dictionary_create(nullptr, nullptr, 0); - xpc_dictionary_set_uint64(payload, "usageCode", 0); - xpc_dictionary_set_uint64(payload, "state", 2); dispatch_semaphore_t answered = dispatch_semaphore_create(0); __block bool ok = false; __block std::string reply_error; - xpc_connection_send_message_with_reply(connection, envelope("IndigoKeyboardButtonEvent", payload, true), + xpc_connection_send_message_with_reply(connection, dtuhid::liveness_probe(), dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^(xpc_object_t reply) { if (xpc_get_type(reply) == XPC_TYPE_ERROR) { const char* description = xpc_dictionary_get_string(reply, XPC_ERROR_KEY_DESCRIPTION); @@ -285,22 +259,12 @@ static bool confirm_liveness(xpc_connection_t connection, std::string& error) { return ok; } - xpc_object_t point(double x, double y) const { - xpc_object_t value = xpc_dictionary_create(nullptr, nullptr, 0); - xpc_dictionary_set_double(value, "x", point_width_ > 0 ? x / point_width_ : 0); - xpc_dictionary_set_double(value, "y", point_height_ > 0 ? y / point_height_ : 0); - return value; - } - - static std::uint64_t contact_phase(bool& active, Direction direction) { - if (direction == Direction::Up) { active = false; return 2; } - if (active) return 1; - active = true; - return 0; + dtuhid::UnitPoint unit(double x, double y) const { + return dtuhid::unit_point(x, y, point_width_, point_height_); } void send(const char* type, xpc_object_t payload) { - xpc_connection_send_message(connection_, envelope(type, payload, false)); + xpc_connection_send_message(connection_, dtuhid::envelope(type, payload, false)); ++sent_; dispatch_semaphore_t done = dispatch_semaphore_create(0); xpc_connection_send_barrier(connection_, ^{ dispatch_semaphore_signal(done); }); @@ -310,8 +274,8 @@ void send(const char* type, xpc_object_t payload) { double point_width_; double point_height_; xpc_connection_t connection_ = nullptr; - bool contact_active_ = false; - bool two_finger_active_ = false; + dtuhid::ContactTracker contact_; + dtuhid::ContactTracker two_fingers_; std::uint64_t sent_ = 0; std::uint64_t drained_ = 0; }; @@ -332,7 +296,8 @@ bool lookup_button(const std::string& name, HidButton& out) { } std::string default_transport() { - return version_at_least(environment().core_simulator_version, kFirstDTUHIDVersion) ? "dtuhid" : "indigo"; + return version_at_least(environment().core_simulator_version, dtuhid::kFirstCoreSimulatorVersion) ? "dtuhid" + : "indigo"; } std::unique_ptr open_hid_session(id device, double point_width, double point_height, diff --git a/src/adapters/tool/ios_simulator/helper/sim_output.h b/src/adapters/tool/ios_simulator/helper/sim_output.h new file mode 100644 index 000000000..615829f3a --- /dev/null +++ b/src/adapters/tool/ios_simulator/helper/sim_output.h @@ -0,0 +1,16 @@ +// The helper's framed stdout writer (helper process only; the Desktop shell +// links the shared simulator code without it). +#pragma once + +#include + +#include + +namespace acecode::ios_simulator::helper { + +// Serialized and safe from any queue. Frames with drop_if_busy are skipped +// while more than a couple of droppable frames are pending. +void emit(const nlohmann::json& header, const std::string& binary = {}, bool drop_if_busy = false); +void emit_log(const std::string& level, const std::string& message); + +} // namespace acecode::ios_simulator::helper diff --git a/src/adapters/tool/ios_simulator/helper/sim_output.mm b/src/adapters/tool/ios_simulator/helper/sim_output.mm new file mode 100644 index 000000000..db2de1992 --- /dev/null +++ b/src/adapters/tool/ios_simulator/helper/sim_output.mm @@ -0,0 +1,45 @@ +#import "sim_output.h" + +#include "tool/ios_simulator/wire.hpp" + +#include +#include +#include +#include + +namespace acecode::ios_simulator::helper { +namespace { + +std::mutex g_write_mu; +std::atomic g_droppable_pending{0}; + +void write_all(const std::string& bytes) { + std::size_t sent = 0; + while (sent < bytes.size()) { + const auto count = ::write(STDOUT_FILENO, bytes.data() + sent, bytes.size() - sent); + if (count > 0) { sent += static_cast(count); continue; } + if (count < 0 && errno == EINTR) continue; + // The daemon is gone; there is nobody left to serve. + _exit(0); + } +} + +} // namespace + +void emit(const nlohmann::json& header, const std::string& binary, bool drop_if_busy) { + if (drop_if_busy && g_droppable_pending.load() > 2) return; + std::string frame; + try { frame = wire::encode(header, binary); } catch (const std::exception&) { return; } + if (drop_if_busy) g_droppable_pending.fetch_add(1); + { + std::lock_guard lock(g_write_mu); + write_all(frame); + } + if (drop_if_busy) g_droppable_pending.fetch_sub(1); +} + +void emit_log(const std::string& level, const std::string& message) { + emit({{"event", "log"}, {"level", level}, {"message", message}}); +} + +} // namespace acecode::ios_simulator::helper diff --git a/src/adapters/tool/ios_simulator/helper_client.cpp b/src/adapters/tool/ios_simulator/helper_client.cpp index 6620c7a6a..044e982e4 100644 --- a/src/adapters/tool/ios_simulator/helper_client.cpp +++ b/src/adapters/tool/ios_simulator/helper_client.cpp @@ -1,5 +1,6 @@ #include "helper_client.hpp" +#include "helper_sandbox.hpp" #include "utils/logger.hpp" #include @@ -33,7 +34,8 @@ std::string ios_simulator_helper_path() { HelperClient::~HelperClient() { stop(); } -bool HelperClient::start(const std::string& helper_path, const std::string& developer_dir, std::string* error) { +bool HelperClient::start(const std::string& helper_path, const std::string& developer_dir, bool sandboxed, + std::string* error) { stop(); platform::SpawnOptions options; options.argv = {helper_path, "--developer-dir", developer_dir}; @@ -41,8 +43,11 @@ bool HelperClient::start(const std::string& helper_path, const std::string& deve options.argv.push_back("--hid"); options.argv.push_back(hid); } + if (sandboxed) + options.argv = sandboxed_helper_argv(current_helper_sandbox_paths(helper_path, developer_dir), options.argv); if (!process_.start(options, error)) return false; developer_dir_ = developer_dir; + sandboxed_ = sandboxed; alive_.store(true); reader_ = JoiningThread(&HelperClient::read_loop, this); return true; diff --git a/src/adapters/tool/ios_simulator/helper_client.hpp b/src/adapters/tool/ios_simulator/helper_client.hpp index df4e32373..96d766d04 100644 --- a/src/adapters/tool/ios_simulator/helper_client.hpp +++ b/src/adapters/tool/ios_simulator/helper_client.hpp @@ -41,9 +41,12 @@ class HelperClient { HelperClient(const HelperClient&) = delete; HelperClient& operator=(const HelperClient&) = delete; - // Spawns the helper for the given Xcode developer directory. - bool start(const std::string& helper_path, const std::string& developer_dir, std::string* error); + // Spawns the helper for the given Xcode developer directory, inside the + // Seatbelt profile from helper_sandbox.hpp when `sandboxed`. + bool start(const std::string& helper_path, const std::string& developer_dir, bool sandboxed, + std::string* error); bool alive() const { return alive_.load(); } + bool sandboxed() const { return sandboxed_; } const std::string& developer_dir() const { return developer_dir_; } void set_listener(HelperEventListener listener); @@ -66,6 +69,7 @@ class HelperClient { void fail_all(const std::string& code, const std::string& message); std::string developer_dir_; + bool sandboxed_ = false; platform::PipedProcess process_; std::mutex write_mu_; std::mutex mu_; diff --git a/src/adapters/tool/ios_simulator/helper_sandbox.cpp b/src/adapters/tool/ios_simulator/helper_sandbox.cpp new file mode 100644 index 000000000..076def8c0 --- /dev/null +++ b/src/adapters/tool/ios_simulator/helper_sandbox.cpp @@ -0,0 +1,175 @@ +#include "helper_sandbox.hpp" + +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#endif + +namespace acecode::ios_simulator { +namespace { + +constexpr const char* kSandboxExec = "/usr/bin/sandbox-exec"; + +struct Param { + std::string name; + std::string value; +}; + +// Parameters in a fixed order; empty values are dropped together with the +// rules that would use them. +std::vector sandbox_params(const HelperSandboxPaths& paths) { + std::vector params{{"HELPER_DIR", paths.helper_dir}, {"XCODE_APP", paths.xcode_app}, + {"TEMP_DIR", paths.temp_dir}, {"CACHE_DIR", paths.cache_dir}, + {"HOME_DIR", paths.home}}; + if (!paths.home.empty()) { + const auto& home = paths.home; + params.push_back({"SIM_DATA", home + "/Library/Developer/CoreSimulator"}); + params.push_back({"SIM_LOGS", home + "/Library/Logs/CoreSimulator"}); + params.push_back({"CORE_DEVICE", home + "/Library/Developer/CoreDevice"}); + params.push_back({"PREF_SIMULATOR", home + "/Library/Preferences/com.apple.iphonesimulator.plist"}); + params.push_back({"PREF_CORESIMULATOR", home + "/Library/Preferences/com.apple.CoreSimulator.plist"}); + params.push_back({"PREF_GLOBAL", home + "/Library/Preferences/.GlobalPreferences.plist"}); + } + for (std::size_t index = 0; index < paths.listable_dirs.size(); ++index) + params.push_back({"LISTABLE_" + std::to_string(index), paths.listable_dirs[index]}); + std::vector present; + for (auto& param : params) + if (!param.value.empty()) present.push_back(std::move(param)); + return present; +} + +bool has_param(const std::vector& params, const std::string& name) { + for (const auto& param : params) + if (param.name == name) return true; + return false; +} + +std::string filters(const std::vector& params, const std::vector>& wanted) { + std::string out; + for (const auto& [name, subtree] : wanted) { + if (!has_param(params, name)) continue; + out += subtree ? " (subpath (param \"" : " (literal (param \""; + out += name + "\"))"; + } + return out; +} + +std::string listable_filters(const std::vector& params) { + std::string out; + for (const auto& param : params) + if (param.name.rfind("LISTABLE_", 0) == 0) out += " (literal (param \"" + param.name + "\"))"; + return out; +} + +std::string real_path(const std::string& path) { + if (path.empty()) return {}; + std::error_code error; + auto resolved = std::filesystem::weakly_canonical(path, error); + if (error || resolved.empty()) return path; + auto text = resolved.string(); + while (text.size() > 1 && text.back() == '/') text.pop_back(); + return text; +} + +#ifdef __APPLE__ +std::string darwin_dir(int name) { + char buffer[1024]; + const auto length = confstr(name, buffer, sizeof(buffer)); + if (length == 0 || length > sizeof(buffer)) return {}; + return real_path(buffer); +} +#endif + +} // namespace + +std::string helper_sandbox_profile(const HelperSandboxPaths& paths) { + const auto params = sandbox_params(paths); + std::string profile = + "(version 1)\n" + "(allow default)\n" + "(deny network-outbound (remote ip \"*:*\"))\n" + "(deny network-inbound (local ip \"*:*\"))\n" + "(deny network-bind (local ip \"*:*\"))\n"; + profile += "(deny file-read-data (subpath \"/Users\") (subpath \"/Volumes\")" + + filters(params, {{"HOME_DIR", true}}) + ")\n"; + const auto readable = filters(params, {{"HELPER_DIR", true}, {"XCODE_APP", true}, {"TEMP_DIR", true}, + {"CACHE_DIR", true}, {"SIM_DATA", true}, {"SIM_LOGS", true}, + {"CORE_DEVICE", true}, {"PREF_SIMULATOR", false}, + {"PREF_CORESIMULATOR", false}, {"PREF_GLOBAL", false}}) + + listable_filters(params); + if (!readable.empty()) profile += "(allow file-read-data" + readable + ")\n"; + profile += "(deny file-write* (subpath \"/\"))\n"; + profile += "(allow file-write* (subpath \"/dev\")" + + filters(params, {{"TEMP_DIR", true}, {"CACHE_DIR", true}, {"SIM_DATA", true}, + {"SIM_LOGS", true}, {"CORE_DEVICE", true}}) + + ")\n"; + return profile; +} + +std::vector sandboxed_helper_argv(const HelperSandboxPaths& paths, + const std::vector& helper_argv) { + std::vector argv{kSandboxExec, "-p", helper_sandbox_profile(paths)}; + for (const auto& param : sandbox_params(paths)) argv.push_back("-D" + param.name + "=" + param.value); + argv.insert(argv.end(), helper_argv.begin(), helper_argv.end()); + return argv; +} + +std::string xcode_app_for_developer_dir(const std::string& developer_dir) { + std::string path = developer_dir; + while (path.size() > 1 && path.back() == '/') path.pop_back(); + const std::string suffix = "/Contents/Developer"; + if (path.size() > suffix.size() && path.compare(path.size() - suffix.size(), suffix.size(), suffix) == 0) + return path.substr(0, path.size() - suffix.size()); + return path; +} + +HelperSandboxPaths current_helper_sandbox_paths(const std::string& helper_path, + const std::string& developer_dir) { + HelperSandboxPaths paths; + const std::filesystem::path executable_dir = real_path(std::filesystem::path(helper_path).parent_path().string()); + paths.helper_dir = executable_dir.string(); + if (executable_dir.filename() == "MacOS" && executable_dir.parent_path().filename() == "Contents" && + executable_dir.parent_path().parent_path().extension() == ".app") + paths.helper_dir = executable_dir.parent_path().parent_path().string(); + auto ancestor = executable_dir; + for (int level = 0; level < 2 && ancestor.has_parent_path() && ancestor.parent_path() != ancestor; ++level) { + ancestor = ancestor.parent_path(); + paths.listable_dirs.push_back(ancestor.string()); + } + paths.xcode_app = real_path(xcode_app_for_developer_dir(developer_dir)); +#ifdef __APPLE__ + // CoreSimulator finds its device set from the user record, not $HOME. + if (const passwd* user = getpwuid(getuid()); user && user->pw_dir && *user->pw_dir) + paths.home = real_path(user->pw_dir); +#endif + if (paths.home.empty()) + if (const char* home = std::getenv("HOME"); home && *home) paths.home = real_path(home); +#ifdef __APPLE__ + paths.temp_dir = darwin_dir(_CS_DARWIN_USER_TEMP_DIR); + paths.cache_dir = darwin_dir(_CS_DARWIN_USER_CACHE_DIR); +#endif + return paths; +} + +bool helper_sandbox_enabled(std::string* reason) { + if (const char* value = std::getenv("ACECODE_IOS_SIM_SANDBOX"); value && std::string(value) == "0") { + if (reason) *reason = "disabled by ACECODE_IOS_SIM_SANDBOX=0"; + return false; + } +#ifdef __APPLE__ + if (access(kSandboxExec, X_OK) != 0) { + if (reason) *reason = "/usr/bin/sandbox-exec is not available"; + return false; + } + return true; +#else + if (reason) *reason = "Seatbelt is only available on macOS"; + return false; +#endif +} + +} // namespace acecode::ios_simulator diff --git a/src/adapters/tool/ios_simulator/helper_sandbox.hpp b/src/adapters/tool/ios_simulator/helper_sandbox.hpp new file mode 100644 index 000000000..38fcc08f2 --- /dev/null +++ b/src/adapters/tool/ios_simulator/helper_sandbox.hpp @@ -0,0 +1,55 @@ +#pragma once + +// Seatbelt confinement for acecode-ios-sim. The helper reaches CoreSimulator, +// SimulatorKit and the simulator's accessibility over XPC / Mach and returns +// pixels over stdout; it never needs IP networking or the user's documents. +// The profile keeps system services reachable (allow default) and removes +// what a compromised helper could abuse: +// - IP networking (inbound, outbound, bind); Unix sockets stay available; +// - reading file contents under /Users, /Volumes and the home directory, +// except the helper's bundle, the selected Xcode, simulator data/logs, +// the simulator preference files and the per-user temp / cache folders; +// - writing anywhere except /dev, simulator data/logs and temp / cache. +// The profile text is ACECode's own and is passed to /usr/bin/sandbox-exec +// with every path as a -D parameter, so no path is ever spliced into SBPL. + +#include +#include + +namespace acecode::ios_simulator { + +struct HelperSandboxPaths { + // The helper's .app bundle (…/ACECode.app) or, outside a bundle, the + // directory holding the executable. + std::string helper_dir; + // Directories CFBundle lists while locating the main bundle (the + // executable's ancestors up to the would-be X.app of X.app/Contents/MacOS). + // Listing only: their other contents stay unreadable. Without them + // CoreSimulator's NSUserDefaults setup crashes on a NULL main bundle. + std::vector listable_dirs; + std::string xcode_app; // .../Xcode.app (or the developer dir itself) + std::string home; // the user's home directory (user record, as CoreSimulator uses) + std::string temp_dir; // DARWIN_USER_TEMP_DIR + std::string cache_dir; // DARWIN_USER_CACHE_DIR +}; + +// Profile for the given paths; rules whose path is empty are omitted. +std::string helper_sandbox_profile(const HelperSandboxPaths& paths); + +// `/usr/bin/sandbox-exec -p -D… `. +std::vector sandboxed_helper_argv(const HelperSandboxPaths& paths, + const std::vector& helper_argv); + +// .../Xcode.app/Contents/Developer -> .../Xcode.app; other shapes unchanged. +std::string xcode_app_for_developer_dir(const std::string& developer_dir); + +// Real (symlink-resolved) paths for this process: Seatbelt matches resolved +// vnode paths, so /var/folders must become /private/var/folders. +HelperSandboxPaths current_helper_sandbox_paths(const std::string& helper_path, + const std::string& developer_dir); + +// False when ACECODE_IOS_SIM_SANDBOX=0 or sandbox-exec is unavailable; the +// reason is filled in that case. +bool helper_sandbox_enabled(std::string* reason = nullptr); + +} // namespace acecode::ios_simulator diff --git a/src/adapters/tool/ios_simulator/runtime.cpp b/src/adapters/tool/ios_simulator/runtime.cpp index 1d703ed9d..c6ce2a1f1 100644 --- a/src/adapters/tool/ios_simulator/runtime.cpp +++ b/src/adapters/tool/ios_simulator/runtime.cpp @@ -1,6 +1,7 @@ #include "runtime.hpp" #include "tool/ios_simulator/device_consent.hpp" +#include "tool/ios_simulator/helper_sandbox.hpp" #include "utils/logger.hpp" #include @@ -118,7 +119,9 @@ nlohmann::json Runtime::status(bool refresh) { {"stream_scale", config_.stream_scale}, {"stream_quality", config_.stream_quality}}; out["helper"] = {{"installed", helper_installed(helper_path)}, {"running", helper_ && helper_->alive()}, - {"developer_dir", helper_ ? helper_->developer_dir() : std::string()}}; + {"developer_dir", helper_ ? helper_->developer_dir() : std::string()}, + {"sandboxed", helper_ && helper_->sandboxed()}, + {"sandbox_note", sandbox_note_}}; nlohmann::json bindings = nlohmann::json::object(); for (const auto& [session, udid] : session_devices_) bindings[session] = udid; out["sessions"] = bindings; @@ -153,23 +156,46 @@ std::shared_ptr Runtime::ensure_helper(std::string* error) { if (error) *error = "acecode-ios-sim is missing beside the ACECode executable; reinstall ACECode for macOS."; return nullptr; } - auto helper = std::make_shared(); - helper->set_listener([ref = lifetime_.ref(*this)](const nlohmann::json& header, const std::string& binary) { - ref.with([&header, &binary](Runtime& self) { self.on_helper_event(header, binary); }); - }); - std::string start_error; - if (!helper->start(path, install->developer_dir, &start_error)) { - if (error) *error = "Could not start the iOS Simulator helper: " + start_error; - return nullptr; + std::string sandbox_note; + const bool sandboxed = helper_sandbox_enabled(&sandbox_note); + auto helper = launch_helper(path, install->developer_dir, sandboxed, error); + if (helper && sandboxed) { + // sandbox-exec exits at once when the profile cannot be applied, e.g. + // when ACECode itself already runs inside a sandbox. The helper still + // works unconfined; the status reports it and the log says why. + const auto hello = helper->request({{"op", "hello"}}, 15000); + if (!hello.ok && hello.error_code == "HELPER_EXITED") { + LOG_WARN("[ios-sim] sandbox-exec refused the helper profile; running the helper unsandboxed"); + helper->stop(); + sandbox_note = "sandbox-exec could not apply the helper profile"; + helper = launch_helper(path, install->developer_dir, false, error); + } } + if (!helper) return nullptr; std::shared_ptr retired; { std::lock_guard lock(mu_); retired = std::move(helper_); helper_ = helper; + sandbox_note_ = helper->sandboxed() ? std::string() : sandbox_note; } if (retired) retired->stop(); - LOG_INFO("[ios-sim] helper started for " + install->developer_dir); + LOG_INFO("[ios-sim] helper started for " + install->developer_dir + + (helper->sandboxed() ? " (sandboxed)" : " (unsandboxed: " + sandbox_note + ")")); + return helper; +} + +std::shared_ptr Runtime::launch_helper(const std::string& path, const std::string& developer_dir, + bool sandboxed, std::string* error) { + auto helper = std::make_shared(); + helper->set_listener([ref = lifetime_.ref(*this)](const nlohmann::json& header, const std::string& binary) { + ref.with([&header, &binary](Runtime& self) { self.on_helper_event(header, binary); }); + }); + std::string start_error; + if (!helper->start(path, developer_dir, sandboxed, &start_error)) { + if (error) *error = "Could not start the iOS Simulator helper: " + start_error; + return nullptr; + } return helper; } diff --git a/src/adapters/tool/ios_simulator/runtime.hpp b/src/adapters/tool/ios_simulator/runtime.hpp index dc47b5b43..8f50bec9f 100644 --- a/src/adapters/tool/ios_simulator/runtime.hpp +++ b/src/adapters/tool/ios_simulator/runtime.hpp @@ -110,6 +110,8 @@ class Runtime { }; std::shared_ptr ensure_helper(std::string* error); + std::shared_ptr launch_helper(const std::string& path, const std::string& developer_dir, + bool sandboxed, std::string* error); void on_helper_event(const nlohmann::json& header, const std::string& binary); void end_streams(const std::string& reason, const std::string& only_udid); @@ -117,6 +119,7 @@ class Runtime { RuntimeConfig config_; std::optional resolution_; std::shared_ptr helper_; + std::string sandbox_note_; // why the running helper is not sandboxed std::map session_devices_; std::map streams_; // by udid std::map subscriptions_; // subscription -> udid diff --git a/src/apps/desktop/ios_simulator_view.cpp b/src/apps/desktop/ios_simulator_view.cpp new file mode 100644 index 000000000..29e511898 --- /dev/null +++ b/src/apps/desktop/ios_simulator_view.cpp @@ -0,0 +1,30 @@ +// Platforms without CoreSimulator: the web panel keeps the JPEG stream. +#include "ios_simulator_view.hpp" + +namespace acecode::desktop { + +struct IosSimulatorView::Impl { + StateHandler on_state; +}; + +IosSimulatorView::IosSimulatorView(void* parent_window, StateHandler state_handler) + : impl_(std::make_shared()) { + (void)parent_window; + impl_->on_state = std::move(state_handler); +} + +IosSimulatorView::~IosSimulatorView() = default; + +bool IosSimulatorView::supported() const { return false; } + +bool IosSimulatorView::set_layout(const IosSimulatorViewLayout& layout, std::string* error) { + (void)layout; + if (error) *error = "the native iOS Simulator view is only available on macOS"; + return false; +} + +void IosSimulatorView::unmount() {} + +nlohmann::json IosSimulatorView::state() const { return {{"supported", false}}; } + +} // namespace acecode::desktop diff --git a/src/apps/desktop/ios_simulator_view.hpp b/src/apps/desktop/ios_simulator_view.hpp new file mode 100644 index 000000000..d238a2b68 --- /dev/null +++ b/src/apps/desktop/ios_simulator_view.hpp @@ -0,0 +1,58 @@ +#pragma once + +// Native iOS Simulator screen for the Desktop shell. On macOS the booted +// simulator's IOSurface is set as the contents of a layer placed over the +// web panel's screen area, so frames reach the window server without being +// copied, scaled or JPEG-encoded. The view never takes input: hit testing +// passes through to the web panel, which keeps sending touches and keys to +// the daemon. Other platforms report unsupported and the web panel keeps the +// JPEG stream. +// +// Only CoreSimulator is loaded (fixed system path); the developer directory +// from the web page is passed to CoreSimulator as a string and never used to +// load code. + +#include "ios_simulator_view_layout.hpp" + +#include + +#include +#include +#include + +namespace acecode::desktop { + +class WebHost; + +class IosSimulatorView { +public: + // Receives {udid, attached, rendering, rotation, pixel_width, + // pixel_height, layout_revision, error} on the UI thread. + using StateHandler = std::function; + + IosSimulatorView(void* parent_window, StateHandler state_handler); + ~IosSimulatorView(); + IosSimulatorView(const IosSimulatorView&) = delete; + IosSimulatorView& operator=(const IosSimulatorView&) = delete; + + bool supported() const; + // Attaches to layout.udid (asynchronously the first time) and places the + // screen. Call on the UI thread. + bool set_layout(const IosSimulatorViewLayout& layout, std::string* error); + void unmount(); + nlohmann::json state() const; + +private: + struct Impl; + std::shared_ptr impl_; +}; + +// Creates the view over the host window and binds +// aceDesktop_iosSimulatorViewSetLayout([layout]) / ...Unmount() / +// ...GetState(), each answering {ok, ...state} or {ok:false, error}. State +// changes reach the page as `acecode:ios-simulator-view-state` events. The +// view is a subview of the window's content view, so it hides with the window; +// the page re-sends the layout whenever the panel moves or resizes. +std::unique_ptr install_ios_simulator_view(WebHost& host); + +} // namespace acecode::desktop diff --git a/src/apps/desktop/ios_simulator_view_bridge.cpp b/src/apps/desktop/ios_simulator_view_bridge.cpp new file mode 100644 index 000000000..5d8492fb7 --- /dev/null +++ b/src/apps/desktop/ios_simulator_view_bridge.cpp @@ -0,0 +1,54 @@ +// aceDesktop_iosSimulatorView* bridges between the web panel and the native +// iOS Simulator view. +#include "ios_simulator_view.hpp" + +#include "utils/logger.hpp" +#include "web_host.hpp" + +namespace acecode::desktop { +namespace { + +std::string failure(const std::string& message) { return nlohmann::json{{"ok", false}, {"error", message}}.dump(); } + +std::string with_ok(nlohmann::json state) { + state["ok"] = true; + return state.dump(); +} + +IosSimulatorView::StateHandler state_dispatcher(WebHost& host) { + auto last_error = std::make_shared(); + return [&host, last_error](const nlohmann::json& state) { + const auto error = state.value("error", std::string()); + if (error != *last_error) { + *last_error = error; + if (!error.empty()) LOG_WARN("[ios-sim] native view: " + error); + } + host.eval("(function(){try{window.dispatchEvent(new CustomEvent('acecode:ios-simulator-view-state'," + "{detail:" + state.dump(-1, ' ', true) + "}));}catch(e){}})();"); + }; +} + +} // namespace + +std::unique_ptr install_ios_simulator_view(WebHost& host) { + auto owned = std::make_unique(host.native_window(), state_dispatcher(host)); + IosSimulatorView& view = *owned; + host.bind("aceDesktop_iosSimulatorViewGetState", + [&view](const std::string& /*req*/) -> std::string { return with_ok(view.state()); }); + host.bind("aceDesktop_iosSimulatorViewSetLayout", [&view](const std::string& req) -> std::string { + const auto args = nlohmann::json::parse(req, nullptr, false); + if (!args.is_array() || args.empty()) return failure("expect [layout:object]"); + std::string error; + const auto layout = parse_ios_simulator_view_layout(args[0], &error); + if (!layout) return failure(error); + if (!view.set_layout(*layout, &error)) return failure(error); + return with_ok(view.state()); + }); + host.bind("aceDesktop_iosSimulatorViewUnmount", [&view](const std::string& /*req*/) -> std::string { + view.unmount(); + return with_ok(view.state()); + }); + return owned; +} + +} // namespace acecode::desktop diff --git a/src/apps/desktop/ios_simulator_view_layout.cpp b/src/apps/desktop/ios_simulator_view_layout.cpp new file mode 100644 index 000000000..9b2e84925 --- /dev/null +++ b/src/apps/desktop/ios_simulator_view_layout.cpp @@ -0,0 +1,130 @@ +#include "ios_simulator_view_layout.hpp" + +#include +#include +#include + +namespace acecode::desktop { +namespace { + +bool read_int(const nlohmann::json& object, const char* key, int& out) { + const auto found = object.find(key); + if (found == object.end()) return true; + if (!found->is_number()) return false; + const double value = found->get(); + if (value < -1e6 || value > 1e6) return false; + out = static_cast(value); + return true; +} + +bool fail(std::string* error, const std::string& message) { + if (error) *error = message; + return false; +} + +} // namespace + +bool is_simulator_udid(const std::string& value) { + if (value.size() != 36) return false; + for (std::size_t index = 0; index < value.size(); ++index) { + const bool dash = index == 8 || index == 13 || index == 18 || index == 23; + if (dash ? value[index] != '-' : !std::isxdigit(static_cast(value[index]))) return false; + } + return true; +} + +std::optional parse_ios_simulator_view_layout(const nlohmann::json& value, + std::string* error) { + if (!value.is_object()) { + fail(error, "layout must be an object"); + return std::nullopt; + } + IosSimulatorViewLayout layout; + layout.udid = value.value("udid", std::string()); + layout.developer_dir = value.value("developer_dir", std::string()); + if (!is_simulator_udid(layout.udid)) { + fail(error, "layout udid must be a simulator UDID"); + return std::nullopt; + } + if (layout.developer_dir.empty() || layout.developer_dir.front() != '/') { + fail(error, "layout developer_dir must be an absolute path"); + return std::nullopt; + } + if (!read_int(value, "x", layout.frame.x) || !read_int(value, "y", layout.frame.y) || + !read_int(value, "width", layout.frame.width) || !read_int(value, "height", layout.frame.height)) { + fail(error, "layout frame values must be numbers"); + return std::nullopt; + } + layout.frame.width = std::max(0, layout.frame.width); + layout.frame.height = std::max(0, layout.frame.height); + if (const auto visible = value.find("visible"); visible != value.end() && visible->is_boolean()) + layout.visible = visible->get(); + if (const auto radius = value.find("corner_radius"); radius != value.end()) { + if (!radius->is_number()) { + fail(error, "corner_radius must be a number"); + return std::nullopt; + } + layout.corner_radius = std::clamp(radius->get(), 0.0, 400.0); + } + if (const auto revision = value.find("layout_revision"); revision != value.end()) { + if (!revision->is_number_integer() || revision->get() < 0) { + fail(error, "layout_revision must be a non-negative integer"); + return std::nullopt; + } + layout.layout_revision = revision->get(); + } + if (const auto rects = value.find("occlusion_rects"); rects != value.end()) { + if (!rects->is_array() || rects->size() > kIosSimulatorViewMaxOcclusionRects) { + fail(error, "occlusion_rects must be a bounded array"); + return std::nullopt; + } + for (const auto& item : *rects) { + IosSimulatorViewRect rect; + if (!item.is_object() || !read_int(item, "x", rect.x) || !read_int(item, "y", rect.y) || + !read_int(item, "width", rect.width) || !read_int(item, "height", rect.height)) { + fail(error, "occlusion rectangle must be an object of numbers"); + return std::nullopt; + } + layout.occlusion_rects.push_back(rect); + } + } + return layout; +} + +std::vector clipped_occlusion_rects(const IosSimulatorViewLayout& layout) { + std::vector out; + const int width = layout.frame.width; + const int height = layout.frame.height; + for (const auto& rect : layout.occlusion_rects) { + const int left = std::clamp(rect.x, 0, width); + const int top = std::clamp(rect.y, 0, height); + const int right = std::clamp(rect.x + std::max(0, rect.width), left, width); + const int bottom = std::clamp(rect.y + std::max(0, rect.height), top, height); + if (right > left && bottom > top) out.push_back({left, top, right - left, bottom - top}); + } + return out; +} + +IosSimulatorLayerGeometry fit_simulator_layer(double frame_width, double frame_height, double surface_width, + double surface_height, int rotation) { + IosSimulatorLayerGeometry geometry; + rotation = ((rotation % 360) + 360) % 360; + geometry.rotation = rotation == 90 || rotation == 180 || rotation == 270 ? rotation : 0; + geometry.center_x = frame_width / 2; + geometry.center_y = frame_height / 2; + if (frame_width <= 0 || frame_height <= 0 || surface_width <= 0 || surface_height <= 0) return geometry; + const bool quarter = geometry.rotation == 90 || geometry.rotation == 270; + const double oriented_width = quarter ? surface_height : surface_width; + const double oriented_height = quarter ? surface_width : surface_height; + const double scale = std::min(frame_width / oriented_width, frame_height / oriented_height); + geometry.width = surface_width * scale; + geometry.height = surface_height * scale; + return geometry; +} + +double simulator_layer_angle_radians(int rotation, bool flipped_geometry) { + const double radians = static_cast(((rotation % 360) + 360) % 360) * std::acos(-1.0) / 180.0; + return flipped_geometry ? radians : -radians; +} + +} // namespace acecode::desktop diff --git a/src/apps/desktop/ios_simulator_view_layout.hpp b/src/apps/desktop/ios_simulator_view_layout.hpp new file mode 100644 index 000000000..aab723e5f --- /dev/null +++ b/src/apps/desktop/ios_simulator_view_layout.hpp @@ -0,0 +1,68 @@ +#pragma once + +// Pure parts of the Desktop shell's native iOS Simulator view: the bridge +// layout contract and the layer geometry. The macOS host +// (ios_simulator_view_mac.mm) shows the simulator's IOSurface directly in a +// layer above the main web view; the web panel keeps handling input. + +#include + +#include +#include +#include +#include +#include + +namespace acecode::desktop { + +inline constexpr std::size_t kIosSimulatorViewMaxOcclusionRects = 64; + +struct IosSimulatorViewRect { + int x = 0; + int y = 0; + int width = 0; + int height = 0; +}; + +struct IosSimulatorViewLayout { + std::string udid; + std::string developer_dir; // selected Xcode's Contents/Developer + // Screen area in main web view CSS pixels, top-left origin. + IosSimulatorViewRect frame; + bool visible = false; + double corner_radius = 0; // the device's rounded screen corners, CSS px + std::uint64_t layout_revision = 0; + // Web content drawn above the screen (menus, dialogs), relative to frame. + std::vector occlusion_rects; +}; + +// 8-4-4-4-12 hexadecimal, the CoreSimulator device UDID shape. +bool is_simulator_udid(const std::string& value); + +// Parses the aceDesktop_iosSimulatorViewSetLayout argument object. +std::optional parse_ios_simulator_view_layout(const nlohmann::json& value, + std::string* error); + +// Occlusion rectangles clipped to the frame; empty ones dropped. +std::vector clipped_occlusion_rects(const IosSimulatorViewLayout& layout); + +// Placement of the framebuffer layer inside the frame. The framebuffer is in +// the device's native (portrait) orientation; the layer keeps those bounds +// and is turned clockwise by `rotation` around its center, aspect-fitted. +struct IosSimulatorLayerGeometry { + double center_x = 0; + double center_y = 0; + double width = 0; // unrotated bounds + double height = 0; + int rotation = 0; // 0, 90, 180 or 270, clockwise +}; + +IosSimulatorLayerGeometry fit_simulator_layer(double frame_width, double frame_height, double surface_width, + double surface_height, int rotation); + +// Core Animation z-rotation for a clockwise turn: a positive angle turns +// clockwise on screen only in a flipped (top-left origin) geometry, which a +// flipped AppKit view's backing layer uses. +double simulator_layer_angle_radians(int rotation, bool flipped_geometry); + +} // namespace acecode::desktop diff --git a/src/apps/desktop/ios_simulator_view_mac.mm b/src/apps/desktop/ios_simulator_view_mac.mm new file mode 100644 index 000000000..b48b76f7c --- /dev/null +++ b/src/apps/desktop/ios_simulator_view_mac.mm @@ -0,0 +1,339 @@ +#include "ios_simulator_view.hpp" + +#import "tool/ios_simulator/helper/sim_core.h" +#import "tool/ios_simulator/helper/sim_framebuffer.h" + +#import +#import +#import + +#include +#include +#include +#include +#include +#include + +namespace sim = acecode::ios_simulator::helper; +using acecode::desktop::IosSimulatorViewRect; + +namespace { + +constexpr const char* kCoreSimulatorFramework = "/Library/Developer/PrivateFrameworks/CoreSimulator.framework"; + +} // namespace + +// The screen surface. It never takes input: the web panel below keeps +// receiving pointer and keyboard events and forwards them to the daemon. +@interface ACECodeIosSimulatorScreenView : NSView +@property(nonatomic, readonly) CALayer* screenLayer; +- (void)applyOcclusionRects:(const std::vector&)rects; +@end + +@implementation ACECodeIosSimulatorScreenView + +- (instancetype)initWithFrame:(NSRect)frame { + self = [super initWithFrame:frame]; + if (self) { + self.wantsLayer = YES; + self.layer.masksToBounds = YES; + _screenLayer = [CALayer layer]; + _screenLayer.magnificationFilter = kCAFilterLinear; + _screenLayer.minificationFilter = kCAFilterTrilinear; + _screenLayer.actions = @{@"contents": [NSNull null], @"bounds": [NSNull null], + @"position": [NSNull null], @"transform": [NSNull null]}; + [self.layer addSublayer:_screenLayer]; + } + return self; +} + +- (BOOL)isFlipped { + return YES; +} + +- (NSView*)hitTest:(NSPoint)point { + (void)point; + return nil; +} + +- (BOOL)acceptsFirstResponder { + return NO; +} + +// Web content drawn over the screen (menus, dialogs) stays visible: those +// rectangles are cut out of the layer. Rectangles use the DOM's top-left +// coordinates, which a flipped view's backing layer already shares. +- (void)applyOcclusionRects:(const std::vector&)rects { + CALayer* layer = self.layer; + if (!layer) return; + if (rects.empty()) { + layer.mask = nil; + return; + } + const NSRect bounds = self.bounds; + CGMutablePathRef path = CGPathCreateMutable(); + CGPathAddRect(path, nullptr, CGRectMake(0, 0, NSWidth(bounds), NSHeight(bounds))); + for (const auto& rect : rects) { + CGPathAddRect(path, nullptr, CGRectMake(rect.x, rect.y, rect.width, rect.height)); + } + CAShapeLayer* mask = [CAShapeLayer layer]; + mask.frame = CGRectMake(0, 0, NSWidth(bounds), NSHeight(bounds)); + mask.path = path; + mask.fillRule = kCAFillRuleEvenOdd; + mask.fillColor = CGColorGetConstantColor(kCGColorBlack); + CGPathRelease(path); + layer.mask = mask; +} + +@end + +namespace acecode::desktop { + +struct IosSimulatorView::Impl : std::enable_shared_from_this { + __weak NSWindow* window = nil; + ACECodeIosSimulatorScreenView* view = nil; + StateHandler on_state; + dispatch_queue_t attach_queue = nil; + + // Main thread only. + IosSimulatorViewLayout layout; + bool has_layout = false; + std::string attached_udid; + std::string attaching_udid; + std::uint64_t generation = 0; + std::shared_ptr source; + id surface = nil; + int rotation = 0; + std::string error; + nlohmann::json last_published; + + // Frame coalescing between the framebuffer queue and the main thread. + std::atomic frame_pending{false}; + std::atomic pending_rotation{0}; + + bool rendering() const { return view && !view.hidden; } + + nlohmann::json snapshot() const { + IOSurfaceRef ref = (__bridge IOSurfaceRef)surface; + return {{"supported", true}, + {"udid", has_layout ? layout.udid : std::string()}, + {"attached", source != nullptr && has_layout && attached_udid == layout.udid}, + {"attaching", !attaching_udid.empty()}, + {"rendering", rendering()}, + {"rotation", rotation}, + {"pixel_width", ref ? IOSurfaceGetWidth(ref) : 0}, + {"pixel_height", ref ? IOSurfaceGetHeight(ref) : 0}, + {"layout_revision", layout.layout_revision}, + {"error", error}}; + } + + void publish() { + auto state = snapshot(); + if (state == last_published) return; + last_published = state; + if (on_state) on_state(state); + } + + void place() { + NSWindow* parent = window; + NSView* content = parent.contentView; + if (!view || !content) return; + const auto& frame = layout.frame; + const bool show = has_layout && layout.visible && surface != nil && frame.width > 0 && + frame.height > 0 && attached_udid == layout.udid; + if (!show) { + view.hidden = YES; + return; + } + const CGFloat y = content.isFlipped ? frame.y : NSHeight(content.bounds) - frame.y - frame.height; + [CATransaction begin]; + [CATransaction setDisableActions:YES]; + view.frame = NSMakeRect(frame.x, y, frame.width, frame.height); + view.layer.cornerRadius = layout.corner_radius; + IOSurfaceRef ref = (__bridge IOSurfaceRef)surface; + const auto geometry = fit_simulator_layer(frame.width, frame.height, IOSurfaceGetWidth(ref), + IOSurfaceGetHeight(ref), rotation); + CALayer* screen = view.screenLayer; + screen.bounds = CGRectMake(0, 0, geometry.width, geometry.height); + screen.position = CGPointMake(geometry.center_x, geometry.center_y); + screen.transform = CATransform3DMakeRotation( + simulator_layer_angle_radians(geometry.rotation, view.isFlipped), 0, 0, 1); + [view applyOcclusionRects:clipped_occlusion_rects(layout)]; + [CATransaction commit]; + if (view.superview != content || content.subviews.lastObject != view) + [content addSubview:view positioned:NSWindowAbove relativeTo:nil]; + view.hidden = NO; + } + + void release_device() { + ++generation; + attaching_udid.clear(); + attached_udid.clear(); + surface = nil; + view.screenLayer.contents = nil; + if (source) { + // Unregistering talks to CoreSimulator and waits for a running + // callback; keep that off the main thread. + __block std::shared_ptr retired = std::move(source); + dispatch_async(attach_queue, ^{ retired.reset(); }); + } + } + + void attach(const std::string& requested_udid, const std::string& requested_developer_dir) { + release_device(); + error.clear(); + attaching_udid = requested_udid; + const std::uint64_t attempt = generation; + std::weak_ptr weak = weak_from_this(); + // Blocks capture C++ reference parameters by reference; copy them. + const std::string udid = requested_udid; + const std::string developer_dir = requested_developer_dir; + dispatch_async(attach_queue, ^{ + std::string failure; + std::shared_ptr opened; + int initial_rotation = 0; + try { + // CoreSimulator only: SimulatorKit (input) stays in the helper. + sim::load_frameworks(developer_dir, false); + id device = sim::device_for_udid(udid); + std::weak_ptr surface_owner = weak; + std::weak_ptr frame_owner = weak; + opened = std::make_shared( + device, + [surface_owner, attempt](id changed) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (auto self = surface_owner.lock()) self->surface_changed(attempt, changed); + }); + }, + [frame_owner, attempt, device, udid]() { + // The rotation lookup is cached; keep it off the main thread. + const int turned = sim::display_rotation(device, udid); + auto self = frame_owner.lock(); + if (!self) return; + self->pending_rotation.store(turned); + if (self->frame_pending.exchange(true)) return; + dispatch_async(dispatch_get_main_queue(), ^{ + self->frame_pending.store(false); + self->frame_arrived(attempt, self->pending_rotation.load()); + }); + }); + initial_rotation = sim::display_rotation(device, udid); + } catch (const sim::SimError& failed) { + failure = failed.code + ": " + failed.what(); + } catch (const std::exception& failed) { + failure = failed.what(); + } + dispatch_async(dispatch_get_main_queue(), ^{ + auto self = weak.lock(); + if (!self) return; + self->attached(attempt, udid, opened, initial_rotation, failure); + }); + }); + } + + void attached(std::uint64_t attempt, const std::string& udid, + const std::shared_ptr& opened, int initial_rotation, + const std::string& failure) { + if (attempt != generation) { + if (opened) { + __block std::shared_ptr retired = opened; + dispatch_async(attach_queue, ^{ retired.reset(); }); + } + return; + } + attaching_udid.clear(); + if (!failure.empty()) { + error = failure; + place(); + publish(); + return; + } + source = opened; + attached_udid = udid; + rotation = initial_rotation; + surface = source->surface(); + view.screenLayer.contents = surface; + place(); + publish(); + } + + void surface_changed(std::uint64_t attempt, id changed) { + if (attempt != generation) return; + surface = changed; + view.screenLayer.contents = changed; + if (!changed) error = "DISPLAY_GONE: the simulator display went away"; + place(); + publish(); + } + + void frame_arrived(std::uint64_t attempt, int turned) { + if (attempt != generation || !surface) return; + CALayer* screen = view.screenLayer; + if (turned != rotation) { + rotation = turned; + place(); + publish(); + } + // The IOSurface is updated in place; tell Core Animation to re-read it. + if ([screen respondsToSelector:@selector(setContentsChanged)]) { + [screen performSelector:@selector(setContentsChanged)]; + } else { + [CATransaction begin]; + [CATransaction setDisableActions:YES]; + screen.contents = nil; + screen.contents = surface; + [CATransaction commit]; + } + } +}; + +IosSimulatorView::IosSimulatorView(void* parent_window, StateHandler state_handler) + : impl_(std::make_shared()) { + impl_->window = (__bridge NSWindow*)parent_window; + impl_->on_state = std::move(state_handler); + impl_->attach_queue = dispatch_queue_create("dev.acecode.desktop.ios-simulator-view", DISPATCH_QUEUE_SERIAL); + impl_->view = [[ACECodeIosSimulatorScreenView alloc] initWithFrame:NSZeroRect]; + impl_->view.hidden = YES; +} + +IosSimulatorView::~IosSimulatorView() { + impl_->release_device(); + [impl_->view removeFromSuperview]; +} + +bool IosSimulatorView::supported() const { + std::error_code ignored; + return std::filesystem::exists(kCoreSimulatorFramework, ignored); +} + +bool IosSimulatorView::set_layout(const IosSimulatorViewLayout& layout, std::string* error) { + std::error_code ignored; + if (!std::filesystem::exists(layout.developer_dir + "/usr/bin/simctl", ignored)) { + if (error) *error = "developer_dir is not an Xcode developer directory"; + return false; + } + auto& impl = *impl_; + // Attach once per device; after a failure the page remounts to retry. + const bool device_changed = !impl.has_layout || layout.udid != impl.layout.udid; + impl.layout = layout; + impl.has_layout = true; + if (device_changed) impl.attach(layout.udid, layout.developer_dir); + impl.place(); + impl.publish(); + return true; +} + +void IosSimulatorView::unmount() { + impl_->has_layout = false; + impl_->error.clear(); + impl_->release_device(); + impl_->view.hidden = YES; + [impl_->view removeFromSuperview]; + impl_->publish(); +} + +nlohmann::json IosSimulatorView::state() const { + return impl_->snapshot(); +} + +} // namespace acecode::desktop diff --git a/src/apps/desktop/main.cpp b/src/apps/desktop/main.cpp index 85f5d1270..5b75058b6 100644 --- a/src/apps/desktop/main.cpp +++ b/src/apps/desktop/main.cpp @@ -15,6 +15,7 @@ #include "daemon_pool.hpp" #include "agent_browser_host.hpp" +#include "ios_simulator_view.hpp" #include "ipc/agent_browser_runtime.hpp" #include "context_items.hpp" #include "platform/native_ui/context_picker.hpp" @@ -1111,9 +1112,8 @@ int main(int argc, char** argv) { agent_browser_state_json(state).dump() + "}));}catch(e){}})();"); }, - [&host](std::function task) { - host.dispatch(std::move(task)); - }); + [&host](std::function task) { host.dispatch(std::move(task)); }); + const auto ios_simulator_view = install_ios_simulator_view(host); std::atomic restart_requested{false}; host.set_title("ACECode"); host.set_size(kDefaultDesktopWindowWidth, kDefaultDesktopWindowHeight); diff --git a/src/apps/tui/settings/settings_center.cpp b/src/apps/tui/settings/settings_center.cpp index 93c2fbb99..be34f8495 100644 --- a/src/apps/tui/settings/settings_center.cpp +++ b/src/apps/tui/settings/settings_center.cpp @@ -2242,24 +2242,24 @@ struct SettingsCenter::Impl { vbox({ hbox({ text("ACECode") | bold | size(WIDTH, EQUAL, 20), - text(deps.acecode_version), + text(deps.acecode_version) | flex_shrink, }), separator() | color(theme().ui.text_dim), hbox({ text("FTXUI") | bold | size(WIDTH, EQUAL, 20), - text(kFtxuiVersion), + text(kFtxuiVersion) | flex_shrink, }), separator() | color(theme().ui.text_dim), hbox({ text("Config path") | bold | size(WIDTH, EQUAL, 20), - text(config_path), + text(config_path) | flex_shrink, }), separator() | color(theme().ui.text_dim), hbox({ text("Project") | bold | size(WIDTH, EQUAL, 20), text(kProjectUrl) | - color(theme().markdown.link), + color(theme().markdown.link) | flex_shrink, }), }) | border | color(theme().ui.border), hbox({ diff --git a/src/apps/web/remote_web_proxy.cpp b/src/apps/web/remote_web_proxy.cpp index 86875c88c..6072f3b7c 100644 --- a/src/apps/web/remote_web_proxy.cpp +++ b/src/apps/web/remote_web_proxy.cpp @@ -31,6 +31,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -163,6 +166,65 @@ bool write_child_state( return acecode::atomic_write_file(path, state.dump(), true); } +[[maybe_unused]] bool usable_untrusted_source(const asio::ip::address_v4& address) { + return !address.is_loopback() && !address.is_unspecified() && + !address.is_multicast(); +} + +// The daemon must see proxied connections as untrusted. Windows and Linux +// route all of 127.0.0.0/8, so the upstream socket binds 127.0.0.2. macOS +// assigns only 127.0.0.1 to lo0 (EADDRNOTAVAIL); there the source is one of +// this machine's own non-loopback IPv4 addresses, preferring the one the +// remote client reached. A non-loopback source is still never trusted: the +// daemon only listens on 127.0.0.1, so it can only come from this machine. +bool bind_untrusted_upstream_source( + tcp::socket& upstream, + const tcp::socket& client) { + asio::error_code ec; + upstream.bind( + tcp::endpoint( + asio::ip::make_address_v4(kRemoteWebProxyUpstreamSource), 0), + ec); + if (!ec) return true; +#ifdef _WIN32 + (void)client; + return false; // Windows routes all of 127.0.0.0/8 +#else + // asio reports it in its own category, so compare the errno value. + if (ec.value() != EADDRNOTAVAIL) return false; + std::vector candidates; + asio::error_code local_ec; + const auto local = client.local_endpoint(local_ec).address(); + if (!local_ec) { + if (local.is_v4()) { + candidates.push_back(local.to_v4()); + } else if (local.to_v6().is_v4_mapped()) { + candidates.push_back( + asio::ip::make_address_v4(asio::ip::v4_mapped, local.to_v6())); + } + } + ifaddrs* interfaces = nullptr; + if (getifaddrs(&interfaces) == 0) { + for (auto* entry = interfaces; entry; entry = entry->ifa_next) { + if (!entry->ifa_addr || entry->ifa_addr->sa_family != AF_INET) { + continue; + } + const auto* in = + reinterpret_cast(entry->ifa_addr); + candidates.push_back( + asio::ip::address_v4(ntohl(in->sin_addr.s_addr))); + } + freeifaddrs(interfaces); + } + for (const auto& candidate : candidates) { + if (!usable_untrusted_source(candidate)) continue; + upstream.bind(tcp::endpoint(candidate, 0), ec); + if (!ec) return true; + } + return false; +#endif +} + class ProxyTunnel : public std::enable_shared_from_this { public: ProxyTunnel( @@ -181,11 +243,9 @@ class ProxyTunnel : public std::enable_shared_from_this { asio::error_code ec; upstream_.open(tcp::v4(), ec); if (ec) return finish(); - upstream_.bind( - tcp::endpoint( - asio::ip::make_address_v4(kRemoteWebProxyUpstreamSource), 0), - ec); - if (ec) return finish(); + if (!bind_untrusted_upstream_source(upstream_, client_)) { + return finish(); + } auto self = shared_from_this(); upstream_.async_connect( diff --git a/src/apps/web/routes/routes_ios_simulator.cpp b/src/apps/web/routes/routes_ios_simulator.cpp index 96a3cf134..0c57e2f32 100644 --- a/src/apps/web/routes/routes_ios_simulator.cpp +++ b/src/apps/web/routes/routes_ios_simulator.cpp @@ -24,6 +24,9 @@ struct SimulatorWsState { std::mutex mu; crow::websocket::connection* conn = nullptr; // null once closed; guarded by mu std::uint64_t subscription = 0; + // False while the Desktop shell draws the screen natively: no JPEG + // stream, the socket only carries input and status. + bool frames_enabled = true; double point_width = 0; double point_height = 0; }; @@ -41,6 +44,38 @@ std::string browser_frame(const sim::StreamFrame& frame) { return out; } +// Subscribes the socket to the device's JPEG stream (off Crow's IO thread: +// the first start can take seconds for Xcode discovery and helper start). +void subscribe_frames(const std::shared_ptr& state) { + std::thread([state] { + auto& runtime = sim::Runtime::instance(); + std::string error; + const auto subscription = runtime.subscribe(state->udid, [state](const sim::StreamFrame& frame) { + std::lock_guard lock(state->mu); + if (!state->conn) return; + if (frame.ended) { + state->conn->send_text(json({{"type", "ended"}, {"reason", frame.reason}}).dump()); + return; + } + if (frame.rotation == 90 || frame.rotation == 270) { + if (state->point_width < state->point_height) std::swap(state->point_width, state->point_height); + } else if (state->point_width > state->point_height) { + std::swap(state->point_width, state->point_height); + } + if (state->frames_enabled) state->conn->send_binary(browser_frame(frame)); + }, &error); + bool keep = false; + { + std::lock_guard lock(state->mu); + keep = state->conn != nullptr && state->frames_enabled && state->subscription == 0; + if (keep) state->subscription = subscription; + if (state->conn && !subscription) + state->conn->send_text(json({{"type", "error"}, {"message", error}}).dump()); + } + if (!keep && subscription) runtime.unsubscribe(subscription); + }).detach(); +} + json devices_payload(std::string* error) { auto& runtime = sim::Runtime::instance(); json devices = json::array(); @@ -207,11 +242,13 @@ void WebServer::Impl::register_ios_simulator() { return with_cors(req, std::move(image)); }); - // WS /ws/ios-simulator/?token=… — server → client: binary frames - // (u32le json_length | json | jpeg) and JSON status texts; client → server: - // {type:"touch",phase,x,y} | {type:"button",name} | {type:"key",name} | - // {type:"type",text} | {type:"text",text}. Points are in the interface - // orientation of the last frame. + // WS /ws/ios-simulator/?token=…[&frames=0] — server → client: binary + // frames (u32le json_length | json | jpeg) and JSON status texts; client → + // server: {type:"touch",phase,x,y} | {type:"button",name} | + // {type:"key",name} | {type:"type",text} | {type:"text",text} | + // {type:"frames",enabled}. Points are in the interface orientation of the + // last frame. frames=0 / enabled:false stop the JPEG stream while the + // Desktop shell draws the screen natively. CROW_WEBSOCKET_ROUTE(app, "/ws/ios-simulator/") .onaccept([this](const crow::request& req, void** userdata) -> bool { std::string header_token = req.get_header_value("X-ACECode-Token"); @@ -230,6 +267,7 @@ void WebServer::Impl::register_ios_simulator() { if (udid.empty() || udid.size() > 64) return false; auto* holder = new std::shared_ptr(std::make_shared()); (*holder)->udid = udid; + if (auto frames = req.url_params.get("frames")) (*holder)->frames_enabled = std::string(frames) != "0"; *userdata = holder; return true; }) @@ -241,43 +279,23 @@ void WebServer::Impl::register_ios_simulator() { std::lock_guard lock(state->mu); state->conn = &conn; } - // Starting the stream can take seconds the first time (Xcode + // Device discovery can take seconds the first time (Xcode // discovery, helper start); keep it off Crow's IO thread. std::thread([state] { - auto& runtime = sim::Runtime::instance(); std::string error; - if (auto info = runtime.describe(state->udid, &error)) { + if (auto info = sim::Runtime::instance().describe(state->udid, &error)) { std::lock_guard lock(state->mu); state->point_width = info->point_width; state->point_height = info->point_height; if (state->conn) state->conn->send_text(json({{"type", "device"}, {"device", sim::to_json(*info)}}).dump()); } - const auto subscription = runtime.subscribe(state->udid, [state](const sim::StreamFrame& frame) { - std::lock_guard lock(state->mu); - if (!state->conn) return; - if (frame.ended) { - state->conn->send_text(json({{"type", "ended"}, {"reason", frame.reason}}).dump()); - return; - } - if (frame.rotation == 90 || frame.rotation == 270) { - if (state->point_width < state->point_height) std::swap(state->point_width, state->point_height); - } else if (state->point_width > state->point_height) { - std::swap(state->point_width, state->point_height); - } - state->conn->send_binary(browser_frame(frame)); - }, &error); - bool closed = false; + bool frames = false; { std::lock_guard lock(state->mu); - closed = state->conn == nullptr; - if (!closed) { - state->subscription = subscription; - if (!subscription) - state->conn->send_text(json({{"type", "error"}, {"message", error}}).dump()); - } + frames = state->conn != nullptr && state->frames_enabled; } - if (closed && subscription) runtime.unsubscribe(subscription); + if (frames) subscribe_frames(state); }).detach(); }) .onmessage([](crow::websocket::connection& conn, const std::string& data, bool is_binary) { @@ -294,14 +312,30 @@ void WebServer::Impl::register_ios_simulator() { if (!std::isfinite(x) || !std::isfinite(y)) return; { std::lock_guard lock(state->mu); - if (state->point_width > 0) x = std::clamp(x, 0.0, state->point_width); - if (state->point_height > 0) y = std::clamp(y, 0.0, state->point_height); + // Without frames the orientation is not tracked here; the + // helper turns touches itself, so only bound them loosely. + const double bound = std::max(state->point_width, state->point_height); + const double width = state->frames_enabled ? state->point_width : bound; + const double height = state->frames_enabled ? state->point_height : bound; + if (width > 0) x = std::clamp(x, 0.0, width); + if (height > 0) y = std::clamp(y, 0.0, height); } const auto phase = msg.value("phase", std::string("down")); json request = {{"op", "touch"}, {"udid", state->udid}, {"phase", phase == "up" ? "up" : "down"}, {"x", x}, {"y", y}}; if (msg.contains("edge") && msg["edge"].is_string()) request["edge"] = msg["edge"]; runtime.post(request); + } else if (type == "frames") { + const bool enabled = msg.value("enabled", true); + std::uint64_t dropped = 0; + { + std::lock_guard lock(state->mu); + if (state->frames_enabled == enabled) return; + state->frames_enabled = enabled; + if (!enabled) std::swap(dropped, state->subscription); + } + if (dropped) runtime.unsubscribe(dropped); + if (enabled) subscribe_frames(state); } else if (type == "button") { runtime.post({{"op", "button"}, {"udid", state->udid}, {"name", msg.value("name", std::string())}}); } else if (type == "key") { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ba2d45259..6332ed2c1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -264,6 +264,16 @@ if(APPLE) MACOSX_BUNDLE_BUNDLE_VERSION "${ACECODE_BUILD_VERSION}" ) + # Manual, opt-in check of the Desktop shell's zero-copy iOS Simulator view + # against a booted simulator: + # ios_simulator_view_mac_smoke /Contents/Developer + add_executable(ios_simulator_view_mac_smoke EXCLUDE_FROM_ALL + ${ACECODE_IOS_SIMULATOR_VIEW_MAC_SMOKE_SOURCES}) + target_compile_options(ios_simulator_view_mac_smoke PRIVATE -fobjc-arc) + target_link_libraries(ios_simulator_view_mac_smoke PRIVATE + acecode_desktop_support acecode_ios_sim_native acecode_include_roots + "-framework AppKit" "-framework QuartzCore" "-framework IOSurface") + # Manual, opt-in runtime check. Run the bundle executable directly so its # authorization state and click callback remain visible in the terminal. add_executable(acecode_macos_notification_smoke diff --git a/tests/channels/core/state_log_test.cpp b/tests/channels/core/state_log_test.cpp index fcc4c1324..1caf38a09 100644 --- a/tests/channels/core/state_log_test.cpp +++ b/tests/channels/core/state_log_test.cpp @@ -187,7 +187,9 @@ TEST(ChannelStateLog, CredentialsAndPlatformExtras) { auto webhook = base; webhook["extra"]["webhook"] = true; - const auto* blocked = find_line(describe_state_changes(base, webhook), + // Keep the vector alive: find_line returns a pointer into it. + lines = describe_state_changes(base, webhook); + const auto* blocked = find_line(lines, "a webhook is set for this bot; long polling stays blocked until it is removed in " "the connect dialog"); ASSERT_NE(blocked, nullptr); diff --git a/tests/desktop/helpers/ios_simulator_view_mac_smoke.mm b/tests/desktop/helpers/ios_simulator_view_mac_smoke.mm new file mode 100644 index 000000000..4f1226972 --- /dev/null +++ b/tests/desktop/helpers/ios_simulator_view_mac_smoke.mm @@ -0,0 +1,151 @@ +// Opt-in smoke for the Desktop shell's native iOS Simulator view: +// ios_simulator_view_mac_smoke /Contents/Developer +// Drives the real IosSimulatorView in an off-screen window and checks the +// layer state (IOSurface contents, geometry, occlusion mask, visibility and +// unmount). Prints SMOKE_OK and exits 0 on success. +#include "desktop/ios_simulator_view.hpp" + +#import +#import +#import + +#include +#include +#include + +using acecode::desktop::IosSimulatorView; +using acecode::desktop::IosSimulatorViewLayout; + +@interface SmokeFlippedView : NSView +@end + +@implementation SmokeFlippedView +- (BOOL)isFlipped { + return YES; +} +@end + +namespace { + +int g_failures = 0; + +void check(bool condition, const char* what) { + std::printf("%s %s\n", condition ? "[ok] " : "[FAIL]", what); + if (!condition) ++g_failures; +} + +bool spin_until(double seconds, bool (^done)(void)) { + NSDate* deadline = [NSDate dateWithTimeIntervalSinceNow:seconds]; + while (!done()) { + if (deadline.timeIntervalSinceNow <= 0) return false; + [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; + } + return true; +} + +NSView* screen_view(NSView* content) { + for (NSView* view in content.subviews) + if ([NSStringFromClass([view class]) isEqualToString:@"ACECodeIosSimulatorScreenView"]) return view; + return nil; +} + +IosSimulatorViewLayout layout_for(const std::string& udid, const std::string& developer_dir, bool visible) { + IosSimulatorViewLayout layout; + layout.udid = udid; + layout.developer_dir = developer_dir; + layout.frame = {40, 30, 390, 844}; + layout.visible = visible; + layout.corner_radius = 48; + layout.occlusion_rects = {{0, 0, 120, 60}}; + return layout; +} + +} // namespace + +int main(int argc, char** argv) { + @autoreleasepool { + if (argc < 3) { + std::fprintf(stderr, "usage: %s \n", argv[0]); + return 64; + } + const std::string udid = argv[1]; + const std::string developer_dir = argv[2]; + [NSApplication sharedApplication]; + [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory]; + NSWindow* window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0, 0, 480, 900) + styleMask:NSWindowStyleMaskBorderless + backing:NSBackingStoreBuffered + defer:NO]; + SmokeFlippedView* content = [[SmokeFlippedView alloc] initWithFrame:NSMakeRect(0, 0, 480, 900)]; + content.wantsLayer = YES; + window.contentView = content; + [window setFrameOrigin:NSMakePoint(-20000, -20000)]; // layer state only, no pixels + [window orderFrontRegardless]; + + nlohmann::json last = nlohmann::json::object(); + nlohmann::json* last_state = &last; // blocks capture the pointer + IosSimulatorView view((__bridge void*)window, + [last_state](const nlohmann::json& state) { *last_state = state; }); + check(view.supported(), "CoreSimulator is installed"); + + std::string error; + auto bad = layout_for(udid, "/nonexistent/Contents/Developer", true); + check(!view.set_layout(bad, &error) && !error.empty(), "a non-Xcode developer_dir is refused"); + + check(view.set_layout(layout_for(udid, developer_dir, true), &error), "layout accepted"); + const bool rendering = spin_until(15, ^{ return last_state->value("rendering", false); }); + std::printf("state: %s\n", last.dump().c_str()); + check(rendering, "the view renders the simulator"); + + NSView* screen = screen_view(content); + check(screen != nil && !screen.hidden, "screen view is placed over the content"); + if (screen) { + check(NSEqualRects(screen.frame, NSMakeRect(40, 30, 390, 844)), "screen view uses the DOM frame"); + check(std::fabs(screen.layer.cornerRadius - 48) < 0.01, "rounded device corners"); + check(screen.layer.mask != nil, "menus over the screen are cut out"); + check([screen hitTest:NSMakePoint(100, 100)] == nil, "input passes through to the page"); + CALayer* layer = screen.layer.sublayers.firstObject; + id contents = layer.contents; + const bool is_surface = contents && CFGetTypeID((__bridge CFTypeRef)contents) == IOSurfaceGetTypeID(); + check(is_surface, "layer contents are the simulator IOSurface (zero copy)"); + if (is_surface) { + IOSurfaceRef surface = (__bridge IOSurfaceRef)contents; + check(static_cast(IOSurfaceGetWidth(surface)) == last.value("pixel_width", 0.0), + "reported pixel size matches the surface"); + const double expected_scale = std::fmin(390.0 / IOSurfaceGetWidth(surface), + 844.0 / IOSurfaceGetHeight(surface)); + if (last.value("rotation", 0) == 0) { + check(std::fabs(layer.bounds.size.width - IOSurfaceGetWidth(surface) * expected_scale) < 0.5, + "layer is aspect-fitted"); + check(CATransform3DIsIdentity(layer.transform), "portrait needs no rotation"); + } + } + } + + check(view.set_layout(layout_for(udid, developer_dir, false), &error), "hide accepted"); + spin_until(2, ^{ return !last_state->value("rendering", true); }); + check(!last.value("rendering", true) && (screen == nil || screen.hidden), "hidden layout hides the view"); + check(last.value("attached", false), "hiding keeps the device attached"); + + check(view.set_layout(layout_for(udid, developer_dir, true), &error), "show accepted"); + check(spin_until(2, ^{ return last_state->value("rendering", false); }), "shows again without re-attaching"); + + view.unmount(); + check(!last.value("attached", true) && screen_view(content) == nil, "unmount removes the view"); + + auto missing = layout_for("00000000-0000-0000-0000-000000000000", developer_dir, true); + check(view.set_layout(missing, &error), "unknown device layout accepted"); + spin_until(10, ^{ return !last_state->value("error", std::string()).empty(); }); + check(last.value("error", std::string()).find("DEVICE_NOT_FOUND") != std::string::npos && + !last.value("rendering", true), + "an unknown device reports DEVICE_NOT_FOUND"); + view.unmount(); + + if (g_failures) { + std::printf("SMOKE_FAILED %d\n", g_failures); + return 1; + } + std::printf("SMOKE_OK\n"); + return 0; + } +} diff --git a/tests/desktop/ios_simulator_view_layout_test.cpp b/tests/desktop/ios_simulator_view_layout_test.cpp new file mode 100644 index 000000000..697f9cd7d --- /dev/null +++ b/tests/desktop/ios_simulator_view_layout_test.cpp @@ -0,0 +1,126 @@ +#include + +#include "desktop/ios_simulator_view_layout.hpp" + +#include + +using namespace acecode::desktop; +using nlohmann::json; + +namespace { + +json sample_layout() { + return {{"udid", "EBDA0D80-2BB4-4C9E-804C-5F5299CC1A2D"}, + {"developer_dir", "/Applications/Xcode.app/Contents/Developer"}, + {"x", 120}, + {"y", 64.6}, + {"width", 390}, + {"height", 844}, + {"visible", true}, + {"corner_radius", 42.5}, + {"layout_revision", 7}, + {"occlusion_rects", json::array({{{"x", 10}, {"y", 20}, {"width", 30}, {"height", 40}}})}}; +} + +} // namespace + +// 场景:网页面板把屏幕区域交给 Desktop 原生视图。期望:字段按 CSS 像素解析, +// UDID / developer_dir / 遮挡矩形逐项校验,坏输入给出可读错误。 +TEST(IosSimulatorViewLayout, ParsesTheBridgeLayout) { + std::string error; + const auto layout = parse_ios_simulator_view_layout(sample_layout(), &error); + ASSERT_TRUE(layout) << error; + EXPECT_EQ(layout->udid, "EBDA0D80-2BB4-4C9E-804C-5F5299CC1A2D"); + EXPECT_EQ(layout->frame.x, 120); + EXPECT_EQ(layout->frame.y, 64); + EXPECT_EQ(layout->frame.width, 390); + EXPECT_EQ(layout->frame.height, 844); + EXPECT_TRUE(layout->visible); + EXPECT_DOUBLE_EQ(layout->corner_radius, 42.5); + EXPECT_EQ(layout->layout_revision, 7u); + ASSERT_EQ(layout->occlusion_rects.size(), 1u); + EXPECT_EQ(layout->occlusion_rects[0].height, 40); +} + +TEST(IosSimulatorViewLayout, RejectsMalformedLayouts) { + std::string error; + auto bad = sample_layout(); + bad["udid"] = "../../etc"; + EXPECT_FALSE(parse_ios_simulator_view_layout(bad, &error)); + EXPECT_NE(error.find("UDID"), std::string::npos); + + bad = sample_layout(); + bad["developer_dir"] = "relative/Developer"; + EXPECT_FALSE(parse_ios_simulator_view_layout(bad, &error)); + + bad = sample_layout(); + bad["width"] = "wide"; + EXPECT_FALSE(parse_ios_simulator_view_layout(bad, &error)); + + bad = sample_layout(); + bad["corner_radius"] = "round"; + EXPECT_FALSE(parse_ios_simulator_view_layout(bad, &error)); + + bad = sample_layout(); + bad["layout_revision"] = -1; + EXPECT_FALSE(parse_ios_simulator_view_layout(bad, &error)); + + bad = sample_layout(); + bad["occlusion_rects"] = json::array(); + for (int index = 0; index < 65; ++index) bad["occlusion_rects"].push_back({{"x", 0}}); + EXPECT_FALSE(parse_ios_simulator_view_layout(bad, &error)); + + EXPECT_FALSE(parse_ios_simulator_view_layout(json::array(), &error)); + EXPECT_FALSE(is_simulator_udid("EBDA0D80-2BB4-4C9E-804C-5F5299CC1A2")); + EXPECT_FALSE(is_simulator_udid("EBDA0D80X2BB4-4C9E-804C-5F5299CC1A2D")); + EXPECT_TRUE(is_simulator_udid("ebda0d80-2bb4-4c9e-804c-5f5299cc1a2d")); +} + +// 场景:菜单 / 对话框部分盖住屏幕。期望:遮挡矩形裁到屏幕区域内,完全在外的丢弃。 +TEST(IosSimulatorViewLayout, ClipsOcclusionRectsToTheFrame) { + IosSimulatorViewLayout layout; + layout.frame = {0, 0, 100, 200}; + layout.occlusion_rects = {{-10, -10, 30, 30}, {90, 190, 50, 50}, {200, 0, 10, 10}, {10, 10, 0, 5}}; + const auto clipped = clipped_occlusion_rects(layout); + ASSERT_EQ(clipped.size(), 2u); + EXPECT_EQ(clipped[0].x, 0); + EXPECT_EQ(clipped[0].y, 0); + EXPECT_EQ(clipped[0].width, 20); + EXPECT_EQ(clipped[0].height, 20); + EXPECT_EQ(clipped[1].x, 90); + EXPECT_EQ(clipped[1].width, 10); + EXPECT_EQ(clipped[1].height, 10); +} + +// 场景:竖屏帧缓冲按界面方向转后等比放进屏幕区域。期望:图层保持帧缓冲本身的宽高比, +// 横屏时按转过来的尺寸适配,中心在区域中心。 +TEST(IosSimulatorViewLayout, FitsThePortraitFramebufferForEachRotation) { + auto portrait = fit_simulator_layer(390, 844, 1170, 2532, 0); + EXPECT_NEAR(portrait.width, 390, 1e-6); + EXPECT_NEAR(portrait.height, 844, 1e-6); + EXPECT_NEAR(portrait.center_x, 195, 1e-6); + EXPECT_NEAR(portrait.center_y, 422, 1e-6); + + auto landscape = fit_simulator_layer(844, 390, 1170, 2532, 90); + EXPECT_EQ(landscape.rotation, 90); + EXPECT_NEAR(landscape.width, 390, 1e-6) << "unrotated bounds stay portrait"; + EXPECT_NEAR(landscape.height, 844, 1e-6); + + auto letterboxed = fit_simulator_layer(400, 400, 1170, 2532, 270); + EXPECT_EQ(letterboxed.rotation, 270); + EXPECT_NEAR(letterboxed.height, 400, 1e-6) << "the turned width fills the box"; + EXPECT_NEAR(letterboxed.width, 400 * 1170.0 / 2532.0, 1e-6); + + EXPECT_EQ(fit_simulator_layer(100, 100, 10, 20, -90).rotation, 270); + EXPECT_EQ(fit_simulator_layer(100, 100, 10, 20, 45).rotation, 0); + EXPECT_EQ(fit_simulator_layer(0, 100, 10, 20, 0).width, 0); +} + +TEST(IosSimulatorViewLayout, LayerAngleTurnsClockwiseInFlippedGeometry) { + const double pi = std::acos(-1.0); + EXPECT_NEAR(simulator_layer_angle_radians(90, true), pi / 2, 1e-9); + EXPECT_NEAR(simulator_layer_angle_radians(90, false), -pi / 2, 1e-9); + EXPECT_NEAR(simulator_layer_angle_radians(270, true), 3 * pi / 2, 1e-9); + EXPECT_NEAR(simulator_layer_angle_radians(-90, true), 3 * pi / 2, 1e-9); + EXPECT_NEAR(simulator_layer_angle_radians(0, false), 0, 1e-9); +} diff --git a/tests/network/websocket_client_test.cpp b/tests/network/websocket_client_test.cpp index ba9ea9f4a..707137121 100644 --- a/tests/network/websocket_client_test.cpp +++ b/tests/network/websocket_client_test.cpp @@ -151,7 +151,11 @@ TEST(WebSocketClient, ServerShutdownIsReportedAsDisconnect) { WebSocketClient client; std::string error; ASSERT_TRUE(client.connect(local_options(*server), &error)) << error; + // Crow 的 stop() 只把关闭帧投递到 io_context 后立刻停掉它,关闭帧能否发出 + // 取决于调度(macOS 上常发不出去);连接的套接字要等应用析构才释放。 + // 连同应用一起销毁才等价于「服务端进程退出」。 server.reset(); + echo.reset(); WebSocketMessage message; const auto result = client.receive(message, std::chrono::seconds(5), &error); EXPECT_TRUE(result == WebSocketRecv::Closed || result == WebSocketRecv::Error); diff --git a/tests/sandbox/sandbox_policy_test.cpp b/tests/sandbox/sandbox_policy_test.cpp index 6a10832de..5eabddd57 100644 --- a/tests/sandbox/sandbox_policy_test.cpp +++ b/tests/sandbox/sandbox_policy_test.cpp @@ -97,21 +97,23 @@ std::string canon(const std::string& path) { return canonical_policy_path(path); // 场景:deny 名单、可写根、只读子路径、可读根共存时的访问判定。期望:最长前缀 // 命中优先(写根下的 deny 子目录仍 Deny;deny 目录下配置的写根反而可写),同深度 // deny > write > read;全盘可读时未命中的路径 Read,受限读时 Deny;FullAccess 恒 Write。 +// macOS 的 /home 是 autofs 挂载点,规范化 /home/u 会触发自动挂载查询、两次结果 +// 可能不同(条目与查询对不上),所以用不存在的普通根目录。 // 回归:条目曾直接写 `/work` 而不经规范化,在存在 C:\work 的 Windows 机器上 // `/work/src/main.cpp` 判成 Read、`/work/secrets/key` 判成 Read(期望 Write / Deny)。 TEST(SandboxPolicy, ResolveAccessPrefersDeepestEntryThenDenyWriteRead) { SandboxPolicy policy; policy.mode = SandboxMode::WorkspaceWrite; policy.writable_roots = {{canon("/work"), {canon("/work/.acecode/rules")}}, - {canon("/home/u/.cache/allowed"), {}}}; - policy.denied_paths = {canon("/work/secrets"), canon("/home/u/.cache")}; + {canon("/policy-home/u/.cache/allowed"), {}}}; + policy.denied_paths = {canon("/work/secrets"), canon("/policy-home/u/.cache")}; policy.denied_globs = {canon("/work") + "/**/.env"}; EXPECT_EQ(policy.resolve_access("/work/src/main.cpp"), FsAccess::Write); EXPECT_EQ(policy.resolve_access("/work/.acecode/rules/x.rules"), FsAccess::Read); EXPECT_EQ(policy.resolve_access("/work/secrets/key"), FsAccess::Deny); EXPECT_EQ(policy.resolve_access("/work/app/.env"), FsAccess::Deny); - EXPECT_EQ(policy.resolve_access("/home/u/.cache/other"), FsAccess::Deny); - EXPECT_EQ(policy.resolve_access("/home/u/.cache/allowed/pkg"), FsAccess::Write) << "更深的写根压过浅的 deny"; + EXPECT_EQ(policy.resolve_access("/policy-home/u/.cache/other"), FsAccess::Deny); + EXPECT_EQ(policy.resolve_access("/policy-home/u/.cache/allowed/pkg"), FsAccess::Write) << "更深的写根压过浅的 deny"; EXPECT_EQ(policy.resolve_access("/etc/hosts"), FsAccess::Read) << "全盘可读"; EXPECT_TRUE(policy.can_read("/etc/hosts")); EXPECT_FALSE(policy.can_write("/etc/hosts")); diff --git a/tests/tool/builtin_tool_registry_test.cpp b/tests/tool/builtin_tool_registry_test.cpp index a47163710..d12961b66 100644 --- a/tests/tool/builtin_tool_registry_test.cpp +++ b/tests/tool/builtin_tool_registry_test.cpp @@ -32,7 +32,8 @@ TEST(BuiltinToolRegistry, RegistersSharedCoreAndPlatformBrowserTools) { EXPECT_FALSE(tools.is_read_only("ExitPlanMode")); EXPECT_TRUE(tools.is_read_only("show_image")); EXPECT_FALSE(tools.has_tool("browser_start")); -#ifdef _WIN32 + // Agent Browser 有 Windows(WebView2)与 macOS(WKWebView)两个原生宿主。 +#if defined(_WIN32) || defined(__APPLE__) EXPECT_TRUE(tools.has_tool("browser_open")); EXPECT_TRUE(tools.has_tool("browser_read_page")); EXPECT_TRUE(tools.has_tool("browser_click")); @@ -50,7 +51,7 @@ TEST(BuiltinToolRegistry, NativeBrowserToolsCanBeUnregisteredAsOneGroup) { register_session_builtin_tools(tools, config); const std::size_t removed = agent_browser::unregister_agent_browser_tools(tools); -#ifdef _WIN32 +#if defined(_WIN32) || defined(__APPLE__) EXPECT_GT(removed, 0u); #else EXPECT_EQ(removed, 0u); diff --git a/tests/tool/ios_simulator/dtuhid_messages_test.cpp b/tests/tool/ios_simulator/dtuhid_messages_test.cpp new file mode 100644 index 000000000..89512464b --- /dev/null +++ b/tests/tool/ios_simulator/dtuhid_messages_test.cpp @@ -0,0 +1,121 @@ +#include + +#include "tool/ios_simulator/dtuhid_messages.hpp" + +#include + +using namespace acecode::ios_simulator::dtuhid; + +// 场景:一根手指按下 → 移动 → 抬起,再按下;两根手指各自独立计相位。 +// 期望:0 began / 1 moved / 2 ended,抬起后的下一次按下重新开始。 +TEST(IosSimulatorDtuhid, ContactPhasesFollowDownMoveUp) { + ContactTracker finger; + EXPECT_EQ(finger.next(false), Phase::Began); + EXPECT_TRUE(finger.active()); + EXPECT_EQ(finger.next(false), Phase::Moved); + EXPECT_EQ(finger.next(false), Phase::Moved); + EXPECT_EQ(finger.next(true), Phase::Ended); + EXPECT_FALSE(finger.active()); + EXPECT_EQ(finger.next(false), Phase::Began); + EXPECT_EQ(static_cast(Phase::Began), 0u); + EXPECT_EQ(static_cast(Phase::Moved), 1u); + EXPECT_EQ(static_cast(Phase::Ended), 2u); +} + +TEST(IosSimulatorDtuhid, PointsAreFractionsOfThePortraitPointSize) { + const auto point = unit_point(195, 422, 390, 844); + EXPECT_DOUBLE_EQ(point.x, 0.5); + EXPECT_DOUBLE_EQ(point.y, 0.5); + const auto corner = unit_point(390, 0, 390, 844); + EXPECT_DOUBLE_EQ(corner.x, 1.0); + EXPECT_DOUBLE_EQ(corner.y, 0.0); + const auto unknown = unit_point(10, 10, 0, 0); + EXPECT_DOUBLE_EQ(unknown.x, 0.0); + EXPECT_DOUBLE_EQ(unknown.y, 0.0); + EXPECT_EQ(button_state(true), 1u); + EXPECT_EQ(button_state(false), 2u); +} + +#ifdef __APPLE__ +namespace { + +std::string text(xpc_object_t dictionary, const char* key) { + const char* value = xpc_dictionary_get_string(dictionary, key); + return value ? value : ""; +} + +bool has_key(xpc_object_t dictionary, const char* key) { return xpc_dictionary_get_value(dictionary, key) != nullptr; } + +std::size_t key_count(xpc_object_t dictionary) { return xpc_dictionary_get_count(dictionary); } + +} // namespace + +// 场景:单指触摸消息。期望:信封四个键齐全,payload 是 pointOne(0..1)+eventType+edge+target, +// 没有 pointTwo;信封持有 payload,调用方释放自己那份后仍可读。 +TEST(IosSimulatorDtuhid, DigitizerEnvelopeMatchesTheServiceSchema) { + xpc_object_t payload = digitizer_payload(unit_point(39, 84.4, 390, 844), nullptr, Phase::Began, 3); + xpc_object_t message = envelope(kDigitizerEvent, payload, false); + xpc_release(payload); + + EXPECT_EQ(key_count(message), 4u); + EXPECT_EQ(text(message, "messageType"), "IndigoDigitizerEvent"); + EXPECT_EQ(text(message, "featureIdentifier"), "com.apple.coredevice.feature.remote.hid.digitizer"); + EXPECT_FALSE(xpc_dictionary_get_bool(message, "isBarrier")); + xpc_object_t body = xpc_dictionary_get_value(message, "payload"); + ASSERT_NE(body, nullptr); + ASSERT_EQ(xpc_get_type(body), XPC_TYPE_DICTIONARY); + EXPECT_EQ(key_count(body), 4u); + xpc_object_t one = xpc_dictionary_get_value(body, "pointOne"); + ASSERT_NE(one, nullptr); + EXPECT_DOUBLE_EQ(xpc_dictionary_get_double(one, "x"), 0.1); + EXPECT_DOUBLE_EQ(xpc_dictionary_get_double(one, "y"), 0.1); + EXPECT_FALSE(has_key(body, "pointTwo")); + EXPECT_EQ(xpc_dictionary_get_uint64(body, "eventType"), 0u); + EXPECT_EQ(xpc_dictionary_get_uint64(body, "edge"), 3u); + EXPECT_EQ(xpc_dictionary_get_uint64(body, "target"), 0u); + EXPECT_EQ(xpc_get_type(xpc_dictionary_get_value(body, "eventType")), XPC_TYPE_UINT64); + xpc_release(message); +} + +// 场景:双指(捏合 / 旋转)。期望:带 pointTwo,edge 恒为 0(双指不触发边缘手势)。 +TEST(IosSimulatorDtuhid, TwoFingerEventCarriesPointTwoWithoutEdge) { + const UnitPoint second{0.75, 0.25}; + xpc_object_t payload = digitizer_payload({0.25, 0.75}, &second, Phase::Moved, 4); + EXPECT_EQ(key_count(payload), 5u); + xpc_object_t two = xpc_dictionary_get_value(payload, "pointTwo"); + ASSERT_NE(two, nullptr); + EXPECT_DOUBLE_EQ(xpc_dictionary_get_double(two, "x"), 0.75); + EXPECT_DOUBLE_EQ(xpc_dictionary_get_double(two, "y"), 0.25); + EXPECT_EQ(xpc_dictionary_get_uint64(payload, "eventType"), 1u); + EXPECT_EQ(xpc_dictionary_get_uint64(payload, "edge"), 0u); + xpc_release(payload); +} + +TEST(IosSimulatorDtuhid, ButtonAndKeyboardPayloads) { + xpc_object_t home = button_payload(0x0C, 0x40, true); + EXPECT_EQ(key_count(home), 3u); + EXPECT_EQ(xpc_dictionary_get_uint64(home, "usagePage"), 0x0Cu); + EXPECT_EQ(xpc_dictionary_get_uint64(home, "usageCode"), 0x40u); + EXPECT_EQ(xpc_dictionary_get_uint64(home, "state"), 1u); + xpc_release(home); + + xpc_object_t key = keyboard_payload(0x04, false); + EXPECT_EQ(key_count(key), 2u); + EXPECT_EQ(xpc_dictionary_get_uint64(key, "usageCode"), 0x04u); + EXPECT_EQ(xpc_dictionary_get_uint64(key, "state"), 2u); + EXPECT_FALSE(has_key(key, "usagePage")); + xpc_release(key); +} + +// 场景:连接后的存活探测。期望:barrier 信封里是 usage 0 的键盘抬起(对设备无副作用)。 +TEST(IosSimulatorDtuhid, LivenessProbeIsABarrierKeyUpOfUsageZero) { + xpc_object_t probe = liveness_probe(); + EXPECT_EQ(text(probe, "messageType"), "IndigoKeyboardButtonEvent"); + EXPECT_TRUE(xpc_dictionary_get_bool(probe, "isBarrier")); + xpc_object_t body = xpc_dictionary_get_value(probe, "payload"); + ASSERT_NE(body, nullptr); + EXPECT_EQ(xpc_dictionary_get_uint64(body, "usageCode"), 0u); + EXPECT_EQ(xpc_dictionary_get_uint64(body, "state"), 2u); + xpc_release(probe); +} +#endif diff --git a/tests/tool/ios_simulator/helper_sandbox_test.cpp b/tests/tool/ios_simulator/helper_sandbox_test.cpp new file mode 100644 index 000000000..46f02bda3 --- /dev/null +++ b/tests/tool/ios_simulator/helper_sandbox_test.cpp @@ -0,0 +1,142 @@ +#include + +#include "tool/ios_simulator/helper_sandbox.hpp" + +#include +#include +#include +#include + +using namespace acecode::ios_simulator; +namespace fs = std::filesystem; + +namespace { + +HelperSandboxPaths sample_paths() { + HelperSandboxPaths paths; + paths.helper_dir = "/Applications/ACECode.app"; + paths.listable_dirs = {"/Applications/ACECode.app/Contents", "/Applications/ACECode.app"}; + paths.xcode_app = "/Users/dev/Downloads/Xcode \"beta\".app"; + paths.home = "/Users/dev"; + paths.temp_dir = "/private/var/folders/xx/T"; + paths.cache_dir = "/private/var/folders/xx/C"; + return paths; +} + +bool contains(const std::vector& values, const std::string& value) { + return std::find(values.begin(), values.end(), value) != values.end(); +} + +} // namespace + +// 场景:辅助进程的 Seatbelt 配置。期望:禁 IP 网络、禁读 /Users 与 /Volumes 下的 +// 文件内容(放行 Xcode、辅助进程包、模拟器数据与偏好文件)、只许写模拟器数据与临时目录。 +TEST(IosSimulatorHelperSandbox, ProfileDeniesNetworkPersonalFilesAndStrayWrites) { + const auto profile = helper_sandbox_profile(sample_paths()); + EXPECT_EQ(profile.rfind("(version 1)\n(allow default)\n", 0), 0u); + EXPECT_NE(profile.find("(deny network-outbound (remote ip \"*:*\"))"), std::string::npos); + EXPECT_NE(profile.find("(deny network-inbound (local ip \"*:*\"))"), std::string::npos); + EXPECT_NE(profile.find("(deny network-bind (local ip \"*:*\"))"), std::string::npos); + EXPECT_NE(profile.find("(deny file-read-data (subpath \"/Users\") (subpath \"/Volumes\") " + "(subpath (param \"HOME_DIR\")))"), + std::string::npos); + for (const char* name : {"HELPER_DIR", "XCODE_APP", "SIM_DATA", "SIM_LOGS", "CORE_DEVICE", "TEMP_DIR"}) + EXPECT_NE(profile.find(std::string("(subpath (param \"") + name + "\"))"), std::string::npos) << name; + EXPECT_NE(profile.find("(literal (param \"PREF_SIMULATOR\"))"), std::string::npos); + EXPECT_NE(profile.find("(literal (param \"LISTABLE_0\"))"), std::string::npos); + EXPECT_NE(profile.find("(deny file-write* (subpath \"/\"))"), std::string::npos); + const auto writes = profile.substr(profile.find("(allow file-write*")); + EXPECT_NE(writes.find("(subpath \"/dev\")"), std::string::npos); + EXPECT_NE(writes.find("SIM_DATA"), std::string::npos); + EXPECT_EQ(writes.find("XCODE_APP"), std::string::npos) << "Xcode stays read-only"; + EXPECT_EQ(writes.find("HELPER_DIR"), std::string::npos) << "the helper bundle stays read-only"; + // 后写的规则优先:放行必须排在对应的拒绝之后。 + EXPECT_LT(profile.find("(deny file-read-data"), profile.find("(allow file-read-data")); + EXPECT_LT(profile.find("(deny file-write*"), profile.find("(allow file-write*")); +} + +// 场景:路径(含引号、空格)只经 -D 参数传入,绝不拼进 SBPL 文本。 +TEST(IosSimulatorHelperSandbox, PathsTravelAsParametersNotProfileText) { + const auto paths = sample_paths(); + const auto argv = sandboxed_helper_argv(paths, {"/Applications/ACECode.app/Contents/MacOS/acecode-ios-sim", + "--developer-dir", "/x/Contents/Developer"}); + ASSERT_GE(argv.size(), 4u); + EXPECT_EQ(argv[0], "/usr/bin/sandbox-exec"); + EXPECT_EQ(argv[1], "-p"); + EXPECT_EQ(argv[2].find("Downloads"), std::string::npos); + EXPECT_EQ(argv[2].find("/Users/dev"), std::string::npos); + EXPECT_TRUE(contains(argv, "-DXCODE_APP=/Users/dev/Downloads/Xcode \"beta\".app")); + EXPECT_TRUE(contains(argv, "-DSIM_DATA=/Users/dev/Library/Developer/CoreSimulator")); + EXPECT_TRUE(contains(argv, "-DLISTABLE_1=/Applications/ACECode.app")); + EXPECT_EQ(argv[argv.size() - 3], "/Applications/ACECode.app/Contents/MacOS/acecode-ios-sim"); + EXPECT_EQ(argv.back(), "/x/Contents/Developer"); + // 每个 -D 参数都在被沙盒的命令之前。 + const auto command = std::find(argv.begin(), argv.end(), argv[argv.size() - 3]); + for (auto it = command; it != argv.end(); ++it) EXPECT_NE(it->rfind("-D", 0), 0u) << *it; +} + +// 场景:缺的路径(没有 HOME、取不到缓存目录)连同用到它的规则一起省略, +// 否则 (param "X") 求值为 #f,sandbox-exec 直接拒绝整份配置。 +TEST(IosSimulatorHelperSandbox, MissingPathsDropTheirRules) { + HelperSandboxPaths paths; + paths.helper_dir = "/opt/acecode"; + paths.xcode_app = "/Applications/Xcode.app"; + const auto profile = helper_sandbox_profile(paths); + EXPECT_EQ(profile.find("HOME_DIR"), std::string::npos); + EXPECT_EQ(profile.find("SIM_DATA"), std::string::npos); + EXPECT_EQ(profile.find("CACHE_DIR"), std::string::npos); + EXPECT_NE(profile.find("(deny file-read-data (subpath \"/Users\") (subpath \"/Volumes\"))"), std::string::npos); + const auto argv = sandboxed_helper_argv(paths, {"helper"}); + for (const auto& arg : argv) { + if (arg.rfind("-D", 0) != 0) continue; + EXPECT_NE(arg.back(), '=') << "empty parameter " << arg; + } +} + +TEST(IosSimulatorHelperSandbox, XcodeAppForDeveloperDir) { + EXPECT_EQ(xcode_app_for_developer_dir("/Applications/Xcode.app/Contents/Developer"), "/Applications/Xcode.app"); + EXPECT_EQ(xcode_app_for_developer_dir("/Users/a/Xcode-beta.app/Contents/Developer/"), "/Users/a/Xcode-beta.app"); + EXPECT_EQ(xcode_app_for_developer_dir("/Library/Developer/CommandLineTools"), "/Library/Developer/CommandLineTools"); +} + +#ifdef __APPLE__ +// 场景:辅助进程装在 X.app/Contents/MacOS 里时整个包可读(Info.plist 在 Contents/); +// 不在包里时只放行可执行文件所在目录,祖先目录只许列名(CFBundle 定位主包要用)。 +TEST(IosSimulatorHelperSandbox, CurrentPathsUseTheBundleRootAndListableAncestors) { + const auto root = fs::temp_directory_path() / "acecode-helper-sandbox-test"; + fs::remove_all(root); + const auto macos = root / "Apps" / "ACECode.app" / "Contents" / "MacOS"; + fs::create_directories(macos); + std::ofstream(macos / "acecode-ios-sim") << ""; + const auto canonical_root = fs::canonical(root); + + auto paths = current_helper_sandbox_paths((macos / "acecode-ios-sim").string(), + "/Applications/Xcode.app/Contents/Developer"); + EXPECT_EQ(paths.helper_dir, (canonical_root / "Apps" / "ACECode.app").string()); + ASSERT_EQ(paths.listable_dirs.size(), 2u); + EXPECT_EQ(paths.listable_dirs[0], (canonical_root / "Apps" / "ACECode.app" / "Contents").string()); + EXPECT_EQ(paths.listable_dirs[1], (canonical_root / "Apps" / "ACECode.app").string()); + EXPECT_EQ(paths.xcode_app, fs::weakly_canonical("/Applications/Xcode.app").string()); + + const auto build = root / "build" / "macos"; + fs::create_directories(build); + paths = current_helper_sandbox_paths((build / "acecode-ios-sim").string(), "/x/Xcode.app/Contents/Developer"); + EXPECT_EQ(paths.helper_dir, (canonical_root / "build" / "macos").string()); + ASSERT_EQ(paths.listable_dirs.size(), 2u); + EXPECT_EQ(paths.listable_dirs[0], (canonical_root / "build").string()); + EXPECT_EQ(paths.listable_dirs[1], canonical_root.string()); + fs::remove_all(root); +} + +TEST(IosSimulatorHelperSandbox, EnvironmentEscapeHatchDisablesIt) { + const char* previous = std::getenv("ACECODE_IOS_SIM_SANDBOX"); + const std::string saved = previous ? previous : ""; + setenv("ACECODE_IOS_SIM_SANDBOX", "0", 1); + std::string reason; + EXPECT_FALSE(helper_sandbox_enabled(&reason)); + EXPECT_NE(reason.find("ACECODE_IOS_SIM_SANDBOX=0"), std::string::npos); + if (previous) setenv("ACECODE_IOS_SIM_SANDBOX", saved.c_str(), 1); + else unsetenv("ACECODE_IOS_SIM_SANDBOX"); + EXPECT_TRUE(helper_sandbox_enabled()); +} +#endif diff --git a/tests/tool/ios_simulator/ios_simulator_e2e_test.cpp b/tests/tool/ios_simulator/ios_simulator_e2e_test.cpp index b29b13d6d..578eb838f 100644 --- a/tests/tool/ios_simulator/ios_simulator_e2e_test.cpp +++ b/tests/tool/ios_simulator/ios_simulator_e2e_test.cpp @@ -5,6 +5,7 @@ #include #include "tool/ios_simulator/device_consent.hpp" +#include "tool/ios_simulator/helper_sandbox.hpp" #include "tool/ios_simulator/runtime.hpp" #include "tool/ios_simulator/tool_actions.hpp" #include "utils/state_file.hpp" @@ -57,6 +58,11 @@ TEST_F(IosSimulatorE2E, AttachScreenshotInspectAndInput) { ASSERT_TRUE(attach.success) << attach.output; EXPECT_EQ(attach.metadata["ios_simulator"]["udid"], udid_); EXPECT_TRUE(device_consent_granted(udid_)); + // The helper runs inside its Seatbelt profile unless explicitly disabled. + if (helper_sandbox_enabled()) { + const auto helper = Runtime::instance().status(false)["helper"]; + EXPECT_TRUE(helper.value("sandboxed", false)) << helper.dump(); + } auto shot = run_simulator_action("screenshot", args({}), ctx_); ASSERT_TRUE(shot.success) << shot.output; diff --git a/tests/utils/state_file_test.cpp b/tests/utils/state_file_test.cpp index 59f1a9a02..0b8a019b8 100644 --- a/tests/utils/state_file_test.cpp +++ b/tests/utils/state_file_test.cpp @@ -30,25 +30,48 @@ #include #include +#ifndef _WIN32 +#include +#include +extern char** environ; +#endif + namespace fs = std::filesystem; using acecode::test_support::StateFileTest; using acecode::test_support::write_raw; namespace { +#ifdef _WIN32 std::string quote_arg(const std::string& value) { return "\"" + value + "\""; } +#endif int run_claim_worker(const std::string& state_path, const std::string& key, const std::string& ready_path, const std::string& go_path, const std::string& result_path) { +#ifndef _WIN32 + // macOS 的 system() 持进程级互斥锁直到子进程退出,两个线程同时调用会 + // 串行化:第二个 worker 要等第一个结束才启动,而第一个在等 go,go 又在 + // 等两个 ready —— 只能 posix_spawn 直接起进程。 + std::vector args{ACECODE_STATE_FILE_CLAIM_WORKER_PATH, + state_path, key, ready_path, go_path, + result_path}; + std::vector argv; + for (auto& arg : args) argv.push_back(arg.data()); + argv.push_back(nullptr); + pid_t pid = 0; + if (posix_spawn(&pid, argv[0], nullptr, nullptr, argv.data(), environ) != 0) + return -1; + int status = 0; + if (waitpid(pid, &status, 0) != pid) return -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; +#else std::ostringstream command; -#ifdef _WIN32 command << "\""; -#endif command << quote_arg(ACECODE_STATE_FILE_CLAIM_WORKER_PATH) << " " << quote_arg(state_path) << " " @@ -56,10 +79,9 @@ int run_claim_worker(const std::string& state_path, << quote_arg(ready_path) << " " << quote_arg(go_path) << " " << quote_arg(result_path); -#ifdef _WIN32 command << "\""; -#endif return std::system(command.str().c_str()); +#endif } std::pair read_claim_result(const fs::path& path) { diff --git a/tests/web/remote_web_proxy_test.cpp b/tests/web/remote_web_proxy_test.cpp index fa71d861b..5f7934d47 100644 --- a/tests/web/remote_web_proxy_test.cpp +++ b/tests/web/remote_web_proxy_test.cpp @@ -163,7 +163,15 @@ void assert_round_trip(int proxy_port, EchoServer& echo) { for (int attempt = 0; attempt < 40 && echo.peer_address().empty(); ++attempt) { std::this_thread::sleep_for(std::chrono::milliseconds(10)); } +#ifdef __APPLE__ + // macOS 的 lo0 只分配 127.0.0.1,代理改用本机的非回环 IPv4 作源地址; + // 守护进程照样把它当成需要 token 的远程来源。 + const std::string peer = echo.peer_address(); + EXPECT_FALSE(peer.empty()); + EXPECT_NE(peer.rfind("127.", 0), 0u) << peer; +#else EXPECT_EQ(echo.peer_address(), "127.0.0.2"); +#endif } } // namespace diff --git a/web/scripts/i18n-en-overrides.mjs b/web/scripts/i18n-en-overrides.mjs index b0e18883d..00ffa7aed 100644 --- a/web/scripts/i18n-en-overrides.mjs +++ b/web/scripts/i18n-en-overrides.mjs @@ -2187,6 +2187,7 @@ export const ENGLISH_SOURCE_OVERRIDES = Object.freeze({ '模拟器画面已中断': 'Simulator stream interrupted', '模拟器键盘输入': 'Simulator keyboard input', '模拟器控制组件缺失，请重新安装完整的 ACECode 安装包': 'The simulator control component is missing. Reinstall the full ACECode package.', + '模拟器控制组件未在沙盒中运行：{{p0}}': 'The simulator control component is running without its sandbox: {{p0}}', '模拟器屏幕': 'Simulator screen', '模拟器设备': 'Simulator devices', '模拟器未启动': 'Simulator is not booted', diff --git a/web/src/components/IosSimulatorPanel.jsx b/web/src/components/IosSimulatorPanel.jsx index ff3c0248c..c94799347 100644 --- a/web/src/components/IosSimulatorPanel.jsx +++ b/web/src/components/IosSimulatorPanel.jsx @@ -31,6 +31,24 @@ import { touchGestureUp, wheelDragGesture, } from '../lib/iosSimulator.js'; +import { + IOS_SIMULATOR_VIEW_STATE_EVENT, + hasNativeIosSimulatorView, + nativeFrameFromState, + nativeSimulatorLayout, + nativeSimulatorLayoutSignature, + nativeViewRendering, + setNativeSimulatorLayout, + simulatorDeveloperDir, + unmountNativeSimulatorView, +} from '../lib/iosSimulatorNativeView.js'; +import { + NATIVE_SURFACE_OVERLAY_EVENT, + nativeSurfaceOcclusionRectsFromClientRects, + nativeSurfaceOverlayGeometryByDocument, + nativeSurfaceShouldShow, + nativeSurfaceViewportRect, +} from '../lib/agentBrowserSurfaceCoordinator.js'; import { iosSimulatorErrorMessage } from '../lib/iosSimulatorSettings.js'; import { requestOpenSetting } from '../lib/settingsNavigation.js'; import { AnchoredMenu } from './AnchoredMenu.jsx'; @@ -146,6 +164,7 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi const [menuOpen, setMenuOpen] = useState(false); const [capturing, setCapturing] = useState(false); const [reconnectNonce, setReconnectNonce] = useState(0); + const [nativeState, setNativeState] = useState(null); const mountedRef = useRef(true); const wsRef = useRef(null); @@ -164,6 +183,10 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi const autoPickedRef = useRef(false); const onSelectDeviceRef = useRef(onSelectDevice); onSelectDeviceRef.current = onSelectDevice; + const nativeOwnerRef = useRef({}); + const nativeLayoutRef = useRef(''); + const nativeFrameRef = useRef(null); + const nativeLiveRef = useRef(false); useEffect(() => { mountedRef.current = true; @@ -194,6 +217,11 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi }, [api, loadDevices]); const unsupported = status?.supported === false; + // Desktop (macOS) draws the screen natively; the canvas stays for input. + const developerDir = simulatorDeveloperDir(status); + const nativeAvailable = !!udid && !unsupported && !!developerDir && hasNativeIosSimulatorView(); + const nativeLive = nativeAvailable && nativeViewRendering(nativeState, udid); + nativeLiveRef.current = nativeLive; // 手动打开的选择页:会话已绑定设备或只有一台已启动设备时直接选中。 useEffect(() => { @@ -308,9 +336,16 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi token: getToken() || '', pageProtocol: location.protocol, pageHost: location.host, + frames: !nativeLiveRef.current, })); ws.binaryType = 'arraybuffer'; wsRef.current = ws; + // The native view may have gone live while this socket was connecting. + ws.onopen = () => { + if (!disposed && wsRef.current === ws && nativeLiveRef.current) { + ws.send(JSON.stringify({ type: 'frames', enabled: false })); + } + }; ws.onmessage = (event) => { if (disposed || wsRef.current !== ws) return; if (typeof event.data === 'string') { @@ -318,6 +353,13 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi if (message?.type === 'device') { deviceRef.current = message.device; setDevice(message.device); + // Without JPEG frames the native view is the picture: go live now. + if (nativeLiveRef.current && nativeFrameRef.current) { + attempt = 0; + frameRef.current = nativeFrameRef.current; + setFrameSize(nativeFrameRef.current); + setConn({ phase: 'live', message: '', retrySeconds: 0 }); + } } else if (message?.type === 'ended') { stop(ws); scheduleReconnect(message.reason || '模拟器画面已结束'); @@ -374,6 +416,42 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi return () => observer.disconnect(); }, [udid, unsupported]); + // Native view state comes from the shell (attach, rotation, failures). + useEffect(() => { + if (!nativeAvailable) return undefined; + const onState = (event) => setNativeState(event?.detail || null); + window.addEventListener(IOS_SIMULATOR_VIEW_STATE_EVENT, onState); + return () => window.removeEventListener(IOS_SIMULATOR_VIEW_STATE_EVENT, onState); + }, [nativeAvailable]); + + // While the native view draws, stop the JPEG stream and map input with the + // native framebuffer's size and rotation; fall back when it stops. + useEffect(() => { + const frame = nativeLive ? nativeFrameFromState(nativeState) : null; + nativeFrameRef.current = frame; + if (frame) { + frameRef.current = frame; + setFrameSize((prev) => (prev && prev.width === frame.width && prev.height === frame.height + && prev.rotation === frame.rotation ? prev : frame)); + } + }, [nativeLive, nativeState]); + + useEffect(() => { + send({ type: 'frames', enabled: !nativeLive }); + }, [nativeLive, send]); + + // One native view per shell: unmount on device change / reconnect / leave. + useEffect(() => { + if (!nativeAvailable) return undefined; + const owner = nativeOwnerRef.current; + return () => { + nativeLayoutRef.current = ''; + nativeFrameRef.current = null; + setNativeState(null); + void unmountNativeSimulatorView(owner); + }; + }, [nativeAvailable, udid, developerDir, reconnectNonce]); + const mapPoint = useCallback((event, clamp) => { const canvas = canvasRef.current; const frame = frameRef.current; @@ -562,6 +640,68 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi ? containedImageRect(stageSize.width, stageSize.height, frameSize.width, frameSize.height) : null), [frameSize, stageSize.height, stageSize.width]); const cornerRadius = displayRect ? deviceCornerRadius(device, displayRect.width, displayRect.height) : 0; + const streamLive = conn.phase === 'live' && !!frameSize; + + // Places the native view over the canvas whenever the canvas, overlays or + // viewport change. It shows only while the stream is live and no blocking + // dialog covers it; menus drawn over the screen are cut out of it. + useEffect(() => { + if (!nativeAvailable) return undefined; + let frameRequest = 0; + const sync = () => { + frameRequest = 0; + const rect = canvasRef.current?.getBoundingClientRect?.() || null; + const overlay = rect + ? nativeSurfaceOverlayGeometryByDocument(rect, document, window) + : { blocking: false, occlusionRects: [] }; + const visible = !!rect && streamLive && nativeSurfaceShouldShow({ + pageLive: true, + documentVisible: document.visibilityState !== 'hidden' && !document.hidden, + surfaceRect: rect, + viewportRect: nativeSurfaceViewportRect(window), + overlayBlocked: overlay.blocking, + }); + const layout = nativeSimulatorLayout({ + udid, + developerDir, + rect, + visible, + cornerRadius, + occlusionRects: visible ? nativeSurfaceOcclusionRectsFromClientRects(rect, overlay.occlusionRects, 1) : [], + }); + const signature = nativeSimulatorLayoutSignature(layout); + if (signature === nativeLayoutRef.current) return; + nativeLayoutRef.current = signature; + void setNativeSimulatorLayout(layout, nativeOwnerRef.current).then((result) => { + if (!mountedRef.current) return; + if (result?.ok) setNativeState(result); + else if (result?.error) setNativeState({ udid, rendering: false, error: result.error }); + }); + }; + const schedule = () => { + if (!frameRequest) frameRequest = requestAnimationFrame(sync); + }; + schedule(); + const mutationObserver = typeof MutationObserver === 'function' ? new MutationObserver(schedule) : null; + mutationObserver?.observe(document.body, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ['aria-hidden', 'class', 'data-ace-native-overlay', 'hidden', 'style'], + }); + window.addEventListener('resize', schedule); + window.addEventListener('scroll', schedule, true); + window.addEventListener(NATIVE_SURFACE_OVERLAY_EVENT, schedule); + document.addEventListener('visibilitychange', schedule); + return () => { + if (frameRequest) cancelAnimationFrame(frameRequest); + mutationObserver?.disconnect(); + window.removeEventListener('resize', schedule); + window.removeEventListener('scroll', schedule, true); + window.removeEventListener(NATIVE_SURFACE_OVERLAY_EVENT, schedule); + document.removeEventListener('visibilitychange', schedule); + }; + }, [cornerRadius, developerDir, displayRect, nativeAvailable, streamLive, udid]); if (unsupported) { return ( @@ -607,12 +747,12 @@ export function IosSimulatorPanel({ api, sessionId = '', udid = '', onSelectDevi ); } - const live = conn.phase === 'live' && !!frameSize; + const live = streamLive; const notBooted = !!listedDevice && !listedDevice.booted; return (
+ data-ios-simulator-phase={conn.phase} data-ios-simulator-native={nativeLive ? 'live' : undefined}>