diff --git a/docs/superpowers/contracts/android-ui-scroll-to-2026-11-02.json b/docs/superpowers/contracts/android-ui-scroll-to-2026-11-02.json new file mode 100644 index 00000000..220ddd2c --- /dev/null +++ b/docs/superpowers/contracts/android-ui-scroll-to-2026-11-02.json @@ -0,0 +1,11 @@ +{ + "title": "android-ui scroll-to", + "objective": "AndroidUi.scroll_to and android-ui scroll-to find a widget outside the viewport, fail loudly at the end of the list, and keep ambiguity an error.", + "acceptance_criteria": [ + "Visible element returns with no swipe", + "Off-screen element is reached by upward swipes", + "End of list or max swipes raises ElementNotFoundError naming the query" + ], + "out_of_scope": ["MCP wrapper", "Rollout to todo and dufs-cloud"], + "verifier": "pytest python_pkg/android_ui and pre-commit run --files " +} diff --git a/docs/superpowers/evidence/android-ui-scroll-to-2026-11-02.json b/docs/superpowers/evidence/android-ui-scroll-to-2026-11-02.json new file mode 100644 index 00000000..d5677e4a --- /dev/null +++ b/docs/superpowers/evidence/android-ui-scroll-to-2026-11-02.json @@ -0,0 +1,25 @@ +{ + "intent": "android-ui can bring a widget below the fold on screen with scroll-to.", + "scope": [ + "python_pkg/android_ui: _scroll.py, driver.py, cli.py, README, tests", + "Non-goal: MCP wrapper and rollout to other repos" + ], + "changes": [ + "Added ScrollMixin.scroll_to (swipe up until the query matches once)", + "Added the scroll-to CLI subcommand and narrowed the TODO" + ], + "verification": [ + { + "command": "pytest python_pkg/android_ui --cov=python_pkg.android_ui", + "result": "pass", + "evidence": "95 passed, every android_ui module 100% covered" + }, + { + "command": "pre-commit run --files ", + "result": "pass", + "evidence": "all hooks pass" + } + ], + "risks": ["Swipe distance is tuned for a 1080x2400 screen only by fraction"], + "rollback": ["git revert the commit", "Re-run pytest python_pkg/android_ui"] +} diff --git a/python_pkg/android_ui/README.md b/python_pkg/android_ui/README.md index bc1df3d6..2ba219e9 100644 --- a/python_pkg/android_ui/README.md +++ b/python_pkg/android_ui/README.md @@ -85,8 +85,9 @@ ui.wait_for("Connected.", timeout=30) `SemanticsNode`, while this tool reads Android's `AccessibilityNodeInfo`. Nine widget tests passed green on the build whose on-device labels had just gone empty. -- Scrolling is not yet modelled: a widget outside the viewport is not in the - tree. Scroll with `adb shell input swipe` first, or add `scroll_to()`. +- A widget outside the viewport is not in the tree. `android-ui scroll-to + QUERY` (`AndroidUi.scroll_to`) swipes up until it appears, and stops with an + error at the end of the list or after 8 swipes. It only scrolls downward. ## One phone, many sessions diff --git a/python_pkg/android_ui/TODO-android-ui-mcp-wrapper.md b/python_pkg/android_ui/TODO-android-ui-mcp-wrapper.md index 9d3e9e0c..db9ab4f8 100644 --- a/python_pkg/android_ui/TODO-android-ui-mcp-wrapper.md +++ b/python_pkg/android_ui/TODO-android-ui-mcp-wrapper.md @@ -1,13 +1,12 @@ # Task: replace blind coordinate tapping with element-targeted Android UI automation -> **STATUS 2026-08-10: BUILT.** The tool now exists at -> `~/src/testsAndMisc/python_pkg/android_ui/` (see its README). It covers `dump` / -> `find` / `tap` / `type` / `wait` / `focus` with verified typing, ambiguity as -> an error, keyboard detection, and partial-tree retries — all exercised -> against the physical Pixel 6a. What remains from the spec below is the -> optional MCP wrapper, `scroll_to()`, and rolling it out to `~/src/todo` and -> `~/src/dufs-cloud/app`. Everything after this line is the original brief, kept -> because it records WHY each guarantee exists. +> **STATUS 2026-11-02: PARTIAL.** The CLI (`dump`/`find`/`tap`/`type`/`wait`/ +> `scroll-to`/`focus`) exists at `~/src/testsAndMisc/python_pkg/android_ui/` (see +> its README). Remaining: (1) the optional MCP wrapper over the same driver; +> (2) rolling it out to `~/src/todo` and `~/src/dufs-cloud/app` (other repos, +> needs their own sessions); (3) the `## done` run on the physical Pixel 6a: +> needs kuhy, phone not connected. Everything after this line is the original +> brief, kept because it records WHY each guarantee exists. ## what diff --git a/python_pkg/android_ui/_scroll.py b/python_pkg/android_ui/_scroll.py new file mode 100644 index 00000000..8f5521cd --- /dev/null +++ b/python_pkg/android_ui/_scroll.py @@ -0,0 +1,72 @@ +"""``scroll_to``: bring a widget that is outside the viewport on screen.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +import time + +from python_pkg.android_ui._elements import ElementNotFoundError, UiElement + +_SWIPE_MS = "350" +# Swipe through the middle 40% of the screen: far enough to move a page, short +# enough that Flutter does not turn it into a fling that overshoots the target. +_FROM_FRAC = 0.7 +_TO_FRAC = 0.3 + + +def _snapshot(elements: list[UiElement]) -> list[tuple[str, tuple[int, ...]]]: + """Reduce a tree to what changes when a list scrolls.""" + return [(e.label, e.bounds) for e in elements] + + +class ScrollMixin(ABC): + """The scrolling part of :class:`~python_pkg.android_ui.driver.AndroidUi`.""" + + _settle: float + + @abstractmethod + def _input(self, *args: str) -> str: + """Supplied by the concrete driver: ``input`` on the driven display.""" + + @abstractmethod + def dump(self, *, retries: int = 4) -> list[UiElement]: + """Supplied by the concrete driver: read the current UI tree.""" + + @abstractmethod + def find(self, query: str, *, exact: bool = False) -> UiElement: + """Supplied by the concrete driver: the single match, or raise.""" + + def scroll_to( + self, query: str, *, exact: bool = False, max_swipes: int = 8 + ) -> UiElement: + """Swipe up until ``query`` matches exactly one element, then return it. + + A widget outside the viewport is not in the accessibility tree, so + ``find`` reports it missing. Stops early, naming the query, when a swipe + leaves the screen unchanged (the end of the list) rather than spinning + for ``max_swipes`` swipes. Ambiguity raises, as it does for ``find``. + """ + elements = self.dump() + for _ in range(max_swipes): + if any(e.matches(query, exact=exact) for e in elements): + return self.find(query, exact=exact) + width = max((e.bounds[2] for e in elements), default=0) + height = max((e.bounds[3] for e in elements), default=0) + x = str(width // 2) + self._input( + "swipe", + x, + str(int(height * _FROM_FRAC)), + x, + str(int(height * _TO_FRAC)), + _SWIPE_MS, + ) + time.sleep(self._settle) + after = self.dump() + if _snapshot(after) == _snapshot(elements): + break + elements = after + if any(e.matches(query, exact=exact) for e in elements): + return self.find(query, exact=exact) + msg = f"scrolled looking for {query!r} but it never appeared" + raise ElementNotFoundError(msg) diff --git a/python_pkg/android_ui/cli.py b/python_pkg/android_ui/cli.py index a13ad3ca..e34d5c97 100644 --- a/python_pkg/android_ui/cli.py +++ b/python_pkg/android_ui/cli.py @@ -9,6 +9,7 @@ successful no-op instead of guessing from a screenshot. android-ui tap "Connect Firebase" android-ui type "Sync account email" kuhy@example.com android-ui wait "Connected to Firebase." --timeout 30 + android-ui scroll-to "Offline backup" # widget below the fold android-ui --display self tap "Save" # this session's phone_vd display """ @@ -63,6 +64,7 @@ def _build_parser() -> argparse.ArgumentParser: ("find", "locate an element without touching it"), ("tap", "tap the element matching QUERY"), ("wait", "poll until QUERY appears"), + ("scroll-to", "swipe up until QUERY is on screen"), ): cmd = sub.add_parser(name, help=helptext) cmd.add_argument("query") @@ -135,6 +137,8 @@ def _run(ui: AndroidUi, args: argparse.Namespace) -> None: elif args.command == "wait": found = ui.wait_for(args.query, timeout=args.timeout, exact=args.exact) sys.stdout.write(f"{found}\n") + elif args.command == "scroll-to": + sys.stdout.write(f"{ui.scroll_to(args.query, exact=args.exact)}\n") elif args.command == "type": ui.type_into(args.query, args.text, exact=args.exact, timeout=args.timeout) sys.stdout.write(f"typed into {args.query!r} and verified\n") diff --git a/python_pkg/android_ui/driver.py b/python_pkg/android_ui/driver.py index 10b58709..acf45eaa 100644 --- a/python_pkg/android_ui/driver.py +++ b/python_pkg/android_ui/driver.py @@ -17,6 +17,7 @@ from python_pkg.android_ui._elements import ( _escape, _parse_tree, ) +from python_pkg.android_ui._scroll import ScrollMixin from python_pkg.android_ui._text_entry import TextEntryMixin _logger = logging.getLogger(__name__) @@ -30,7 +31,7 @@ _REMOTE_DUMP = "/sdcard/window_dump.xml" _SUSPICIOUSLY_SMALL_TREE = 4 -class AndroidUi(TextEntryMixin): +class AndroidUi(ScrollMixin, TextEntryMixin): """Drives one device. Every action re-reads the tree before acting.""" def __init__( diff --git a/python_pkg/android_ui/tests/test_cli.py b/python_pkg/android_ui/tests/test_cli.py index fb5e5467..183328bd 100644 --- a/python_pkg/android_ui/tests/test_cli.py +++ b/python_pkg/android_ui/tests/test_cli.py @@ -140,6 +140,19 @@ class TestWait: assert cli.main(["wait", "Nope"]) == 1 +class TestScrollTo: + def test_prints_the_element_once_scrolled_into_view( + self, ui: MagicMock, capsys: pytest.CaptureFixture[str] + ) -> None: + ui.scroll_to.return_value = _element("Offline backup") + assert cli.main(["scroll-to", "Offline backup"]) == 0 + assert "Offline backup" in capsys.readouterr().out + + def test_never_found_exits_non_zero(self, ui: MagicMock) -> None: + ui.scroll_to.side_effect = ElementNotFoundError("never appeared") + assert cli.main(["scroll-to", "Nope"]) == 1 + + class TestType: def test_confirms_the_text_was_verified( self, ui: MagicMock, capsys: pytest.CaptureFixture[str] diff --git a/python_pkg/android_ui/tests/test_scroll.py b/python_pkg/android_ui/tests/test_scroll.py new file mode 100644 index 00000000..9fa259d5 --- /dev/null +++ b/python_pkg/android_ui/tests/test_scroll.py @@ -0,0 +1,86 @@ +"""Tests for ``AndroidUi.scroll_to``: find a widget outside the viewport.""" + +from __future__ import annotations + +import pytest + +from python_pkg.android_ui import driver as drv +from python_pkg.android_ui._elements import ( + AmbiguousElementError, + ElementNotFoundError, +) +from python_pkg.android_ui.tests.conftest import FakeDevice +from python_pkg.android_ui.tests.conftest import node as _node +from python_pkg.android_ui.tests.conftest import tree as _tree + +MOD = "python_pkg.android_ui.driver" +_SCREEN = "[0,0][1080,2400]" + + +def _screen(*labels: str) -> str: + """A screen of five filler rows (so the dump is not 'partial') plus labels.""" + rows = [_node("Window", bounds=_SCREEN)] + rows += [_node(f"row{i}", bounds="[0,100][1080,200]") for i in range(4)] + rows += [_node(label, bounds="[0,1000][1080,1100]") for label in labels] + return _tree(*rows) + + +def _driver( + monkeypatch: pytest.MonkeyPatch, trees: list[str] +) -> tuple[drv.AndroidUi, FakeDevice]: + device = FakeDevice(trees) + instance = drv.AndroidUi(settle_seconds=0.0) + monkeypatch.setattr(instance, "_run", device.run) + monkeypatch.setattr(f"{MOD}.time.sleep", lambda _s: None) + return instance, device + + +def _swipes(device: FakeDevice) -> list[tuple[str, ...]]: + return [c for c in device.calls if c[4:5] == ("swipe",)] + + +def test_visible_element_needs_no_swipe(monkeypatch: pytest.MonkeyPatch) -> None: + ui, device = _driver(monkeypatch, [_screen("Backup")]) + assert ui.scroll_to("Backup").label == "Backup" + assert _swipes(device) == [] + + +def test_swipes_up_until_the_element_appears(monkeypatch: pytest.MonkeyPatch) -> None: + ui, device = _driver( + monkeypatch, [_screen("A"), _screen("B"), _screen("B", "Backup")] + ) + assert ui.scroll_to("Backup").label == "Backup" + swipes = _swipes(device) + assert len(swipes) == 2 + # finger travels upward through the middle of the screen + x1, y1, x2, y2 = (int(v) for v in swipes[0][5:9]) + assert x1 == x2 == 540 + assert y1 > y2 + assert 0 < y2 < y1 < 2400 + + +def test_stops_at_the_end_of_the_list(monkeypatch: pytest.MonkeyPatch) -> None: + # The screen stops changing once the list bottoms out: say so, don't spin. + ui, device = _driver(monkeypatch, [_screen("A")]) + with pytest.raises(ElementNotFoundError, match="Backup"): + ui.scroll_to("Backup", max_swipes=10) + assert len(_swipes(device)) == 1 + + +def test_gives_up_after_max_swipes(monkeypatch: pytest.MonkeyPatch) -> None: + trees = [_screen(f"item{i}") for i in range(10)] + ui, device = _driver(monkeypatch, trees) + with pytest.raises(ElementNotFoundError, match="Backup"): + ui.scroll_to("Backup", max_swipes=3) + assert len(_swipes(device)) == 3 + + +def test_ambiguity_is_still_an_error(monkeypatch: pytest.MonkeyPatch) -> None: + ui, _ = _driver(monkeypatch, [_screen("Backup now", "Backup later")]) + with pytest.raises(AmbiguousElementError): + ui.scroll_to("Backup") + + +def test_the_last_allowed_swipe_may_reveal_it(monkeypatch: pytest.MonkeyPatch) -> None: + ui, _ = _driver(monkeypatch, [_screen("A"), _screen("Backup")]) + assert ui.scroll_to("Backup", max_swipes=1).label == "Backup"