android_ui: add scroll-to, narrow TODO

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WzxdWVUTaKFW8m7YrvWbzV

docs/superpowers/contracts/android-ui-scroll-to-2026-11-02.json

@@ -0,0 +1,11 @@
+{
+  "title": "android-ui scroll-to",
+  "objective": "AndroidUi.scroll_to and android-ui scroll-to find a widget outside the viewport, fail loudly at the end of the list, and keep ambiguity an error.",
+  "acceptance_criteria": [
+    "Visible element returns with no swipe",
+    "Off-screen element is reached by upward swipes",
+    "End of list or max swipes raises ElementNotFoundError naming the query"
+  ],
+  "out_of_scope": ["MCP wrapper", "Rollout to todo and dufs-cloud"],
+  "verifier": "pytest python_pkg/android_ui and pre-commit run --files <changed-files>"
+}

docs/superpowers/evidence/android-ui-scroll-to-2026-11-02.json

@@ -0,0 +1,25 @@
+{
+  "intent": "android-ui can bring a widget below the fold on screen with scroll-to.",
+  "scope": [
+    "python_pkg/android_ui: _scroll.py, driver.py, cli.py, README, tests",
+    "Non-goal: MCP wrapper and rollout to other repos"
+  ],
+  "changes": [
+    "Added ScrollMixin.scroll_to (swipe up until the query matches once)",
+    "Added the scroll-to CLI subcommand and narrowed the TODO"
+  ],
+  "verification": [
+    {
+      "command": "pytest python_pkg/android_ui --cov=python_pkg.android_ui",
+      "result": "pass",
+      "evidence": "95 passed, every android_ui module 100% covered"
+    },
+    {
+      "command": "pre-commit run --files <changed-files>",
+      "result": "pass",
+      "evidence": "all hooks pass"
+    }
+  ],
+  "risks": ["Swipe distance is tuned for a 1080x2400 screen only by fraction"],
+  "rollback": ["git revert the commit", "Re-run pytest python_pkg/android_ui"]
+}

python_pkg/android_ui/README.md

@@ -85,8 +85,9 @@ ui.wait_for("Connected.", timeout=30)
     `SemanticsNode`, while this tool reads Android's `AccessibilityNodeInfo`.
     Nine widget tests passed green on the build whose on-device labels had
     just gone empty.
-- Scrolling is not yet modelled: a widget outside the viewport is not in the
-  tree. Scroll with `adb shell input swipe` first, or add `scroll_to()`.
+- A widget outside the viewport is not in the tree. `android-ui scroll-to
+  QUERY` (`AndroidUi.scroll_to`) swipes up until it appears, and stops with an
+  error at the end of the list or after 8 swipes. It only scrolls downward.
 
 ## One phone, many sessions
 

python_pkg/android_ui/TODO-android-ui-mcp-wrapper.md

@@ -1,13 +1,12 @@
 # Task: replace blind coordinate tapping with element-targeted Android UI automation
 
-> **STATUS 2026-08-10: BUILT.** The tool now exists at
-> `~/src/testsAndMisc/python_pkg/android_ui/` (see its README). It covers `dump` /
-> `find` / `tap` / `type` / `wait` / `focus` with verified typing, ambiguity as
-> an error, keyboard detection, and partial-tree retries — all exercised
-> against the physical Pixel 6a. What remains from the spec below is the
-> optional MCP wrapper, `scroll_to()`, and rolling it out to `~/src/todo` and
-> `~/src/dufs-cloud/app`. Everything after this line is the original brief, kept
-> because it records WHY each guarantee exists.
+> **STATUS 2026-11-02: PARTIAL.** The CLI (`dump`/`find`/`tap`/`type`/`wait`/
+> `scroll-to`/`focus`) exists at `~/src/testsAndMisc/python_pkg/android_ui/` (see
+> its README). Remaining: (1) the optional MCP wrapper over the same driver;
+> (2) rolling it out to `~/src/todo` and `~/src/dufs-cloud/app` (other repos,
+> needs their own sessions); (3) the `## done` run on the physical Pixel 6a:
+> needs kuhy, phone not connected. Everything after this line is the original
+> brief, kept because it records WHY each guarantee exists.
 
 ## what
 

python_pkg/android_ui/_scroll.py

@@ -0,0 +1,72 @@
+"""``scroll_to``: bring a widget that is outside the viewport on screen."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import time
+
+from python_pkg.android_ui._elements import ElementNotFoundError, UiElement
+
+_SWIPE_MS = "350"
+# Swipe through the middle 40% of the screen: far enough to move a page, short
+# enough that Flutter does not turn it into a fling that overshoots the target.
+_FROM_FRAC = 0.7
+_TO_FRAC = 0.3
+
+
+def _snapshot(elements: list[UiElement]) -> list[tuple[str, tuple[int, ...]]]:
+    """Reduce a tree to what changes when a list scrolls."""
+    return [(e.label, e.bounds) for e in elements]
+
+
+class ScrollMixin(ABC):
+    """The scrolling part of :class:`~python_pkg.android_ui.driver.AndroidUi`."""
+
+    _settle: float
+
+    @abstractmethod
+    def _input(self, *args: str) -> str:
+        """Supplied by the concrete driver: ``input`` on the driven display."""
+
+    @abstractmethod
+    def dump(self, *, retries: int = 4) -> list[UiElement]:
+        """Supplied by the concrete driver: read the current UI tree."""
+
+    @abstractmethod
+    def find(self, query: str, *, exact: bool = False) -> UiElement:
+        """Supplied by the concrete driver: the single match, or raise."""
+
+    def scroll_to(
+        self, query: str, *, exact: bool = False, max_swipes: int = 8
+    ) -> UiElement:
+        """Swipe up until ``query`` matches exactly one element, then return it.
+
+        A widget outside the viewport is not in the accessibility tree, so
+        ``find`` reports it missing. Stops early, naming the query, when a swipe
+        leaves the screen unchanged (the end of the list) rather than spinning
+        for ``max_swipes`` swipes. Ambiguity raises, as it does for ``find``.
+        """
+        elements = self.dump()
+        for _ in range(max_swipes):
+            if any(e.matches(query, exact=exact) for e in elements):
+                return self.find(query, exact=exact)
+            width = max((e.bounds[2] for e in elements), default=0)
+            height = max((e.bounds[3] for e in elements), default=0)
+            x = str(width // 2)
+            self._input(
+                "swipe",
+                x,
+                str(int(height * _FROM_FRAC)),
+                x,
+                str(int(height * _TO_FRAC)),
+                _SWIPE_MS,
+            )
+            time.sleep(self._settle)
+            after = self.dump()
+            if _snapshot(after) == _snapshot(elements):
+                break
+            elements = after
+        if any(e.matches(query, exact=exact) for e in elements):
+            return self.find(query, exact=exact)
+        msg = f"scrolled looking for {query!r} but it never appeared"
+        raise ElementNotFoundError(msg)

python_pkg/android_ui/cli.py

@@ -9,6 +9,7 @@ successful no-op instead of guessing from a screenshot.
     android-ui tap "Connect Firebase"
     android-ui type "Sync account email" kuhy@example.com
     android-ui wait "Connected to Firebase." --timeout 30
+    android-ui scroll-to "Offline backup"    # widget below the fold
     android-ui --display self tap "Save"     # this session's phone_vd display
 """
 
@@ -63,6 +64,7 @@ def _build_parser() -> argparse.ArgumentParser:
         ("find", "locate an element without touching it"),
         ("tap", "tap the element matching QUERY"),
         ("wait", "poll until QUERY appears"),
+        ("scroll-to", "swipe up until QUERY is on screen"),
     ):
         cmd = sub.add_parser(name, help=helptext)
         cmd.add_argument("query")
@@ -135,6 +137,8 @@ def _run(ui: AndroidUi, args: argparse.Namespace) -> None:
     elif args.command == "wait":
         found = ui.wait_for(args.query, timeout=args.timeout, exact=args.exact)
         sys.stdout.write(f"{found}\n")
+    elif args.command == "scroll-to":
+        sys.stdout.write(f"{ui.scroll_to(args.query, exact=args.exact)}\n")
     elif args.command == "type":
         ui.type_into(args.query, args.text, exact=args.exact, timeout=args.timeout)
         sys.stdout.write(f"typed into {args.query!r} and verified\n")

python_pkg/android_ui/driver.py

@@ -17,6 +17,7 @@ from python_pkg.android_ui._elements import (
     _escape,
     _parse_tree,
 )
+from python_pkg.android_ui._scroll import ScrollMixin
 from python_pkg.android_ui._text_entry import TextEntryMixin
 
 _logger = logging.getLogger(__name__)
@@ -30,7 +31,7 @@ _REMOTE_DUMP = "/sdcard/window_dump.xml"
 _SUSPICIOUSLY_SMALL_TREE = 4
 
 
-class AndroidUi(TextEntryMixin):
+class AndroidUi(ScrollMixin, TextEntryMixin):
     """Drives one device. Every action re-reads the tree before acting."""
 
     def __init__(

python_pkg/android_ui/tests/test_cli.py

@@ -140,6 +140,19 @@ class TestWait:
         assert cli.main(["wait", "Nope"]) == 1
 
 
+class TestScrollTo:
+    def test_prints_the_element_once_scrolled_into_view(
+        self, ui: MagicMock, capsys: pytest.CaptureFixture[str]
+    ) -> None:
+        ui.scroll_to.return_value = _element("Offline backup")
+        assert cli.main(["scroll-to", "Offline backup"]) == 0
+        assert "Offline backup" in capsys.readouterr().out
+
+    def test_never_found_exits_non_zero(self, ui: MagicMock) -> None:
+        ui.scroll_to.side_effect = ElementNotFoundError("never appeared")
+        assert cli.main(["scroll-to", "Nope"]) == 1
+
+
 class TestType:
     def test_confirms_the_text_was_verified(
         self, ui: MagicMock, capsys: pytest.CaptureFixture[str]

python_pkg/android_ui/tests/test_scroll.py

@@ -0,0 +1,86 @@
+"""Tests for ``AndroidUi.scroll_to``: find a widget outside the viewport."""
+
+from __future__ import annotations
+
+import pytest
+
+from python_pkg.android_ui import driver as drv
+from python_pkg.android_ui._elements import (
+    AmbiguousElementError,
+    ElementNotFoundError,
+)
+from python_pkg.android_ui.tests.conftest import FakeDevice
+from python_pkg.android_ui.tests.conftest import node as _node
+from python_pkg.android_ui.tests.conftest import tree as _tree
+
+MOD = "python_pkg.android_ui.driver"
+_SCREEN = "[0,0][1080,2400]"
+
+
+def _screen(*labels: str) -> str:
+    """A screen of five filler rows (so the dump is not 'partial') plus labels."""
+    rows = [_node("Window", bounds=_SCREEN)]
+    rows += [_node(f"row{i}", bounds="[0,100][1080,200]") for i in range(4)]
+    rows += [_node(label, bounds="[0,1000][1080,1100]") for label in labels]
+    return _tree(*rows)
+
+
+def _driver(
+    monkeypatch: pytest.MonkeyPatch, trees: list[str]
+) -> tuple[drv.AndroidUi, FakeDevice]:
+    device = FakeDevice(trees)
+    instance = drv.AndroidUi(settle_seconds=0.0)
+    monkeypatch.setattr(instance, "_run", device.run)
+    monkeypatch.setattr(f"{MOD}.time.sleep", lambda _s: None)
+    return instance, device
+
+
+def _swipes(device: FakeDevice) -> list[tuple[str, ...]]:
+    return [c for c in device.calls if c[4:5] == ("swipe",)]
+
+
+def test_visible_element_needs_no_swipe(monkeypatch: pytest.MonkeyPatch) -> None:
+    ui, device = _driver(monkeypatch, [_screen("Backup")])
+    assert ui.scroll_to("Backup").label == "Backup"
+    assert _swipes(device) == []
+
+
+def test_swipes_up_until_the_element_appears(monkeypatch: pytest.MonkeyPatch) -> None:
+    ui, device = _driver(
+        monkeypatch, [_screen("A"), _screen("B"), _screen("B", "Backup")]
+    )
+    assert ui.scroll_to("Backup").label == "Backup"
+    swipes = _swipes(device)
+    assert len(swipes) == 2
+    # finger travels upward through the middle of the screen
+    x1, y1, x2, y2 = (int(v) for v in swipes[0][5:9])
+    assert x1 == x2 == 540
+    assert y1 > y2
+    assert 0 < y2 < y1 < 2400
+
+
+def test_stops_at_the_end_of_the_list(monkeypatch: pytest.MonkeyPatch) -> None:
+    # The screen stops changing once the list bottoms out: say so, don't spin.
+    ui, device = _driver(monkeypatch, [_screen("A")])
+    with pytest.raises(ElementNotFoundError, match="Backup"):
+        ui.scroll_to("Backup", max_swipes=10)
+    assert len(_swipes(device)) == 1
+
+
+def test_gives_up_after_max_swipes(monkeypatch: pytest.MonkeyPatch) -> None:
+    trees = [_screen(f"item{i}") for i in range(10)]
+    ui, device = _driver(monkeypatch, trees)
+    with pytest.raises(ElementNotFoundError, match="Backup"):
+        ui.scroll_to("Backup", max_swipes=3)
+    assert len(_swipes(device)) == 3
+
+
+def test_ambiguity_is_still_an_error(monkeypatch: pytest.MonkeyPatch) -> None:
+    ui, _ = _driver(monkeypatch, [_screen("Backup now", "Backup later")])
+    with pytest.raises(AmbiguousElementError):
+        ui.scroll_to("Backup")
+
+
+def test_the_last_allowed_swipe_may_reveal_it(monkeypatch: pytest.MonkeyPatch) -> None:
+    ui, _ = _driver(monkeypatch, [_screen("A"), _screen("Backup")])
+    assert ui.scroll_to("Backup", max_swipes=1).label == "Backup"