///
/**
* Content script for Browser CLI Controller
* Optimized API for LLM agents with limited context windows
*
* @typedef {'css' | 'text' | 'aria-label' | 'placeholder'} SelectorType
*
* @typedef {object} ConsoleLog
* @property {string} type - Log type (log, error, warn, info, debug)
* @property {string} message - Log message
* @property {string} timestamp - ISO timestamp
*
* @typedef {object} ElementData
* @property {number} ref - Element reference number
* @property {string} role - Element role (button, link, input, etc.)
* @property {string} name - Accessible name
* @property {string} [value] - Current value (for inputs)
* @property {string[]} attrs - Attributes like [clickable], [disabled]
* @property {Element} domElement - Reference to DOM element
*/
/** @type {ConsoleLog[]} Store console logs */
const consoleLogs = [];
/** @type {number} */
const MAX_CONSOLE_LOGS = 1000;
/** @type {HTMLDivElement|undefined} Virtual cursor element */
let virtualCursor;
/** @type {HTMLInputElement|HTMLTextAreaElement|undefined} Last focused input element */
let lastFocusedInput;
/** @type {Map} Current ref to element mapping */
let currentRefMap = new Map();
/** @type {Snapshot|undefined} Last snapshot for diffing */
let lastSnapshot;
// ============================================================================
// Snapshot and Element Classes
// ============================================================================
/**
* Element wrapper with LLM-friendly toString
*/
class SnapshotElement {
/**
* @param {ElementData} data
*/
constructor(data) {
/** @type {number} */
this.ref = data.ref;
/** @type {string} */
this.role = data.role;
/** @type {string} */
this.name = data.name;
/** @type {string|undefined} */
this.value = data.value;
/** @type {string[]} */
this.attrs = data.attrs;
/** @type {Element} */
this.domElement = data.domElement;
}
/**
* LLM-friendly string representation
* @returns {string}
*/
toString() {
let s = `[${this.ref}] ${this.role}`;
if (this.name) s += ` "${this.name}"`;
if (this.attrs.length > 0) s += ` [${this.attrs.join(", ")}]`;
if (this.value !== undefined) s += ` value="${this.value}"`;
return s;
}
/**
* JSON representation (without DOM element)
* @returns {object}
*/
toJSON() {
return {
ref: this.ref,
role: this.role,
name: this.name,
value: this.value,
attrs: this.attrs,
};
}
}
/**
* Snapshot class with filtering and LLM-friendly output
*/
class Snapshot {
/**
* @param {SnapshotElement[]} elements
* @param {string} url
* @param {string} title
*/
constructor(elements, url, title) {
/** @type {SnapshotElement[]} */
this.elements = elements;
/** @type {string} */
this.url = url;
/** @type {string} */
this.title = title;
/** @type {Map} */
this._refMap = new Map();
for (const el of elements) {
this._refMap.set(el.ref, el);
}
}
/**
* Find element by ref
* @param {number} ref
* @returns {SnapshotElement|undefined}
*/
find(ref) {
return this._refMap.get(ref);
}
/**
* Get all buttons
* @returns {SnapshotElement[]}
*/
get buttons() {
return this.elements.filter((e) => e.role === "button");
}
/**
* Get all links
* @returns {SnapshotElement[]}
*/
get links() {
return this.elements.filter((e) => e.role === "link");
}
/**
* Get all inputs
* @returns {SnapshotElement[]}
*/
get inputs() {
return this.elements.filter(
(e) =>
["input", "textarea", "select"].includes(e.role) ||
e.role.startsWith("input["),
);
}
/**
* Get all form elements
* @returns {SnapshotElement[]}
*/
get forms() {
return this.elements.filter(
(e) =>
["input", "textarea", "select", "checkbox", "radio"].includes(e.role) ||
e.role.startsWith("input["),
);
}
/**
* Search by text
* @param {string} text
* @returns {SnapshotElement[]}
*/
search(text) {
const lower = text.toLowerCase();
return this.elements.filter(
(e) =>
e.name?.toLowerCase().includes(lower) ||
e.value?.toLowerCase().includes(lower),
);
}
/**
* Compare with another snapshot and return differences
* @param {Snapshot} other - Previous snapshot to compare against
* @returns {SnapshotDiff}
*/
diff(other) {
return new SnapshotDiff(other, this);
}
/**
* LLM-friendly output
* @returns {string}
*/
toString() {
let out = `Page: ${this.title}\nURL: ${this.url}\n\n`;
for (const el of this.elements) {
out += el.toString() + "\n";
}
return out;
}
/**
* JSON representation
* @returns {object}
*/
toJSON() {
return {
url: this.url,
title: this.title,
elements: this.elements.map((e) => e.toJSON()),
};
}
}
/**
* Diff between two snapshots
*/
class SnapshotDiff {
/**
* @param {Snapshot} before
* @param {Snapshot} after
*/
constructor(before, after) {
/** @type {boolean} */
this.urlChanged = before.url !== after.url;
/** @type {string|undefined} */
this.oldUrl = this.urlChanged ? before.url : undefined;
/** @type {string|undefined} */
this.newUrl = this.urlChanged ? after.url : undefined;
/** @type {boolean} */
this.titleChanged = before.title !== after.title;
/** @type {string|undefined} */
this.oldTitle = this.titleChanged ? before.title : undefined;
/** @type {string|undefined} */
this.newTitle = this.titleChanged ? after.title : undefined;
// Build maps by element signature (role + name) for comparison
/** @type {Map} */
const beforeMap = new Map();
for (const el of before.elements) {
const key = `${el.role}|${el.name}`;
beforeMap.set(key, el);
}
/** @type {Map} */
const afterMap = new Map();
for (const el of after.elements) {
const key = `${el.role}|${el.name}`;
afterMap.set(key, el);
}
/** @type {SnapshotElement[]} */
this.added = [];
/** @type {SnapshotElement[]} */
this.removed = [];
/** @type {Array<{element: SnapshotElement, changes: string[]}>} */
this.changed = [];
// Find added and changed elements
for (const [key, el] of afterMap) {
const beforeEl = beforeMap.get(key);
if (beforeEl) {
// Check for changes
const changes = [];
if (beforeEl.value !== el.value) {
changes.push(
`value: "${beforeEl.value ?? ""}" → "${el.value ?? ""}"`,
);
}
const beforeAttrs = beforeEl.attrs.join(",");
const afterAttrs = el.attrs.join(",");
if (beforeAttrs !== afterAttrs) {
changes.push(`attrs: [${beforeAttrs}] → [${afterAttrs}]`);
}
if (changes.length > 0) {
this.changed.push({ element: el, changes });
}
} else {
this.added.push(el);
}
}
// Find removed elements
for (const [key, el] of beforeMap) {
if (!afterMap.has(key)) {
this.removed.push(el);
}
}
}
/**
* Check if anything changed
* @returns {boolean}
*/
get hasChanges() {
return (
this.urlChanged ||
this.titleChanged ||
this.added.length > 0 ||
this.removed.length > 0 ||
this.changed.length > 0
);
}
/**
* LLM-friendly output
* @returns {string}
*/
toString() {
if (!this.hasChanges) {
return "No changes";
}
const lines = [];
if (this.urlChanged) {
lines.push(`URL: ${this.oldUrl} → ${this.newUrl}`);
}
if (this.titleChanged) {
lines.push(`Title: "${this.oldTitle}" → "${this.newTitle}"`);
}
if (this.added.length > 0) {
lines.push("", `Added (${this.added.length}):`);
for (const el of this.added) {
lines.push(` + ${el.toString()}`);
}
}
if (this.removed.length > 0) {
lines.push("", `Removed (${this.removed.length}):`);
for (const el of this.removed) {
lines.push(` - ${el.toString()}`);
}
}
if (this.changed.length > 0) {
lines.push("", `Changed (${this.changed.length}):`);
for (const { element, changes } of this.changed) {
lines.push(
` ~ ${element.toString()}`,
...changes.map((c) => ` ${c}`),
);
}
}
return lines.join("\n");
}
/**
* JSON representation
* @returns {object}
*/
toJSON() {
return {
urlChanged: this.urlChanged,
oldUrl: this.oldUrl,
newUrl: this.newUrl,
titleChanged: this.titleChanged,
oldTitle: this.oldTitle,
newTitle: this.newTitle,
added: this.added.map((e) => e.toJSON()),
removed: this.removed.map((e) => e.toJSON()),
changed: this.changed.map(({ element, changes }) => ({
element: element.toJSON(),
changes,
})),
};
}
}
// ============================================================================
// Virtual Cursor
// ============================================================================
/**
* Create and initialize the virtual cursor
*/
function initializeVirtualCursor() {
if (virtualCursor) return;
virtualCursor = document.createElement("div");
virtualCursor.id = "browser-cli-cursor";
virtualCursor.style.cssText = `
position: fixed;
width: 20px;
height: 20px;
pointer-events: none;
z-index: 999999;
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
opacity: 1;
`;
const inner = document.createElement("div");
inner.style.cssText = `
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 8px;
height: 8px;
background: #ff6b6b;
border-radius: 50%;
box-shadow: 0 0 10px rgba(255, 107, 107, 0.5);
`;
const outer = document.createElement("div");
outer.style.cssText = `
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 20px;
height: 20px;
border: 2px solid #ff6b6b;
border-radius: 50%;
animation: browser-cli-pulse 1.5s ease-out infinite;
`;
const style = document.createElement("style");
style.textContent = `
@keyframes browser-cli-pulse {
0% { transform: translate(-50%, -50%) scale(1); opacity: 1; }
100% { transform: translate(-50%, -50%) scale(2); opacity: 0; }
}
@keyframes browser-cli-click {
0% { transform: translate(-50%, -50%) scale(1); }
50% { transform: translate(-50%, -50%) scale(0.8); }
100% { transform: translate(-50%, -50%) scale(1); }
}
`;
virtualCursor.append(inner);
virtualCursor.append(outer);
document.head.append(style);
document.body.append(virtualCursor);
}
/**
* Move virtual cursor to element with animation
* @param {Element} element - Target element
* @param {boolean} [showClick=false] - Whether to show click animation
*/
function moveCursorToElement(element, showClick = false) {
if (!virtualCursor) {
initializeVirtualCursor();
}
const rect = element.getBoundingClientRect();
const x = rect.left + rect.width / 2;
const y = rect.top + rect.height / 2;
if (virtualCursor) {
virtualCursor.style.left = `${x}px`;
virtualCursor.style.top = `${y}px`;
virtualCursor.style.opacity = "1";
if (showClick) {
const inner = virtualCursor.querySelector("div:first-child");
if (inner && inner instanceof HTMLElement) {
inner.style.animation = "browser-cli-click 0.3s ease-out";
setTimeout(() => {
inner.style.animation = "";
}, 300);
}
}
}
}
// ============================================================================
// Console Override
// ============================================================================
/**
* Inject console override into page context to capture all logs
*/
function injectConsoleOverride() {
const script = document.createElement("script");
script.textContent = `
(function() {
const consoleMethods = ['log', 'error', 'warn', 'info', 'debug'];
consoleMethods.forEach(method => {
const original = console[method];
console[method] = function(...args) {
window.postMessage({
type: 'browser-cli-console',
method: method,
args: args.map(arg => {
try {
return typeof arg === 'object' ? JSON.stringify(arg) : String(arg);
} catch {
return '[Circular reference]';
}
}),
timestamp: new Date().toISOString()
}, '*');
original.apply(console, args);
};
});
})();
`;
(document.head || document.documentElement).append(script);
script.remove();
}
window.addEventListener("message", (event) => {
if (event.source !== window) return;
if (event.data && event.data.type === "browser-cli-console") {
consoleLogs.push({
type: event.data.method,
message: event.data.args.join(" "),
timestamp: event.data.timestamp,
});
if (consoleLogs.length > MAX_CONSOLE_LOGS) {
consoleLogs.shift();
}
}
});
injectConsoleOverride();
/** @type {Array} */
const consoleMethods = /** @type {any} */ ([
"log",
"error",
"warn",
"info",
"debug",
]);
for (const method of consoleMethods) {
// @ts-ignore
const original = console[method];
// @ts-ignore
console[method] = function (.../** @type {any[]} */ args) {
consoleLogs.push({
type: method,
message: args
.map((arg) => {
try {
return typeof arg === "object" ? JSON.stringify(arg) : String(arg);
} catch {
return String(arg);
}
})
.join(" "),
timestamp: new Date().toISOString(),
});
if (consoleLogs.length > MAX_CONSOLE_LOGS) {
consoleLogs.shift();
}
original.apply(console, args);
};
}
// ============================================================================
// Element Finding Functions
// ============================================================================
/**
* Find element by ref number
* @param {number} ref - Reference number from snapshot
* @returns {Element} Found element
*/
function findByRef(ref) {
const element = currentRefMap.get(ref);
if (element) {
// Verify element is still in DOM
if (document.contains(element)) {
return element;
}
// Element was removed, clear stale ref
currentRefMap.delete(ref);
}
// Build helpful error message with available refs
const available = [];
for (const [r, el] of currentRefMap) {
if (!document.contains(el)) {
continue;
}
const role = getElementRole(el);
const name = getAccessibleName(el);
available.push(` [${r}] ${role}${name ? ` "${name}"` : ""}`);
if (available.length >= 5) {
available.push(" ...");
break;
}
}
let msg = `Element [${ref}] not found.`;
msg +=
available.length > 0
? ` Available:\n${available.join("\n")}`
: " No refs available. Call snap() first.";
throw new Error(msg);
}
/**
* Find element by CSS selector
* @param {string} selector - CSS selector
* @returns {Element} Found element
*/
function findBySelector(selector) {
try {
const element = document.querySelector(selector);
if (element) return element;
throw new Error(`No element matches CSS selector: ${selector}`);
} catch (error) {
if (error instanceof Error && error.name === "SyntaxError") {
throw new Error(`Invalid CSS selector: ${selector}`);
}
throw error;
}
}
/**
* Find element by text content
* @param {string} text - Text to search for
* @returns {Element} Found element
*/
function findByText(text) {
const xpath = text.includes('"')
? `//*[contains(text(), '${text}')]`
: `//*[contains(text(), "${text}")]`;
const result = document.evaluate(
xpath,
document,
undefined,
XPathResult.FIRST_ORDERED_NODE_TYPE,
);
if (result.singleNodeValue && result.singleNodeValue instanceof Element) {
return result.singleNodeValue;
}
throw new Error(`No element found with text: ${text}`);
}
/**
* Find element by aria-label
* @param {string} label - ARIA label to search for
* @returns {Element} Found element
*/
function findByAriaLabel(label) {
const element = document.querySelector(`[aria-label="${CSS.escape(label)}"]`);
if (element) return element;
throw new Error(`No element found with aria-label: ${label}`);
}
/**
* Find element by placeholder
* @param {string} placeholder - Placeholder text to search for
* @returns {Element} Found element
*/
function findByPlaceholder(placeholder) {
const element = document.querySelector(
`[placeholder="${CSS.escape(placeholder)}"]`,
);
if (element) return element;
throw new Error(`No element found with placeholder: ${placeholder}`);
}
/**
* Find element based on selector type or ref
* @param {string|number} selector - The selector value or ref number
* @param {SelectorType} [selectorType='css'] - Type of selector
* @returns {Element} Found element
*/
function find(selector, selectorType = "css") {
// If selector is a number, treat as ref
if (typeof selector === "number") {
return findByRef(selector);
}
switch (selectorType) {
case "css": {
return findBySelector(selector);
}
case "text": {
return findByText(selector);
}
case "aria-label": {
return findByAriaLabel(selector);
}
case "placeholder": {
return findByPlaceholder(selector);
}
default: {
throw new Error(`Unknown selector type: ${selectorType}`);
}
}
}
// ============================================================================
// Snapshot Generation
// ============================================================================
/**
* Get the role of an element
* @param {Element} element
* @returns {string}
*/
function getElementRole(element) {
const explicitRole = element.getAttribute("role");
if (explicitRole) return explicitRole;
const tag = element.tagName.toLowerCase();
// Map common tags to roles
const tagRoleMap = {
a: "link",
button: "button",
input: "input",
textarea: "textarea",
select: "select",
img: "img",
h1: "heading",
h2: "heading",
h3: "heading",
h4: "heading",
h5: "heading",
h6: "heading",
nav: "navigation",
main: "main",
header: "banner",
footer: "contentinfo",
aside: "complementary",
form: "form",
table: "table",
ul: "list",
ol: "list",
li: "listitem",
};
if (tag === "input") {
const type = element.getAttribute("type") || "text";
if (type === "checkbox") return "checkbox";
if (type === "radio") return "radio";
if (type === "submit" || type === "button") return "button";
return `input[${type}]`;
}
return /** @type {Record} */ (tagRoleMap)[tag] || tag;
}
/**
* Get accessible name of an element
* @param {Element} element
* @returns {string}
*/
function getAccessibleName(element) {
// aria-label takes precedence
const ariaLabel = element.getAttribute("aria-label");
if (ariaLabel) return ariaLabel;
// aria-labelledby
const labelledBy = element.getAttribute("aria-labelledby");
if (labelledBy) {
const labelEl = document.querySelector(`#${CSS.escape(labelledBy)}`);
if (labelEl) return labelEl.textContent?.trim() || "";
}
// For inputs, check associated label
if (
element instanceof HTMLInputElement ||
element instanceof HTMLTextAreaElement ||
element instanceof HTMLSelectElement
) {
// Check for label wrapping the input
const parentLabel = element.closest("label");
if (parentLabel) {
const text = parentLabel.textContent?.trim() || "";
return text;
}
// Check for label with for attribute
if (element.id) {
const label = document.querySelector(
`label[for="${CSS.escape(element.id)}"]`,
);
if (label) return label.textContent?.trim() || "";
}
// Fall back to placeholder
const placeholder = element.getAttribute("placeholder");
if (placeholder) return placeholder;
}
// alt text for images
if (element instanceof HTMLImageElement) {
return element.alt || "";
}
// title attribute
const title = element.getAttribute("title");
if (title) return title;
// Text content for buttons, links, etc.
const tag = element.tagName.toLowerCase();
if (["button", "a", "h1", "h2", "h3", "h4", "h5", "h6"].includes(tag)) {
// Get text content excluding script/style elements
let text = "";
const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT, {
acceptNode: (node) => {
const parent = node.parentElement;
if (
parent &&
["SCRIPT", "STYLE", "NOSCRIPT"].includes(parent.tagName)
) {
return NodeFilter.FILTER_REJECT;
}
return NodeFilter.FILTER_ACCEPT;
},
});
while (walker.nextNode()) {
text += walker.currentNode.textContent;
}
text = text.trim();
// Filter out CSS-looking content (starts with . or # followed by { or contains {)
if (text && /^\s*[.#]?[\w-]+\s*\{/.test(text)) {
return "";
}
return text;
}
return "";
}
/**
* Get element attributes for display
* @param {Element} element
* @returns {string[]}
*/
function getElementAttrs(element) {
const attrs = [];
// Disabled state
if (
element.hasAttribute("disabled") ||
element.getAttribute("aria-disabled") === "true"
) {
attrs.push("disabled");
}
// Checked state for checkboxes/radios
if (
element instanceof HTMLInputElement &&
(element.type === "checkbox" || element.type === "radio") &&
element.checked
) {
attrs.push("checked");
}
// Required
if (
element.hasAttribute("required") ||
element.getAttribute("aria-required") === "true"
) {
attrs.push("required");
}
// Expanded/collapsed
const expanded = element.getAttribute("aria-expanded");
if (expanded === "true") attrs.push("expanded");
if (expanded === "false") attrs.push("collapsed");
// Readonly
if (element.hasAttribute("readonly")) {
attrs.push("readonly");
}
// Clickable (for non-obvious elements)
const tag = element.tagName.toLowerCase();
if (
!["a", "button", "input", "select", "textarea"].includes(tag) &&
(element.getAttribute("onclick") ||
element.getAttribute("role") === "button" ||
window.getComputedStyle(element).cursor === "pointer")
) {
attrs.push("clickable");
}
return attrs;
}
/**
* Get element value
* @param {Element} element
* @returns {string|undefined}
*/
function getElementValue(element) {
if (element instanceof HTMLInputElement) {
if (element.type === "checkbox" || element.type === "radio") {
return; // Use checked attr instead
}
if (element.type === "password") {
return element.value ? "••••••" : "";
}
return element.value;
}
if (element instanceof HTMLTextAreaElement) {
return element.value;
}
if (element instanceof HTMLSelectElement) {
const selected = element.options[element.selectedIndex];
return selected ? selected.text : "";
}
}
/**
* Check if element should be included in snapshot
* @param {Element} element
* @returns {boolean}
*/
function shouldIncludeElement(element) {
// Skip hidden elements
const style = window.getComputedStyle(element);
if (style.display === "none" || style.visibility === "hidden") {
return false;
}
// Skip zero-size elements
const rect = element.getBoundingClientRect();
if (rect.width === 0 && rect.height === 0) {
return false;
}
// Skip script, style, noscript, template
const tag = element.tagName.toLowerCase();
if (
["script", "style", "noscript", "template", "svg", "path"].includes(tag)
) {
return false;
}
// Include interactive elements
if (["a", "button", "input", "textarea", "select"].includes(tag)) {
return true;
}
// Include elements with explicit roles
if (element.hasAttribute("role")) {
const role = element.getAttribute("role");
if (role && !["presentation", "none"].includes(role)) return true;
}
// Include headings
if (["h1", "h2", "h3", "h4", "h5", "h6"].includes(tag)) {
return true;
}
// Include landmarks
if (["nav", "main", "header", "footer", "aside", "form"].includes(tag)) {
return true;
}
// Include images with alt text
if (tag === "img" && element.getAttribute("alt")) {
return true;
}
// Include elements with onclick attribute
if (element.getAttribute("onclick")) {
return true;
}
// Include elements with aria-label
if (element.hasAttribute("aria-label")) {
return true;
}
// For cursor:pointer elements, only include if they:
// 1. Have a meaningful accessible name (not empty, not just whitespace)
// 2. Are not generic wrapper tags (div, span, g, rect, circle, etc.)
// 3. Or have tabindex making them keyboard-focusable
if (window.getComputedStyle(element).cursor === "pointer") {
const genericTags = [
"div",
"span",
"g",
"rect",
"circle",
"path",
"svg",
"text",
"br",
"hr",
"b",
"i",
"em",
"strong",
];
if (genericTags.includes(tag)) {
// Only include generic clickable if it has tabindex or meaningful name
if (
element.hasAttribute("tabindex") &&
element.getAttribute("tabindex") !== "-1"
) {
return true;
}
// Check for meaningful accessible name
const name = getAccessibleName(element);
if (name && name.length > 0 && name.length < 200 && !/^\s*$/.test(name)) {
return true;
}
return false;
}
return true;
}
return false;
}
/**
* @typedef {object} SnapOptions
* @property {boolean} [forms] - Only form elements
* @property {boolean} [links] - Only links
* @property {boolean} [buttons] - Only buttons
* @property {string} [text] - Filter by text content
* @property {number} [near] - Elements near ref
*/
/**
* Generate snapshot with optional filtering
* @param {SnapOptions} [options]
* @returns {Snapshot}
*/
function generateSnapshot(options = {}) {
const elements = [];
let ref = 1;
// Clear old ref map
currentRefMap = new Map();
// Collect all relevant elements
const allElements = document.body.querySelectorAll("*");
for (const element of allElements) {
if (!shouldIncludeElement(element)) continue;
const role = getElementRole(element);
const name = getAccessibleName(element);
const attrs = getElementAttrs(element);
const value = getElementValue(element);
// Apply filters
if (
options.forms &&
!["input", "textarea", "select", "checkbox", "radio"].includes(role) &&
!role.startsWith("input[")
) {
continue;
}
if (options.links && role !== "link") continue;
if (options.buttons && role !== "button") continue;
if (options.text) {
const searchText = options.text.toLowerCase();
if (
!name.toLowerCase().includes(searchText) &&
!(value && value.toLowerCase().includes(searchText))
) {
continue;
}
}
const snapshotElement = new SnapshotElement({
ref,
role,
name,
value,
attrs,
domElement: element,
});
elements.push(snapshotElement);
currentRefMap.set(ref, element);
ref++;
}
// Handle "near" filter - find elements near a reference element
if (options.near === undefined) {
return new Snapshot(elements, window.location.href, document.title);
}
const nearElement = currentRefMap.get(options.near);
if (!nearElement) {
return new Snapshot(elements, window.location.href, document.title);
}
const nearRect = nearElement.getBoundingClientRect();
const threshold = 200; // pixels
const filtered = elements.filter((el) => {
const rect = el.domElement.getBoundingClientRect();
const distance = Math.hypot(
rect.left - nearRect.left,
rect.top - nearRect.top,
);
return distance < threshold;
});
return new Snapshot(filtered, window.location.href, document.title);
}
// ============================================================================
// JS API - Main Functions
// ============================================================================
/**
* Click an element
* @param {string|number} selector - Element selector or ref number
* @param {SelectorType|{double?: boolean}} [selectorTypeOrOptions='css']
* @returns {Promise<{clicked: string|number}>}
*/
async function click(selector, selectorTypeOrOptions = "css") {
let selectorType = "css";
let double = false;
if (typeof selectorTypeOrOptions === "object") {
double = selectorTypeOrOptions.double || false;
} else {
selectorType = selectorTypeOrOptions;
}
const element = /** @type {HTMLElement} */ (
find(selector, /** @type {SelectorType} */ (selectorType))
);
moveCursorToElement(element, true);
await new Promise((resolve) => setTimeout(resolve, 300));
if (double) {
element.dispatchEvent(
new MouseEvent("dblclick", {
view: window,
bubbles: true,
cancelable: true,
}),
);
} else {
element.click();
}
if (element.tagName === "INPUT" || element.tagName === "TEXTAREA") {
lastFocusedInput = /** @type {HTMLInputElement|HTMLTextAreaElement} */ (
element
);
}
return { clicked: selector };
}
/**
* Type text into an element
* @param {string|number} selector - Element selector or ref number
* @param {string} text - Text to type
* @param {SelectorType|{clear?: boolean}} [selectorTypeOrOptions='css']
* @returns {Promise<{typed: string, into: string|number}>}
*/
async function type(selector, text, selectorTypeOrOptions = "css") {
let selectorType = "css";
let clear = false;
if (typeof selectorTypeOrOptions === "object") {
clear = selectorTypeOrOptions.clear || false;
} else {
selectorType = selectorTypeOrOptions;
}
const element = /** @type {HTMLInputElement|HTMLTextAreaElement} */ (
find(selector, /** @type {SelectorType} */ (selectorType))
);
if (element.tagName !== "INPUT" && element.tagName !== "TEXTAREA") {
throw new Error(
`Element is not an input field. Found: <${element.tagName.toLowerCase()}>`,
);
}
moveCursorToElement(element);
await new Promise((resolve) => setTimeout(resolve, 300));
element.focus();
lastFocusedInput = element;
if (clear) {
element.value = "";
}
element.value = clear ? text : element.value + text;
element.dispatchEvent(new Event("input", { bubbles: true }));
element.dispatchEvent(new Event("change", { bubbles: true }));
return { typed: text, into: selector };
}
/**
* Hover over an element
* @param {string|number} selector - Element selector or ref number
* @param {SelectorType} [selectorType='css'] - Type of selector
* @returns {Promise<{hovered: string|number}>}
*/
async function hover(selector, selectorType = "css") {
const element = find(selector, selectorType);
moveCursorToElement(element);
await new Promise((resolve) => setTimeout(resolve, 300));
element.dispatchEvent(
new MouseEvent("mouseenter", {
view: window,
bubbles: true,
cancelable: true,
}),
);
element.dispatchEvent(
new MouseEvent("mouseover", {
view: window,
bubbles: true,
cancelable: true,
}),
);
return { hovered: selector };
}
/**
* Drag from one element to another
* @param {string|number} startSelector - Start element selector or ref
* @param {string|number} endSelector - End element selector or ref
* @param {SelectorType} [selectorType='css'] - Type of selector
* @returns {Promise<{dragged: string|number, to: string|number}>}
*/
async function drag(startSelector, endSelector, selectorType = "css") {
const startElement = find(startSelector, selectorType);
const endElement = find(endSelector, selectorType);
const startRect = startElement.getBoundingClientRect();
const endRect = endElement.getBoundingClientRect();
const startX = startRect.left + startRect.width / 2;
const startY = startRect.top + startRect.height / 2;
const endX = endRect.left + endRect.width / 2;
const endY = endRect.top + endRect.height / 2;
moveCursorToElement(startElement, true);
await new Promise((resolve) => setTimeout(resolve, 300));
const dataTransfer = new DataTransfer();
dataTransfer.effectAllowed = "all";
dataTransfer.dropEffect = "move";
startElement.dispatchEvent(
new MouseEvent("mousedown", {
view: window,
bubbles: true,
cancelable: true,
clientX: startX,
clientY: startY,
}),
);
startElement.dispatchEvent(
new DragEvent("dragstart", {
view: window,
bubbles: true,
cancelable: true,
clientX: startX,
clientY: startY,
dataTransfer,
}),
);
if (virtualCursor) {
virtualCursor.style.transition = "all 0.8s cubic-bezier(0.4, 0, 0.2, 1)";
moveCursorToElement(endElement);
await new Promise((resolve) => setTimeout(resolve, 800));
}
endElement.dispatchEvent(
new DragEvent("dragenter", {
view: window,
bubbles: true,
cancelable: true,
clientX: endX,
clientY: endY,
dataTransfer,
}),
);
endElement.dispatchEvent(
new DragEvent("dragover", {
view: window,
bubbles: true,
cancelable: true,
clientX: endX,
clientY: endY,
dataTransfer,
}),
);
await new Promise((resolve) => setTimeout(resolve, 50));
endElement.dispatchEvent(
new DragEvent("drop", {
view: window,
bubbles: true,
cancelable: true,
clientX: endX,
clientY: endY,
dataTransfer,
}),
);
startElement.dispatchEvent(
new DragEvent("dragend", {
view: window,
bubbles: true,
cancelable: true,
clientX: endX,
clientY: endY,
dataTransfer,
}),
);
if (virtualCursor) {
virtualCursor.style.transition = "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)";
}
return { dragged: startSelector, to: endSelector };
}
/**
* Select an option in a dropdown
* @param {string|number} selector - Select element selector or ref
* @param {string} option - Option value to select
* @param {SelectorType} [selectorType='css'] - Type of selector
* @returns {Promise<{selected: string, in: string|number}>}
*/
async function select(selector, option, selectorType = "css") {
const element = /** @type {HTMLSelectElement} */ (
find(selector, selectorType)
);
if (element.tagName !== "SELECT") {
throw new Error(
`Element is not a select field. Found: <${element.tagName.toLowerCase()}>`,
);
}
moveCursorToElement(element, true);
await new Promise((resolve) => setTimeout(resolve, 300));
element.value = option;
element.dispatchEvent(new Event("change", { bubbles: true }));
return { selected: option, in: selector };
}
/**
* Press a keyboard key
* @param {string} keyName - Key to press
* @returns {Snapshot}
*/
/** @type {Record} */
const SPECIAL_KEYS = {
Enter: { code: "Enter", keyCode: 13 },
Tab: { code: "Tab", keyCode: 9 },
Escape: { code: "Escape", keyCode: 27 },
Backspace: { code: "Backspace", keyCode: 8 },
Delete: { code: "Delete", keyCode: 46 },
ArrowUp: { code: "ArrowUp", keyCode: 38 },
ArrowDown: { code: "ArrowDown", keyCode: 40 },
ArrowLeft: { code: "ArrowLeft", keyCode: 37 },
ArrowRight: { code: "ArrowRight", keyCode: 39 },
Home: { code: "Home", keyCode: 36 },
End: { code: "End", keyCode: 35 },
PageUp: { code: "PageUp", keyCode: 33 },
PageDown: { code: "PageDown", keyCode: 34 },
" ": { code: "Space", keyCode: 32 },
Space: { code: "Space", keyCode: 32 },
};
const TEXT_INPUT_TYPES = new Set([
"text",
"search",
"email",
"password",
"tel",
"url",
"number",
]);
/**
* Build keyboard event options for a key
* @param {string} keyName
* @returns {{key: string, code: string, keyCode: number, which: number, bubbles: boolean, cancelable: boolean, composed: boolean}}
*/
function buildKeyEventOptions(keyName) {
let code = keyName;
let keyCode = 0;
let key = keyName;
if (SPECIAL_KEYS[keyName]) {
code = SPECIAL_KEYS[keyName].code;
keyCode = SPECIAL_KEYS[keyName].keyCode;
key = keyName === "Space" ? " " : keyName;
} else if (keyName.length === 1) {
keyCode = keyName.codePointAt(0) || 0;
if (keyName >= "a" && keyName <= "z") {
code = `Key${keyName.toUpperCase()}`;
keyCode = keyName.toUpperCase().codePointAt(0) || 0;
} else if (keyName >= "A" && keyName <= "Z") {
code = `Key${keyName}`;
} else if (keyName >= "0" && keyName <= "9") {
code = `Digit${keyName}`;
}
} else {
code = keyName.charAt(0).toUpperCase() + keyName.slice(1);
}
return {
key,
code,
keyCode,
which: keyCode,
bubbles: true,
cancelable: true,
composed: true,
};
}
/**
* Check if element is a text-editable input
* @param {Element} element
* @returns {boolean}
*/
function isTextInput(element) {
if (element.tagName === "TEXTAREA") {
return true;
}
if (element.tagName === "INPUT") {
const type =
/** @type {HTMLInputElement} */ (element).type?.toLowerCase() || "text";
return TEXT_INPUT_TYPES.has(type);
}
return false;
}
/**
* Insert text at cursor position in input/textarea
* @param {HTMLInputElement|HTMLTextAreaElement} element
* @param {string} text
*/
function insertTextAtCursor(element, text) {
const start = element.selectionStart || 0;
const end = element.selectionEnd || 0;
element.value =
element.value.slice(0, start) + text + element.value.slice(end);
element.selectionStart = element.selectionEnd = start + text.length;
element.dispatchEvent(
new Event("input", { bubbles: true, cancelable: true }),
);
}
/**
* Try to submit the form containing an input
* @param {HTMLInputElement} input
* @returns {boolean} True if form was submitted
*/
function trySubmitForm(input) {
const form = input.form;
if (!form) {
return false;
}
// Try requestSubmit first (triggers submit event handlers)
if (typeof form.requestSubmit === "function") {
try {
form.requestSubmit();
return true;
} catch {
// requestSubmit can throw if form is invalid, fall through
}
}
// Fallback: find submit button and click it
const submitButton = form.querySelector(
'button[type="submit"], input[type="submit"], button:not([type])',
);
if (submitButton) {
/** @type {HTMLButtonElement|HTMLInputElement} */ (submitButton).click();
return true;
}
// Last resort: dispatch submit event
form.dispatchEvent(
new SubmitEvent("submit", { bubbles: true, cancelable: true }),
);
return true;
}
/**
* Handle Tab key navigation
* @param {Element} activeElement
*/
function handleTabKey(activeElement) {
const focusableElements = /** @type {HTMLElement[]} */ ([
...document.querySelectorAll(
'a[href], button, input, textarea, select, [tabindex]:not([tabindex="-1"])',
),
]).filter(
(el) =>
!(/** @type {HTMLButtonElement|HTMLInputElement} */ (el).disabled) &&
el.offsetParent !== null,
);
const currentIndex = focusableElements.indexOf(
/** @type {HTMLElement} */ (activeElement),
);
if (currentIndex !== -1 && currentIndex < focusableElements.length - 1) {
focusableElements[currentIndex + 1].focus();
}
// Track newly focused input
setTimeout(() => {
const newActive = document.activeElement;
if (
newActive &&
(newActive.tagName === "INPUT" || newActive.tagName === "TEXTAREA")
) {
lastFocusedInput = /** @type {HTMLInputElement|HTMLTextAreaElement} */ (
newActive
);
}
}, 50);
}
/**
* Handle character input in contentEditable
* @param {Element} element
* @param {string} char
*/
function insertCharInContentEditable(element, char) {
const selection = window.getSelection();
if (!selection || selection.rangeCount === 0) {
return;
}
const range = selection.getRangeAt(0);
range.deleteContents();
const textNode = document.createTextNode(char);
range.insertNode(textNode);
range.setStartAfter(textNode);
range.setEndAfter(textNode);
selection.removeAllRanges();
selection.addRange(range);
element.dispatchEvent(
new Event("input", { bubbles: true, cancelable: true }),
);
}
/**
* Simulate a key press
* @param {string} keyName - Key to press (e.g., "Enter", "Tab", "a", "A")
* @returns {{key: string}}
*/
function key(keyName) {
// Get active element, restoring last focused input if needed
let activeElement = document.activeElement || document.body;
if (activeElement === document.body && lastFocusedInput) {
activeElement = lastFocusedInput;
lastFocusedInput.focus();
}
const eventOptions = buildKeyEventOptions(keyName);
// Dispatch keydown
const keydownPrevented = !activeElement.dispatchEvent(
new KeyboardEvent("keydown", eventOptions),
);
// Handle special keys
if (!keydownPrevented) {
if (keyName === "Enter") {
if (activeElement.tagName === "INPUT" && isTextInput(activeElement)) {
trySubmitForm(/** @type {HTMLInputElement} */ (activeElement));
} else if (activeElement.tagName === "TEXTAREA") {
insertTextAtCursor(
/** @type {HTMLTextAreaElement} */ (activeElement),
"\n",
);
}
} else if (keyName === "Tab") {
handleTabKey(activeElement);
} else if (keyName.length === 1) {
// Printable character
if (isTextInput(activeElement)) {
insertTextAtCursor(
/** @type {HTMLInputElement|HTMLTextAreaElement} */ (activeElement),
keyName,
);
activeElement.dispatchEvent(
new Event("change", { bubbles: true, cancelable: true }),
);
} else if (
/** @type {HTMLElement} */ (activeElement).contentEditable === "true"
) {
insertCharInContentEditable(activeElement, keyName);
}
}
}
// Dispatch keypress for printable characters
if (keyName.length === 1) {
activeElement.dispatchEvent(new KeyboardEvent("keypress", eventOptions));
}
// Dispatch keyup
activeElement.dispatchEvent(new KeyboardEvent("keyup", eventOptions));
return { key: keyName };
}
/**
* Get snapshot of the page
* Returns full snapshot on first call, diff on subsequent calls (to save tokens)
* @param {SnapOptions & {full?: boolean}} [options] - Filter options, use {full: true} for full snapshot
* @returns {Snapshot|SnapshotDiff}
*/
function snap(options) {
const snapshot = generateSnapshot(options);
// If full snapshot explicitly requested or options are set (filtering), return full
if (
options?.full ||
(options && Object.keys(options).some((k) => k !== "full"))
) {
if (!options || !Object.keys(options).some((k) => k !== "full")) {
lastSnapshot = snapshot;
}
return snapshot;
}
// First call or no previous snapshot - return full snapshot
if (!lastSnapshot) {
lastSnapshot = snapshot;
return snapshot;
}
// Return diff to save tokens
const result = snapshot.diff(lastSnapshot);
lastSnapshot = snapshot;
return result;
}
/**
* Get console logs
* @returns {{logs: ConsoleLog[]}}
*/
function logs() {
return { logs: [...consoleLogs] };
}
/**
* Wait for DOM to stabilize (no mutations for a period)
* @param {number} [stableMs=300] - How long DOM must be stable
* @param {number} [timeout=10000] - Maximum wait time
* @returns {Promise<{waited: string}>}
*/
function waitForIdle(stableMs = 300, timeout = 10_000) {
return new Promise((resolve, reject) => {
let lastMutationTime = Date.now();
let resolved = false;
const startTime = Date.now();
const observer = new MutationObserver(() => {
lastMutationTime = Date.now();
});
observer.observe(document.body, {
childList: true,
subtree: true,
attributes: true,
characterData: true,
});
const checkStable = () => {
if (resolved) {
return;
}
const now = Date.now();
if (now - startTime > timeout) {
resolved = true;
observer.disconnect();
reject(new Error("Timeout waiting for page to stabilize"));
return;
}
if (now - lastMutationTime >= stableMs) {
resolved = true;
observer.disconnect();
resolve({ waited: "idle" });
return;
}
setTimeout(checkStable, 50);
};
// Start checking after initial delay
setTimeout(checkStable, 50);
});
}
/**
* Wait for condition or timeout
* @param {number|string} condition - ms to wait, "idle", "text", or "gone"
* @param {string|number} [value] - Text to wait for, or stableMs for idle
* @param {number} [timeout=10000] - Maximum wait time in ms
* @returns {Promise<{waited: number|string}>}
*/
async function wait(condition, value, timeout = 10_000) {
if (typeof condition === "number") {
// Simple delay
await new Promise((resolve) => setTimeout(resolve, condition));
return { waited: condition };
}
if (condition === "idle") {
// Wait for DOM to stabilize
const stableMs = typeof value === "number" ? value : 300;
return waitForIdle(stableMs, timeout);
}
if (condition === "text" && typeof value === "string") {
// Wait for text to appear
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
if (document.body.textContent?.includes(value)) {
return { waited: `text:${value}` };
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
throw new Error(`Timeout waiting for text: "${value}"`);
}
if (condition === "gone" && typeof value === "string") {
// Wait for text to disappear
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
if (!document.body.textContent?.includes(value)) {
return { waited: `gone:${value}` };
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
throw new Error(`Timeout waiting for text to disappear: "${value}"`);
}
throw new Error(`Invalid wait condition: ${condition}`);
}
// ============================================================================
// Browser-level functions (delegate to background script)
// ============================================================================
/** @type {number} */
let bgMessageCounter = 0;
/**
* Send a command to the background script and wait for response
* @param {string} command
* @param {object} [params={}]
* @returns {Promise}
*/
function sendToBackground(command, params = {}) {
return new Promise((resolve, reject) => {
const messageId = `content_${++bgMessageCounter}`;
/**
* @param {any} response
*/
function handleResponse(response) {
if (response && response.bgMessageId === messageId) {
browser.runtime.onMessage.removeListener(handleResponse);
if (response.error) {
reject(new Error(response.error));
} else {
resolve(response.result);
}
}
}
browser.runtime.onMessage.addListener(handleResponse);
browser.runtime.sendMessage({ command, params, bgMessageId: messageId });
// Timeout after 30 seconds
setTimeout(() => {
browser.runtime.onMessage.removeListener(handleResponse);
reject(new Error("Background script timeout"));
}, 30_000);
});
}
/**
* Take a screenshot
* @param {string} [path] - Output file path
* @returns {Promise} Path to saved screenshot
*/
async function shot(path) {
const result = await sendToBackground("screenshot", { output_path: path });
return result.screenshot_path || result.screenshot;
}
/**
* Create a new tab
* @param {string} [url] - URL to open
* @returns {Promise<{tabId: string, url: string}>} Tab info
*/
async function tab(url) {
const result = await sendToBackground("new-tab", { url });
return { tabId: result.tabId, url: result.url };
}
/**
* List all managed tabs
* @returns {Promise>}
*/
async function tabs() {
const result = await sendToBackground("list-tabs");
return result.tabs;
}
/**
* Download a file
* @param {string} url - URL to download
* @param {string} [filename] - Filename (relative to downloads folder)
* @returns {Promise<{downloaded: string, path: string}>}
*/
async function download(url, filename) {
return sendToBackground("download", { url, filename });
}
// ============================================================================
// Message handling for background script commands
// ============================================================================
/**
* Serialize result for transport
* @param {any} value
* @returns {any}
*/
function serializeResult(value) {
if (value === null || value === undefined) {
return value;
}
// Handle Snapshot and SnapshotElement
if (value instanceof Snapshot || value instanceof SnapshotElement) {
return value.toJSON();
}
// Handle arrays
if (Array.isArray(value)) {
return value.map((v) => serializeResult(v));
}
// Handle objects with toJSON
if (typeof value === "object" && typeof value.toJSON === "function") {
return value.toJSON();
}
// Try structuredClone for other objects
try {
return structuredClone(value);
} catch {
return String(value);
}
}
/**
* Handle exec command - execute JavaScript with API available
* @param {string} code - JavaScript code to execute
* @returns {Promise<{result: any}>}
*/
async function handleExec(code) {
// Create async function to allow await in code
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
// Wrap code to auto-return the last expression if no explicit return
let wrappedCode = code.trim();
if (!wrappedCode.includes("return ") && !wrappedCode.includes("return;")) {
// Find the last statement and return it
const lines = wrappedCode.split("\n");
const lastLine = lines.pop()?.trim() || "";
if (lastLine && !lastLine.endsWith(";")) {
// Last line is an expression - return it
wrappedCode = [...lines, `return (${lastLine})`].join("\n");
} else if (lastLine) {
// Last line ends with semicolon - try to return it anyway
const expr = lastLine.slice(0, -1).trim();
if (
expr &&
!expr.startsWith("const ") &&
!expr.startsWith("let ") &&
!expr.startsWith("var ")
) {
wrappedCode = [...lines, `return (${expr})`].join("\n");
}
}
}
// Make API functions available in the execution context
const fn = new AsyncFunction(
// Content script functions
"click",
"type",
"hover",
"drag",
"select",
"key",
"snap",
"logs",
"find",
"wait",
// Background script functions
"shot",
"tab",
"tabs",
"download",
wrappedCode,
);
const result = await fn(
// Content script functions
click,
type,
hover,
drag,
select,
key,
snap,
logs,
find,
wait,
// Background script functions
shot,
tab,
tabs,
download,
);
return { result: serializeResult(result) };
}
browser.runtime.onMessage.addListener((message, _sender, _sendResponse) => {
const { command, params, messageId } = message;
// Skip if this is a response to a background script request
if (message.bgMessageId) {
return false;
}
if (!virtualCursor && command !== "exec") {
initializeVirtualCursor();
}
/** */
async function executeCommand() {
try {
let result;
switch (command) {
case "exec": {
result = await handleExec(params.code);
break;
}
default: {
throw new Error(`Unknown command: ${command}`);
}
}
browser.runtime.sendMessage({ messageId, result, success: true });
} catch (error) {
browser.runtime.sendMessage({
messageId,
error: error instanceof Error ? error.message : String(error),
success: false,
});
}
}
executeCommand();
return true;
});