hangwin--mcp-chrome
hangerye
1428 行
49 KiB
TypeScript
1428 行
49 KiB
TypeScript
import { createErrorResponse, ToolResult } from '@/common/tool-handler';
|
|
import { BaseBrowserToolExecutor } from '../base-browser';
|
|
import { TOOL_NAMES } from 'chrome-mcp-shared';
|
|
import { ERROR_MESSAGES, TIMEOUTS } from '@/common/constants';
|
|
import { TOOL_MESSAGE_TYPES } from '@/common/message-types';
|
|
import { clickTool, fillTool } from './interaction';
|
|
import { keyboardTool } from './keyboard';
|
|
import { screenshotTool } from './screenshot';
|
|
import { screenshotContextManager, scaleCoordinates } from '@/utils/screenshot-context';
|
|
import { cdpSessionManager } from '@/utils/cdp-session-manager';
|
|
import {
|
|
captureFrameOnAction,
|
|
isAutoCaptureActive,
|
|
type ActionMetadata,
|
|
type ActionType,
|
|
} from './gif-recorder';
|
|
|
|
type MouseButton = 'left' | 'right' | 'middle';
|
|
|
|
interface Coordinates {
|
|
x: number;
|
|
y: number;
|
|
}
|
|
|
|
interface ZoomRegion {
|
|
x0: number;
|
|
y0: number;
|
|
x1: number;
|
|
y1: number;
|
|
}
|
|
|
|
interface Modifiers {
|
|
altKey?: boolean;
|
|
ctrlKey?: boolean;
|
|
metaKey?: boolean;
|
|
shiftKey?: boolean;
|
|
}
|
|
|
|
interface ComputerParams {
|
|
action:
|
|
| 'left_click'
|
|
| 'right_click'
|
|
| 'double_click'
|
|
| 'triple_click'
|
|
| 'left_click_drag'
|
|
| 'scroll'
|
|
| 'type'
|
|
| 'key'
|
|
| 'hover'
|
|
| 'wait'
|
|
| 'fill'
|
|
| 'fill_form'
|
|
| 'resize_page'
|
|
| 'scroll_to'
|
|
| 'zoom'
|
|
| 'screenshot';
|
|
// click/scroll coordinates in screenshot space (if screenshot context exists) or viewport space
|
|
coordinates?: Coordinates; // for click/scroll; for drag, this is endCoordinates
|
|
startCoordinates?: Coordinates; // for drag start
|
|
// Optional element refs (from chrome_read_page) as alternative to coordinates
|
|
ref?: string; // click target or drag end
|
|
startRef?: string; // drag start
|
|
scrollDirection?: 'up' | 'down' | 'left' | 'right';
|
|
scrollAmount?: number;
|
|
text?: string; // for type/key
|
|
repeat?: number; // for key action (1-100)
|
|
modifiers?: Modifiers; // for click actions
|
|
region?: ZoomRegion; // for zoom action
|
|
duration?: number; // seconds for wait
|
|
// For fill
|
|
selector?: string;
|
|
selectorType?: 'css' | 'xpath'; // Type of selector (default: 'css')
|
|
value?: string;
|
|
frameId?: number; // Target frame for selector/ref resolution
|
|
tabId?: number; // target existing tab id
|
|
windowId?: number;
|
|
background?: boolean; // avoid focusing/activating
|
|
}
|
|
|
|
// Minimal CDP helper encapsulated here to avoid scattering CDP code
|
|
class CDPHelper {
|
|
static async attach(tabId: number): Promise<void> {
|
|
await cdpSessionManager.attach(tabId, 'computer');
|
|
}
|
|
|
|
static async detach(tabId: number): Promise<void> {
|
|
await cdpSessionManager.detach(tabId, 'computer');
|
|
}
|
|
|
|
static async send(tabId: number, method: string, params?: object): Promise<any> {
|
|
return await cdpSessionManager.sendCommand(tabId, method, params);
|
|
}
|
|
|
|
static async dispatchMouseEvent(tabId: number, opts: any) {
|
|
const params: any = {
|
|
type: opts.type,
|
|
x: Math.round(opts.x),
|
|
y: Math.round(opts.y),
|
|
modifiers: opts.modifiers || 0,
|
|
};
|
|
if (
|
|
opts.type === 'mousePressed' ||
|
|
opts.type === 'mouseReleased' ||
|
|
opts.type === 'mouseMoved'
|
|
) {
|
|
params.button = opts.button || 'none';
|
|
if (opts.type === 'mousePressed' || opts.type === 'mouseReleased') {
|
|
params.clickCount = opts.clickCount || 1;
|
|
}
|
|
// Per CDP: buttons is ignored for mouseWheel
|
|
params.buttons = opts.buttons !== undefined ? opts.buttons : 0;
|
|
}
|
|
if (opts.type === 'mouseWheel') {
|
|
params.deltaX = opts.deltaX || 0;
|
|
params.deltaY = opts.deltaY || 0;
|
|
}
|
|
await this.send(tabId, 'Input.dispatchMouseEvent', params);
|
|
}
|
|
|
|
static async insertText(tabId: number, text: string) {
|
|
await this.send(tabId, 'Input.insertText', { text });
|
|
}
|
|
|
|
static modifierMask(mods: string[]): number {
|
|
const map: Record<string, number> = {
|
|
alt: 1,
|
|
ctrl: 2,
|
|
control: 2,
|
|
meta: 4,
|
|
cmd: 4,
|
|
command: 4,
|
|
win: 4,
|
|
windows: 4,
|
|
shift: 8,
|
|
};
|
|
let mask = 0;
|
|
for (const m of mods) mask |= map[m] || 0;
|
|
return mask;
|
|
}
|
|
|
|
// Enhanced key mapping for common non-character keys
|
|
private static KEY_ALIASES: Record<string, { key: string; code?: string; text?: string }> = {
|
|
enter: { key: 'Enter', code: 'Enter' },
|
|
return: { key: 'Enter', code: 'Enter' },
|
|
backspace: { key: 'Backspace', code: 'Backspace' },
|
|
delete: { key: 'Delete', code: 'Delete' },
|
|
tab: { key: 'Tab', code: 'Tab' },
|
|
escape: { key: 'Escape', code: 'Escape' },
|
|
esc: { key: 'Escape', code: 'Escape' },
|
|
space: { key: ' ', code: 'Space', text: ' ' },
|
|
pageup: { key: 'PageUp', code: 'PageUp' },
|
|
pagedown: { key: 'PageDown', code: 'PageDown' },
|
|
home: { key: 'Home', code: 'Home' },
|
|
end: { key: 'End', code: 'End' },
|
|
arrowup: { key: 'ArrowUp', code: 'ArrowUp' },
|
|
arrowdown: { key: 'ArrowDown', code: 'ArrowDown' },
|
|
arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft' },
|
|
arrowright: { key: 'ArrowRight', code: 'ArrowRight' },
|
|
};
|
|
|
|
private static resolveKeyDef(token: string): { key: string; code?: string; text?: string } {
|
|
const t = (token || '').toLowerCase();
|
|
if (this.KEY_ALIASES[t]) return this.KEY_ALIASES[t];
|
|
if (/^f([1-9]|1[0-2])$/.test(t)) {
|
|
return { key: t.toUpperCase(), code: t.toUpperCase() };
|
|
}
|
|
if (t.length === 1) {
|
|
const upper = t.toUpperCase();
|
|
return { key: upper, code: `Key${upper}`, text: t };
|
|
}
|
|
return { key: token };
|
|
}
|
|
|
|
static async dispatchSimpleKey(tabId: number, token: string) {
|
|
const def = this.resolveKeyDef(token);
|
|
if (def.text && def.text.length === 1) {
|
|
await this.insertText(tabId, def.text);
|
|
return;
|
|
}
|
|
await this.send(tabId, 'Input.dispatchKeyEvent', {
|
|
type: 'rawKeyDown',
|
|
key: def.key,
|
|
code: def.code,
|
|
});
|
|
await this.send(tabId, 'Input.dispatchKeyEvent', {
|
|
type: 'keyUp',
|
|
key: def.key,
|
|
code: def.code,
|
|
});
|
|
}
|
|
|
|
static async dispatchKeyChord(tabId: number, chord: string) {
|
|
const parts = chord.split('+');
|
|
const modifiers: string[] = [];
|
|
let keyToken = '';
|
|
for (const pRaw of parts) {
|
|
const p = pRaw.trim().toLowerCase();
|
|
if (
|
|
['ctrl', 'control', 'alt', 'shift', 'cmd', 'meta', 'command', 'win', 'windows'].includes(p)
|
|
)
|
|
modifiers.push(p);
|
|
else keyToken = pRaw.trim();
|
|
}
|
|
const mask = this.modifierMask(modifiers);
|
|
const def = this.resolveKeyDef(keyToken);
|
|
await this.send(tabId, 'Input.dispatchKeyEvent', {
|
|
type: 'rawKeyDown',
|
|
key: def.key,
|
|
code: def.code,
|
|
text: def.text,
|
|
modifiers: mask,
|
|
});
|
|
await this.send(tabId, 'Input.dispatchKeyEvent', {
|
|
type: 'keyUp',
|
|
key: def.key,
|
|
code: def.code,
|
|
modifiers: mask,
|
|
});
|
|
}
|
|
}
|
|
|
|
class ComputerTool extends BaseBrowserToolExecutor {
|
|
name = TOOL_NAMES.BROWSER.COMPUTER;
|
|
|
|
async execute(args: ComputerParams): Promise<ToolResult> {
|
|
const params = args || ({} as ComputerParams);
|
|
if (!params.action) return createErrorResponse('Action parameter is required');
|
|
|
|
try {
|
|
const explicit = await this.tryGetTab(args.tabId);
|
|
const tab = explicit || (await this.getActiveTabOrThrowInWindow(args.windowId));
|
|
if (!tab.id)
|
|
return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND + ': Active tab has no ID');
|
|
|
|
// Execute the action and capture frame on success
|
|
const result = await this.executeAction(params, tab);
|
|
|
|
// Trigger auto-capture on successful actions (except screenshot which is read-only)
|
|
if (!result.isError && params.action !== 'screenshot' && params.action !== 'wait') {
|
|
const actionType = this.mapActionToCapture(params.action);
|
|
if (actionType) {
|
|
// Convert to viewport-space coordinates for GIF overlays
|
|
// params.coordinates may be screenshot-space when screenshot context exists
|
|
const ctx = screenshotContextManager.getContext(tab.id);
|
|
const toViewport = (c?: Coordinates): { x: number; y: number } | undefined => {
|
|
if (!c) return undefined;
|
|
if (!ctx) return { x: c.x, y: c.y };
|
|
const scaled = scaleCoordinates(c.x, c.y, ctx);
|
|
return { x: scaled.x, y: scaled.y };
|
|
};
|
|
|
|
const endCoords = toViewport(params.coordinates);
|
|
const startCoords = toViewport(params.startCoordinates);
|
|
|
|
await this.triggerAutoCapture(tab.id, actionType, {
|
|
coordinateSpace: 'viewport',
|
|
coordinates: endCoords,
|
|
startCoordinates: startCoords,
|
|
endCoordinates: actionType === 'drag' ? endCoords : undefined,
|
|
text: params.text,
|
|
ref: params.ref,
|
|
});
|
|
}
|
|
}
|
|
|
|
return result;
|
|
} catch (error) {
|
|
console.error('Error in computer tool:', error);
|
|
return createErrorResponse(
|
|
`Failed to execute action: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private mapActionToCapture(action: string): ActionType | null {
|
|
const mapping: Record<string, ActionType> = {
|
|
left_click: 'click',
|
|
right_click: 'right_click',
|
|
double_click: 'double_click',
|
|
triple_click: 'triple_click',
|
|
left_click_drag: 'drag',
|
|
scroll: 'scroll',
|
|
type: 'type',
|
|
key: 'key',
|
|
hover: 'hover',
|
|
fill: 'fill',
|
|
fill_form: 'fill',
|
|
resize_page: 'other',
|
|
scroll_to: 'scroll',
|
|
zoom: 'other',
|
|
};
|
|
return mapping[action] || null;
|
|
}
|
|
|
|
private async executeAction(params: ComputerParams, tab: chrome.tabs.Tab): Promise<ToolResult> {
|
|
if (!tab.id) {
|
|
return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND + ': Active tab has no ID');
|
|
}
|
|
|
|
// Helper to project coordinates using screenshot context when available
|
|
const project = (c?: Coordinates): Coordinates | undefined => {
|
|
if (!c) return undefined;
|
|
const ctx = screenshotContextManager.getContext(tab.id!);
|
|
if (!ctx) return c;
|
|
const scaled = scaleCoordinates(c.x, c.y, ctx);
|
|
return { x: scaled.x, y: scaled.y };
|
|
};
|
|
|
|
switch (params.action) {
|
|
case 'resize_page': {
|
|
const width = Number((params as any).coordinates?.x || (params as any).text);
|
|
const height = Number((params as any).coordinates?.y || (params as any).value);
|
|
const w = Number((params as any).width ?? width);
|
|
const h = Number((params as any).height ?? height);
|
|
if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) {
|
|
return createErrorResponse('Provide width and height for resize_page (positive numbers)');
|
|
}
|
|
try {
|
|
// Prefer precise CDP emulation
|
|
await CDPHelper.attach(tab.id);
|
|
try {
|
|
await CDPHelper.send(tab.id, 'Emulation.setDeviceMetricsOverride', {
|
|
width: Math.round(w),
|
|
height: Math.round(h),
|
|
deviceScaleFactor: 0,
|
|
mobile: false,
|
|
screenWidth: Math.round(w),
|
|
screenHeight: Math.round(h),
|
|
});
|
|
} finally {
|
|
await CDPHelper.detach(tab.id);
|
|
}
|
|
} catch (e) {
|
|
// Fallback: window resize
|
|
if (tab.windowId !== undefined) {
|
|
await chrome.windows.update(tab.windowId, {
|
|
width: Math.round(w),
|
|
height: Math.round(h),
|
|
});
|
|
} else {
|
|
return createErrorResponse(
|
|
`Failed to resize via CDP and cannot determine windowId: ${e instanceof Error ? e.message : String(e)}`,
|
|
);
|
|
}
|
|
}
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({ success: true, action: 'resize_page', width: w, height: h }),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
}
|
|
case 'hover': {
|
|
// Resolve target point from ref | selector | coordinates
|
|
let coord: Coordinates | undefined = undefined;
|
|
let resolvedBy: 'ref' | 'selector' | 'coordinates' | undefined;
|
|
|
|
try {
|
|
if (params.ref) {
|
|
await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
|
|
// Scroll element into view first to ensure it's visible
|
|
try {
|
|
await this.sendMessageToTab(tab.id, { action: 'focusByRef', ref: params.ref });
|
|
} catch {
|
|
// Best effort - continue even if scroll fails
|
|
}
|
|
// Re-resolve coordinates after scroll
|
|
const resolved = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
|
|
ref: params.ref,
|
|
});
|
|
if (resolved && resolved.success) {
|
|
coord = project({ x: resolved.center.x, y: resolved.center.y });
|
|
resolvedBy = 'ref';
|
|
}
|
|
} else if (params.selector) {
|
|
await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
|
|
const selectorType = params.selectorType || 'css';
|
|
const ensured = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
|
|
selector: params.selector,
|
|
isXPath: selectorType === 'xpath',
|
|
});
|
|
if (ensured && ensured.success) {
|
|
// Scroll element into view first to ensure it's visible
|
|
const resolvedRef = typeof ensured.ref === 'string' ? ensured.ref : undefined;
|
|
if (resolvedRef) {
|
|
try {
|
|
await this.sendMessageToTab(tab.id, { action: 'focusByRef', ref: resolvedRef });
|
|
} catch {
|
|
// Best effort - continue even if scroll fails
|
|
}
|
|
// Re-resolve coordinates after scroll
|
|
const reResolved = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
|
|
ref: resolvedRef,
|
|
});
|
|
if (reResolved && reResolved.success) {
|
|
coord = project({ x: reResolved.center.x, y: reResolved.center.y });
|
|
} else {
|
|
coord = project({ x: ensured.center.x, y: ensured.center.y });
|
|
}
|
|
} else {
|
|
coord = project({ x: ensured.center.x, y: ensured.center.y });
|
|
}
|
|
resolvedBy = 'selector';
|
|
}
|
|
} else if (params.coordinates) {
|
|
coord = project(params.coordinates);
|
|
resolvedBy = 'coordinates';
|
|
}
|
|
} catch (e) {
|
|
// fall through to error handling below
|
|
}
|
|
|
|
if (!coord)
|
|
return createErrorResponse(
|
|
'Provide ref or selector or coordinates for hover, or failed to resolve target',
|
|
);
|
|
{
|
|
const stale = ((): any => {
|
|
if (!params.coordinates) return null;
|
|
const getHostname = (url: string): string => {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
const currentHostname = getHostname(tab.url || '');
|
|
const ctx = screenshotContextManager.getContext(tab.id!);
|
|
const contextHostname = (ctx as any)?.hostname as string | undefined;
|
|
if (contextHostname && contextHostname !== currentHostname) {
|
|
return createErrorResponse(
|
|
`Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during hover. Capture a new screenshot or use ref/selector.`,
|
|
);
|
|
}
|
|
return null;
|
|
})();
|
|
if (stale) return stale;
|
|
}
|
|
|
|
try {
|
|
await CDPHelper.attach(tab.id);
|
|
try {
|
|
// Move pointer to target. We can dispatch a single mouseMoved; browsers will generate mouseover/mouseenter as needed.
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseMoved',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
button: 'none',
|
|
buttons: 0,
|
|
});
|
|
} finally {
|
|
await CDPHelper.detach(tab.id);
|
|
}
|
|
|
|
// Optional hold to allow UI (menus/tooltips) to appear
|
|
const holdMs = Math.max(
|
|
0,
|
|
Math.min(params.duration ? params.duration * 1000 : 400, 5000),
|
|
);
|
|
if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs));
|
|
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'hover',
|
|
coordinates: coord,
|
|
resolvedBy,
|
|
transport: 'cdp',
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (error) {
|
|
console.warn('[ComputerTool] CDP hover failed, attempting DOM fallback', error);
|
|
return await this.domHoverFallback(tab.id, coord, resolvedBy, params.ref);
|
|
}
|
|
}
|
|
case 'left_click':
|
|
case 'right_click': {
|
|
// Calculate CDP modifier mask for click events
|
|
const modifiersMask = CDPHelper.modifierMask(
|
|
[
|
|
params.modifiers?.altKey ? 'alt' : undefined,
|
|
params.modifiers?.ctrlKey ? 'ctrl' : undefined,
|
|
params.modifiers?.metaKey ? 'meta' : undefined,
|
|
params.modifiers?.shiftKey ? 'shift' : undefined,
|
|
].filter((v): v is string => typeof v === 'string'),
|
|
);
|
|
|
|
if (params.ref) {
|
|
// Prefer DOM click via ref
|
|
const domResult = await clickTool.execute({
|
|
ref: params.ref,
|
|
waitForNavigation: false,
|
|
timeout: TIMEOUTS.DEFAULT_WAIT * 5,
|
|
button: params.action === 'right_click' ? 'right' : 'left',
|
|
modifiers: params.modifiers,
|
|
});
|
|
return domResult;
|
|
}
|
|
if (params.selector) {
|
|
// Support selector-based click
|
|
const domResult = await clickTool.execute({
|
|
selector: params.selector,
|
|
selectorType: params.selectorType,
|
|
frameId: params.frameId,
|
|
waitForNavigation: false,
|
|
timeout: TIMEOUTS.DEFAULT_WAIT * 5,
|
|
button: params.action === 'right_click' ? 'right' : 'left',
|
|
modifiers: params.modifiers,
|
|
});
|
|
return domResult;
|
|
}
|
|
if (!params.coordinates)
|
|
return createErrorResponse('Provide ref, selector, or coordinates for click action');
|
|
{
|
|
const stale = ((): any => {
|
|
const getHostname = (url: string): string => {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
const currentHostname = getHostname(tab.url || '');
|
|
const ctx = screenshotContextManager.getContext(tab.id!);
|
|
const contextHostname = (ctx as any)?.hostname as string | undefined;
|
|
if (contextHostname && contextHostname !== currentHostname) {
|
|
return createErrorResponse(
|
|
`Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during ${params.action}. Capture a new screenshot or use ref/selector.`,
|
|
);
|
|
}
|
|
return null;
|
|
})();
|
|
if (stale) return stale;
|
|
}
|
|
const coord = project(params.coordinates)!;
|
|
// Prefer DOM path via existing click tool
|
|
const domResult = await clickTool.execute({
|
|
coordinates: coord,
|
|
waitForNavigation: false,
|
|
timeout: TIMEOUTS.DEFAULT_WAIT * 5,
|
|
button: params.action === 'right_click' ? 'right' : 'left',
|
|
modifiers: params.modifiers,
|
|
});
|
|
if (!domResult.isError) {
|
|
return domResult; // Standardized response from click tool
|
|
}
|
|
// Fallback to CDP if DOM failed
|
|
try {
|
|
await CDPHelper.attach(tab.id);
|
|
const button: MouseButton = params.action === 'right_click' ? 'right' : 'left';
|
|
const clickCount = 1;
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseMoved',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
button: 'none',
|
|
buttons: 0,
|
|
modifiers: modifiersMask,
|
|
});
|
|
for (let i = 1; i <= clickCount; i++) {
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mousePressed',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
button,
|
|
buttons: button === 'left' ? 1 : 2,
|
|
clickCount: i,
|
|
modifiers: modifiersMask,
|
|
});
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseReleased',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
button,
|
|
buttons: 0,
|
|
clickCount: i,
|
|
modifiers: modifiersMask,
|
|
});
|
|
}
|
|
await CDPHelper.detach(tab.id);
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: params.action,
|
|
coordinates: coord,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
await CDPHelper.detach(tab.id);
|
|
return createErrorResponse(
|
|
`CDP click failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
);
|
|
}
|
|
}
|
|
case 'double_click':
|
|
case 'triple_click': {
|
|
// Calculate CDP modifier mask for click events
|
|
const modifiersMask = CDPHelper.modifierMask(
|
|
[
|
|
params.modifiers?.altKey ? 'alt' : undefined,
|
|
params.modifiers?.ctrlKey ? 'ctrl' : undefined,
|
|
params.modifiers?.metaKey ? 'meta' : undefined,
|
|
params.modifiers?.shiftKey ? 'shift' : undefined,
|
|
].filter((v): v is string => typeof v === 'string'),
|
|
);
|
|
|
|
if (!params.coordinates && !params.ref && !params.selector)
|
|
return createErrorResponse(
|
|
'Provide ref, selector, or coordinates for double/triple click',
|
|
);
|
|
let coord = params.coordinates ? project(params.coordinates)! : (undefined as any);
|
|
// If ref is provided, resolve center via accessibility helper
|
|
if (params.ref) {
|
|
try {
|
|
await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
|
|
const resolved = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
|
|
ref: params.ref,
|
|
});
|
|
if (resolved && resolved.success) {
|
|
coord = project({ x: resolved.center.x, y: resolved.center.y })!;
|
|
}
|
|
} catch (e) {
|
|
// ignore and use provided coordinates
|
|
}
|
|
} else if (params.selector) {
|
|
// Support selector-based click
|
|
try {
|
|
await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
|
|
const selectorType = params.selectorType || 'css';
|
|
const ensured = await this.sendMessageToTab(
|
|
tab.id,
|
|
{
|
|
action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
|
|
selector: params.selector,
|
|
isXPath: selectorType === 'xpath',
|
|
},
|
|
params.frameId,
|
|
);
|
|
if (ensured && ensured.success) {
|
|
coord = project({ x: ensured.center.x, y: ensured.center.y })!;
|
|
}
|
|
} catch (e) {
|
|
// ignore
|
|
}
|
|
}
|
|
if (!coord) return createErrorResponse('Failed to resolve coordinates from ref/selector');
|
|
{
|
|
const stale = ((): any => {
|
|
if (!params.coordinates) return null;
|
|
const getHostname = (url: string): string => {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
const currentHostname = getHostname(tab.url || '');
|
|
const ctx = screenshotContextManager.getContext(tab.id!);
|
|
const contextHostname = (ctx as any)?.hostname as string | undefined;
|
|
if (contextHostname && contextHostname !== currentHostname) {
|
|
return createErrorResponse(
|
|
`Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during ${params.action}. Capture a new screenshot or use ref/selector.`,
|
|
);
|
|
}
|
|
return null;
|
|
})();
|
|
if (stale) return stale;
|
|
}
|
|
try {
|
|
await CDPHelper.attach(tab.id);
|
|
const button: MouseButton = 'left';
|
|
const clickCount = params.action === 'double_click' ? 2 : 3;
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseMoved',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
button: 'none',
|
|
buttons: 0,
|
|
modifiers: modifiersMask,
|
|
});
|
|
for (let i = 1; i <= clickCount; i++) {
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mousePressed',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
button,
|
|
buttons: 1,
|
|
clickCount: i,
|
|
modifiers: modifiersMask,
|
|
});
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseReleased',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
button,
|
|
buttons: 0,
|
|
clickCount: i,
|
|
modifiers: modifiersMask,
|
|
});
|
|
}
|
|
await CDPHelper.detach(tab.id);
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: params.action,
|
|
coordinates: coord,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
await CDPHelper.detach(tab.id);
|
|
return createErrorResponse(
|
|
`CDP ${params.action} failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
);
|
|
}
|
|
}
|
|
case 'left_click_drag': {
|
|
if (!params.startCoordinates && !params.startRef)
|
|
return createErrorResponse('Provide startRef or startCoordinates for drag');
|
|
if (!params.coordinates && !params.ref)
|
|
return createErrorResponse('Provide ref or end coordinates for drag');
|
|
let start = params.startCoordinates
|
|
? project(params.startCoordinates)!
|
|
: (undefined as any);
|
|
let end = params.coordinates ? project(params.coordinates)! : (undefined as any);
|
|
{
|
|
const stale = ((): any => {
|
|
if (!params.startCoordinates && !params.coordinates) return null;
|
|
const getHostname = (url: string): string => {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
const currentHostname = getHostname(tab.url || '');
|
|
const ctx = screenshotContextManager.getContext(tab.id!);
|
|
const contextHostname = (ctx as any)?.hostname as string | undefined;
|
|
if (contextHostname && contextHostname !== currentHostname) {
|
|
return createErrorResponse(
|
|
`Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during left_click_drag. Capture a new screenshot or use ref/selector.`,
|
|
);
|
|
}
|
|
return null;
|
|
})();
|
|
if (stale) return stale;
|
|
}
|
|
if (params.startRef || params.ref) {
|
|
await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
|
|
}
|
|
if (params.startRef) {
|
|
try {
|
|
const resolved = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
|
|
ref: params.startRef,
|
|
});
|
|
if (resolved && resolved.success)
|
|
start = project({ x: resolved.center.x, y: resolved.center.y })!;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
if (params.ref) {
|
|
try {
|
|
const resolved = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
|
|
ref: params.ref,
|
|
});
|
|
if (resolved && resolved.success)
|
|
end = project({ x: resolved.center.x, y: resolved.center.y })!;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
if (!start || !end) return createErrorResponse('Failed to resolve drag coordinates');
|
|
try {
|
|
await CDPHelper.attach(tab.id);
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseMoved',
|
|
x: start.x,
|
|
y: start.y,
|
|
button: 'none',
|
|
buttons: 0,
|
|
});
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mousePressed',
|
|
x: start.x,
|
|
y: start.y,
|
|
button: 'left',
|
|
buttons: 1,
|
|
clickCount: 1,
|
|
});
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseMoved',
|
|
x: end.x,
|
|
y: end.y,
|
|
button: 'left',
|
|
buttons: 1,
|
|
});
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseReleased',
|
|
x: end.x,
|
|
y: end.y,
|
|
button: 'left',
|
|
buttons: 0,
|
|
clickCount: 1,
|
|
});
|
|
await CDPHelper.detach(tab.id);
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({ success: true, action: 'left_click_drag', start, end }),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
await CDPHelper.detach(tab.id);
|
|
return createErrorResponse(`Drag failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
}
|
|
}
|
|
case 'scroll': {
|
|
if (!params.coordinates && !params.ref)
|
|
return createErrorResponse('Provide ref or coordinates for scroll');
|
|
let coord = params.coordinates ? project(params.coordinates)! : (undefined as any);
|
|
if (params.ref) {
|
|
try {
|
|
await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
|
|
const resolved = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
|
|
ref: params.ref,
|
|
});
|
|
if (resolved && resolved.success)
|
|
coord = project({ x: resolved.center.x, y: resolved.center.y })!;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
if (!coord) return createErrorResponse('Failed to resolve scroll coordinates');
|
|
{
|
|
const stale = ((): any => {
|
|
if (!params.coordinates) return null;
|
|
const getHostname = (url: string): string => {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
const currentHostname = getHostname(tab.url || '');
|
|
const ctx = screenshotContextManager.getContext(tab.id!);
|
|
const contextHostname = (ctx as any)?.hostname as string | undefined;
|
|
if (contextHostname && contextHostname !== currentHostname) {
|
|
return createErrorResponse(
|
|
`Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during scroll. Capture a new screenshot or use ref/selector.`,
|
|
);
|
|
}
|
|
return null;
|
|
})();
|
|
if (stale) return stale;
|
|
}
|
|
const direction = params.scrollDirection || 'down';
|
|
const amount = Math.max(1, Math.min(params.scrollAmount || 3, 10));
|
|
// Convert to deltas (~100px per tick)
|
|
const unit = 100;
|
|
let deltaX = 0,
|
|
deltaY = 0;
|
|
if (direction === 'up') deltaY = -amount * unit;
|
|
if (direction === 'down') deltaY = amount * unit;
|
|
if (direction === 'left') deltaX = -amount * unit;
|
|
if (direction === 'right') deltaX = amount * unit;
|
|
try {
|
|
await CDPHelper.attach(tab.id);
|
|
await CDPHelper.dispatchMouseEvent(tab.id, {
|
|
type: 'mouseWheel',
|
|
x: coord.x,
|
|
y: coord.y,
|
|
deltaX,
|
|
deltaY,
|
|
});
|
|
await CDPHelper.detach(tab.id);
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'scroll',
|
|
coordinates: coord,
|
|
deltaX,
|
|
deltaY,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
await CDPHelper.detach(tab.id);
|
|
return createErrorResponse(
|
|
`Scroll failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
);
|
|
}
|
|
}
|
|
case 'type': {
|
|
if (!params.text) return createErrorResponse('Text parameter is required for type action');
|
|
try {
|
|
// Optional focus via ref before typing
|
|
if (params.ref) {
|
|
await clickTool.execute({
|
|
ref: params.ref,
|
|
waitForNavigation: false,
|
|
timeout: TIMEOUTS.DEFAULT_WAIT * 5,
|
|
});
|
|
}
|
|
await CDPHelper.attach(tab.id);
|
|
// Use CDP insertText to avoid complex KeyboardEvent emulation for long text
|
|
await CDPHelper.insertText(tab.id, params.text);
|
|
await CDPHelper.detach(tab.id);
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'type',
|
|
length: params.text.length,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
await CDPHelper.detach(tab.id);
|
|
// Fallback to DOM-based keyboard tool
|
|
const res = await keyboardTool.execute({
|
|
keys: params.text.split('').join(','),
|
|
delay: 0,
|
|
selector: undefined,
|
|
});
|
|
return res;
|
|
}
|
|
}
|
|
case 'fill': {
|
|
if (!params.ref && !params.selector) {
|
|
return createErrorResponse('Provide ref or selector and a value for fill');
|
|
}
|
|
// Reuse existing fill tool to leverage robust DOM event behavior
|
|
const res = await fillTool.execute({
|
|
selector: params.selector as any,
|
|
selectorType: params.selectorType as any,
|
|
ref: params.ref as any,
|
|
value: params.value as any,
|
|
} as any);
|
|
return res;
|
|
}
|
|
case 'fill_form': {
|
|
const elements = (params as any).elements as Array<{
|
|
ref: string;
|
|
value: string | number | boolean;
|
|
}>;
|
|
if (!Array.isArray(elements) || elements.length === 0) {
|
|
return createErrorResponse('elements must be a non-empty array for fill_form');
|
|
}
|
|
const results: Array<{ ref: string; ok: boolean; error?: string }> = [];
|
|
for (const item of elements) {
|
|
if (!item || !item.ref) {
|
|
results.push({ ref: String(item?.ref || ''), ok: false, error: 'missing ref' });
|
|
continue;
|
|
}
|
|
try {
|
|
const r = await fillTool.execute({
|
|
ref: item.ref as any,
|
|
value: item.value as any,
|
|
} as any);
|
|
const ok = !r.isError;
|
|
results.push({ ref: item.ref, ok, error: ok ? undefined : 'failed' });
|
|
} catch (e) {
|
|
results.push({
|
|
ref: item.ref,
|
|
ok: false,
|
|
error: String(e instanceof Error ? e.message : e),
|
|
});
|
|
}
|
|
}
|
|
const successCount = results.filter((r) => r.ok).length;
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'fill_form',
|
|
filled: successCount,
|
|
total: results.length,
|
|
results,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
}
|
|
case 'key': {
|
|
if (!params.text)
|
|
return createErrorResponse(
|
|
'text is required for key action (e.g., "Backspace Backspace Enter" or "cmd+a")',
|
|
);
|
|
const tokens = params.text.trim().split(/\s+/).filter(Boolean);
|
|
const repeat = params.repeat ?? 1;
|
|
if (!Number.isInteger(repeat) || repeat < 1 || repeat > 100) {
|
|
return createErrorResponse('repeat must be an integer between 1 and 100 for key action');
|
|
}
|
|
try {
|
|
// Optional focus via ref before key events
|
|
if (params.ref) {
|
|
await clickTool.execute({
|
|
ref: params.ref,
|
|
waitForNavigation: false,
|
|
timeout: TIMEOUTS.DEFAULT_WAIT * 5,
|
|
});
|
|
}
|
|
await CDPHelper.attach(tab.id);
|
|
for (let i = 0; i < repeat; i++) {
|
|
for (const t of tokens) {
|
|
if (t.includes('+')) await CDPHelper.dispatchKeyChord(tab.id, t);
|
|
else await CDPHelper.dispatchSimpleKey(tab.id, t);
|
|
}
|
|
}
|
|
await CDPHelper.detach(tab.id);
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({ success: true, action: 'key', keys: tokens, repeat }),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
await CDPHelper.detach(tab.id);
|
|
// Fallback to DOM keyboard simulation (comma-separated combinations)
|
|
const keysStr = tokens.join(',');
|
|
const repeatedKeys =
|
|
repeat === 1 ? keysStr : Array.from({ length: repeat }, () => keysStr).join(',');
|
|
const res = await keyboardTool.execute({ keys: repeatedKeys });
|
|
return res;
|
|
}
|
|
}
|
|
case 'wait': {
|
|
const hasTextCondition =
|
|
typeof (params as any).text === 'string' && (params as any).text.trim().length > 0;
|
|
if (hasTextCondition) {
|
|
try {
|
|
// Conditional wait for text appearance/disappearance using content script
|
|
await this.injectContentScript(
|
|
tab.id,
|
|
['inject-scripts/wait-helper.js'],
|
|
false,
|
|
'ISOLATED',
|
|
true,
|
|
);
|
|
const appear = (params as any).appear !== false; // default to true
|
|
const timeoutMs = Math.max(
|
|
0,
|
|
Math.min(((params as any).timeout as number) || 10000, 120000),
|
|
);
|
|
const resp = await this.sendMessageToTab(tab.id, {
|
|
action: TOOL_MESSAGE_TYPES.WAIT_FOR_TEXT,
|
|
text: (params as any).text,
|
|
appear,
|
|
timeout: timeoutMs,
|
|
});
|
|
if (!resp || resp.success !== true) {
|
|
return createErrorResponse(
|
|
resp && resp.reason === 'timeout'
|
|
? `wait_for timed out after ${timeoutMs}ms for text: ${(params as any).text}`
|
|
: `wait_for failed: ${resp && resp.error ? resp.error : 'unknown error'}`,
|
|
);
|
|
}
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'wait_for',
|
|
appear,
|
|
text: (params as any).text,
|
|
matched: resp.matched || null,
|
|
tookMs: resp.tookMs,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
return createErrorResponse(
|
|
`wait_for failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
);
|
|
}
|
|
} else {
|
|
const seconds = Math.max(0, Math.min((params as any).duration || 0, 30));
|
|
if (!seconds)
|
|
return createErrorResponse('Duration parameter is required and must be > 0');
|
|
await new Promise((r) => setTimeout(r, seconds * 1000));
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({ success: true, action: 'wait', duration: seconds }),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
}
|
|
}
|
|
case 'scroll_to': {
|
|
if (!params.ref) {
|
|
return createErrorResponse('ref is required for scroll_to action');
|
|
}
|
|
try {
|
|
await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
|
|
const resp = await this.sendMessageToTab(tab.id, {
|
|
action: 'focusByRef',
|
|
ref: params.ref,
|
|
});
|
|
if (!resp || resp.success !== true) {
|
|
return createErrorResponse(resp?.error || 'scroll_to failed: element not found');
|
|
}
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'scroll_to',
|
|
ref: params.ref,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
return createErrorResponse(
|
|
`scroll_to failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
);
|
|
}
|
|
}
|
|
case 'zoom': {
|
|
const region = params.region;
|
|
if (!region) {
|
|
return createErrorResponse('region is required for zoom action');
|
|
}
|
|
const x0 = Number(region.x0);
|
|
const y0 = Number(region.y0);
|
|
const x1 = Number(region.x1);
|
|
const y1 = Number(region.y1);
|
|
if (![x0, y0, x1, y1].every(Number.isFinite)) {
|
|
return createErrorResponse('region must contain finite numbers (x0, y0, x1, y1)');
|
|
}
|
|
if (x0 < 0 || y0 < 0 || x1 <= x0 || y1 <= y0) {
|
|
return createErrorResponse('Invalid region: require x0>=0, y0>=0 and x1>x0, y1>y0');
|
|
}
|
|
|
|
// Project coordinates from screenshot space to viewport space
|
|
const p0 = project({ x: x0, y: y0 })!;
|
|
const p1 = project({ x: x1, y: y1 })!;
|
|
const rx0 = Math.min(p0.x, p1.x);
|
|
const ry0 = Math.min(p0.y, p1.y);
|
|
const rx1 = Math.max(p0.x, p1.x);
|
|
const ry1 = Math.max(p0.y, p1.y);
|
|
const w = rx1 - rx0;
|
|
const h = ry1 - ry0;
|
|
if (w <= 0 || h <= 0) {
|
|
return createErrorResponse('Invalid region after projection');
|
|
}
|
|
|
|
// Security check: verify domain hasn't changed since last screenshot
|
|
{
|
|
const getHostname = (url: string): string => {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
const ctx = screenshotContextManager.getContext(tab.id!);
|
|
const contextHostname = (ctx as any)?.hostname as string | undefined;
|
|
const currentHostname = getHostname(tab.url || '');
|
|
if (contextHostname && contextHostname !== currentHostname) {
|
|
return createErrorResponse(
|
|
`Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during zoom. Capture a new screenshot first.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
try {
|
|
await CDPHelper.attach(tab.id);
|
|
const metrics: any = await CDPHelper.send(tab.id, 'Page.getLayoutMetrics', {});
|
|
const viewport = metrics?.layoutViewport ||
|
|
metrics?.visualViewport || {
|
|
clientWidth: 800,
|
|
clientHeight: 600,
|
|
pageX: 0,
|
|
pageY: 0,
|
|
};
|
|
const vw = Math.round(Number(viewport.clientWidth || 800));
|
|
const vh = Math.round(Number(viewport.clientHeight || 600));
|
|
if (rx1 > vw || ry1 > vh) {
|
|
await CDPHelper.detach(tab.id);
|
|
return createErrorResponse(
|
|
`Region exceeds viewport boundaries (${vw}x${vh}). Choose a region within the visible viewport.`,
|
|
);
|
|
}
|
|
const pageX = Number(viewport.pageX || 0);
|
|
const pageY = Number(viewport.pageY || 0);
|
|
|
|
const shot: any = await CDPHelper.send(tab.id, 'Page.captureScreenshot', {
|
|
format: 'png',
|
|
captureBeyondViewport: false,
|
|
fromSurface: true,
|
|
clip: {
|
|
x: pageX + rx0,
|
|
y: pageY + ry0,
|
|
width: w,
|
|
height: h,
|
|
scale: 1,
|
|
},
|
|
});
|
|
await CDPHelper.detach(tab.id);
|
|
|
|
const base64Data = String(shot?.data || '');
|
|
if (!base64Data) {
|
|
return createErrorResponse('Failed to capture zoom screenshot via CDP');
|
|
}
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'zoom',
|
|
mimeType: 'image/png',
|
|
base64Data,
|
|
region: { x0: rx0, y0: ry0, x1: rx1, y1: ry1 },
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (e) {
|
|
await CDPHelper.detach(tab.id);
|
|
return createErrorResponse(`zoom failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
}
|
|
}
|
|
case 'screenshot': {
|
|
// Reuse existing screenshot tool; it already supports base64 save option
|
|
const result = await screenshotTool.execute({
|
|
name: 'computer',
|
|
storeBase64: true,
|
|
fullPage: false,
|
|
});
|
|
return result;
|
|
}
|
|
default:
|
|
return createErrorResponse(`Unsupported action: ${params.action}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* DOM-based hover fallback when CDP is unavailable
|
|
* Tries ref-based approach first (works with iframes), falls back to coordinates
|
|
*/
|
|
private async domHoverFallback(
|
|
tabId: number,
|
|
coord?: Coordinates,
|
|
resolvedBy?: 'ref' | 'selector' | 'coordinates',
|
|
ref?: string,
|
|
): Promise<ToolResult> {
|
|
// Try ref-based approach first (handles iframes correctly)
|
|
if (ref) {
|
|
try {
|
|
const resp = await this.sendMessageToTab(tabId, {
|
|
action: TOOL_MESSAGE_TYPES.DISPATCH_HOVER_FOR_REF,
|
|
ref,
|
|
});
|
|
if (resp?.success) {
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'hover',
|
|
resolvedBy: 'ref',
|
|
transport: 'dom-ref',
|
|
target: resp.target,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
}
|
|
} catch (error) {
|
|
console.warn('[ComputerTool] DOM ref hover failed, falling back to coordinates', error);
|
|
}
|
|
}
|
|
|
|
// Fallback to coordinate-based approach
|
|
if (!coord) {
|
|
return createErrorResponse('Hover fallback requires coordinates or ref');
|
|
}
|
|
|
|
try {
|
|
const [injection] = await chrome.scripting.executeScript({
|
|
target: { tabId },
|
|
world: 'MAIN',
|
|
func: (point) => {
|
|
const target = document.elementFromPoint(point.x, point.y);
|
|
if (!target) {
|
|
return { success: false, error: 'No element found at coordinates' };
|
|
}
|
|
|
|
// Dispatch hover-related events
|
|
for (const type of ['mousemove', 'mouseover', 'mouseenter']) {
|
|
target.dispatchEvent(
|
|
new MouseEvent(type, {
|
|
bubbles: true,
|
|
cancelable: true,
|
|
clientX: point.x,
|
|
clientY: point.y,
|
|
view: window,
|
|
}),
|
|
);
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
target: {
|
|
tagName: target.tagName,
|
|
id: target.id,
|
|
className: target.className,
|
|
text: target.textContent?.trim()?.slice(0, 100) || '',
|
|
},
|
|
};
|
|
},
|
|
args: [coord],
|
|
});
|
|
|
|
const payload = injection?.result;
|
|
if (!payload?.success) {
|
|
return createErrorResponse(payload?.error || 'DOM hover fallback failed');
|
|
}
|
|
|
|
return {
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: JSON.stringify({
|
|
success: true,
|
|
action: 'hover',
|
|
coordinates: coord,
|
|
resolvedBy,
|
|
transport: 'dom',
|
|
target: payload.target,
|
|
}),
|
|
},
|
|
],
|
|
isError: false,
|
|
};
|
|
} catch (error) {
|
|
return createErrorResponse(
|
|
`DOM hover fallback failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Trigger GIF auto-capture after a successful action.
|
|
* This is a no-op if auto-capture is not active.
|
|
*/
|
|
private async triggerAutoCapture(
|
|
tabId: number,
|
|
actionType: ActionType,
|
|
metadata?: Partial<ActionMetadata>,
|
|
): Promise<void> {
|
|
if (!isAutoCaptureActive(tabId)) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
await captureFrameOnAction(tabId, {
|
|
type: actionType,
|
|
...metadata,
|
|
});
|
|
} catch (error) {
|
|
// Log but don't fail the main action
|
|
console.warn('[ComputerTool] Auto-capture failed:', error);
|
|
}
|
|
}
|
|
}
|
|
|
|
export const computerTool = new ComputerTool();
|