hangwin--mcp-chrome
hangerye
558 行
18 KiB
TypeScript
558 行
18 KiB
TypeScript
/**
|
|
* Element Picker Tool
|
|
*
|
|
* Implements chrome_request_element_selection - a human-in-the-loop tool that allows
|
|
* users to manually select elements on the page when AI cannot reliably locate them.
|
|
*/
|
|
|
|
import { createErrorResponse, type ToolResult } from '@/common/tool-handler';
|
|
import { BaseBrowserToolExecutor } from '../base-browser';
|
|
import { BACKGROUND_MESSAGE_TYPES, TOOL_MESSAGE_TYPES } from '@/common/message-types';
|
|
import { ERROR_MESSAGES } from '@/common/constants';
|
|
import {
|
|
TOOL_NAMES,
|
|
type ElementPickerRequest,
|
|
type ElementPickerResult,
|
|
type ElementPickerResultItem,
|
|
type PickedElement,
|
|
} from 'chrome-mcp-shared';
|
|
|
|
// ============================================================
|
|
// Types
|
|
// ============================================================
|
|
|
|
interface NormalizedRequest {
|
|
id: string;
|
|
name: string;
|
|
description?: string;
|
|
}
|
|
|
|
interface ElementPickerToolParams {
|
|
requests: ElementPickerRequest[];
|
|
timeoutMs?: number;
|
|
tabId?: number;
|
|
windowId?: number;
|
|
}
|
|
|
|
interface PickerUiEvent {
|
|
type: string;
|
|
sessionId: string;
|
|
event: 'cancel' | 'confirm' | 'set_active_request' | 'clear_selection';
|
|
requestId?: string;
|
|
}
|
|
|
|
interface PickerFrameEvent {
|
|
type: string;
|
|
sessionId: string;
|
|
event: 'selected' | 'cancel';
|
|
requestId?: string;
|
|
element?: Omit<PickedElement, 'frameId'>;
|
|
}
|
|
|
|
// ============================================================
|
|
// Constants
|
|
// ============================================================
|
|
|
|
const DEFAULT_TIMEOUT_MS = 3 * 60 * 1000; // 3 minutes
|
|
const MAX_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
|
|
const MIN_TIMEOUT_MS = 10 * 1000; // 10 seconds
|
|
|
|
// ============================================================
|
|
// Utility Functions
|
|
// ============================================================
|
|
|
|
function toTrimmedString(value: unknown): string {
|
|
return typeof value === 'string' ? value.trim() : '';
|
|
}
|
|
|
|
function normalizeTimeoutMs(value: unknown): number {
|
|
if (value === undefined || value === null) return DEFAULT_TIMEOUT_MS;
|
|
const n = Number(value);
|
|
if (!Number.isFinite(n) || n <= 0) return DEFAULT_TIMEOUT_MS;
|
|
return Math.min(Math.max(Math.floor(n), MIN_TIMEOUT_MS), MAX_TIMEOUT_MS);
|
|
}
|
|
|
|
function normalizeRequests(requests: ElementPickerRequest[]): NormalizedRequest[] {
|
|
const out: NormalizedRequest[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (let i = 0; i < requests.length; i++) {
|
|
const r = requests[i] || ({} as ElementPickerRequest);
|
|
const name = toTrimmedString(r.name);
|
|
if (!name) continue;
|
|
|
|
// Generate or use provided ID, ensuring uniqueness
|
|
const baseId = toTrimmedString(r.id) || `req_${i + 1}`;
|
|
let id = baseId;
|
|
let suffix = 2;
|
|
while (seen.has(id)) {
|
|
id = `${baseId}_${suffix++}`;
|
|
}
|
|
seen.add(id);
|
|
|
|
const description = toTrimmedString(r.description);
|
|
out.push({ id, name, description: description || undefined });
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
function buildResultItems(
|
|
requests: NormalizedRequest[],
|
|
pickedById: Map<string, PickedElement>,
|
|
): ElementPickerResultItem[] {
|
|
return requests.map((r) => ({
|
|
id: r.id,
|
|
name: r.name,
|
|
element: pickedById.get(r.id) || null,
|
|
}));
|
|
}
|
|
|
|
function listMissingRequestIds(
|
|
requests: NormalizedRequest[],
|
|
pickedById: Map<string, PickedElement>,
|
|
): string[] {
|
|
const missing: string[] = [];
|
|
for (const r of requests) {
|
|
if (!pickedById.has(r.id)) missing.push(r.id);
|
|
}
|
|
return missing;
|
|
}
|
|
|
|
// ============================================================
|
|
// Element Picker Tool
|
|
// ============================================================
|
|
|
|
class ElementPickerTool extends BaseBrowserToolExecutor {
|
|
name = TOOL_NAMES.BROWSER.REQUEST_ELEMENT_SELECTION;
|
|
|
|
/**
|
|
* Inject picker scripts into all frames of the tab.
|
|
*/
|
|
private async injectPickerScripts(tabId: number): Promise<void> {
|
|
await chrome.scripting.executeScript({
|
|
target: { tabId, allFrames: true },
|
|
files: ['inject-scripts/element-picker.js'],
|
|
world: 'ISOLATED',
|
|
injectImmediately: false,
|
|
} as any);
|
|
}
|
|
|
|
/**
|
|
* Call the picker API in all frames via scripting.executeScript.
|
|
*/
|
|
private async callPickerApi(
|
|
tabId: number,
|
|
method: 'startSession' | 'stopSession' | 'setActiveRequest',
|
|
payload: Record<string, unknown>,
|
|
): Promise<void> {
|
|
await chrome.scripting.executeScript({
|
|
target: { tabId, allFrames: true },
|
|
world: 'ISOLATED',
|
|
injectImmediately: false,
|
|
func: (methodName: string, data: Record<string, unknown>) => {
|
|
try {
|
|
const api = (
|
|
globalThis as unknown as {
|
|
__mcpElementPicker?: Record<string, (data: Record<string, unknown>) => void>;
|
|
}
|
|
).__mcpElementPicker;
|
|
const fn = api && api[methodName];
|
|
if (typeof fn === 'function') {
|
|
fn(data);
|
|
}
|
|
} catch {
|
|
// Best-effort
|
|
}
|
|
},
|
|
args: [method, payload],
|
|
} as any);
|
|
}
|
|
|
|
async execute(args: ElementPickerToolParams): Promise<ToolResult> {
|
|
// Validate requests
|
|
const rawRequests = Array.isArray(args?.requests) ? args.requests : [];
|
|
if (rawRequests.length === 0) {
|
|
return createErrorResponse(`${ERROR_MESSAGES.INVALID_PARAMETERS}: requests[] is required`);
|
|
}
|
|
|
|
const requests = normalizeRequests(rawRequests);
|
|
if (requests.length === 0) {
|
|
return createErrorResponse(
|
|
`${ERROR_MESSAGES.INVALID_PARAMETERS}: requests[] must contain at least one non-empty name`,
|
|
);
|
|
}
|
|
|
|
const timeoutMs = normalizeTimeoutMs(args?.timeoutMs);
|
|
const sessionId = `ep_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
|
|
const deadlineTs = Date.now() + timeoutMs;
|
|
|
|
// Resolve tab
|
|
let tab: chrome.tabs.Tab;
|
|
try {
|
|
const explicit = await this.tryGetTab(args?.tabId);
|
|
tab = explicit || (await this.getActiveTabOrThrowInWindow(args?.windowId));
|
|
} catch (error) {
|
|
return createErrorResponse(
|
|
`${ERROR_MESSAGES.TAB_NOT_FOUND}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
if (!tab.id) {
|
|
return createErrorResponse(`${ERROR_MESSAGES.TAB_NOT_FOUND}: Active tab has no ID`);
|
|
}
|
|
const tabId = tab.id;
|
|
|
|
// Focus the tab/window for user interaction
|
|
try {
|
|
await this.ensureFocus(tab, { activate: true, focusWindow: true });
|
|
} catch {
|
|
// Best-effort: some environments disallow focusing
|
|
}
|
|
|
|
// State tracking
|
|
const pickedById = new Map<string, PickedElement>();
|
|
let activeRequestId: string | null = requests[0]?.id || null;
|
|
let uiErrorMessage: string | null = null;
|
|
let uiAvailable = true;
|
|
|
|
let finished = false;
|
|
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
let resolveResult: ((result: ElementPickerResult) => void) | null = null;
|
|
|
|
// Send UI update to content script
|
|
const sendUiUpdate = async (): Promise<void> => {
|
|
if (!uiAvailable) return;
|
|
try {
|
|
const selections: Record<string, PickedElement | null> = {};
|
|
for (const r of requests) {
|
|
selections[r.id] = pickedById.get(r.id) || null;
|
|
}
|
|
await this.sendMessageToTab(
|
|
tabId,
|
|
{
|
|
action: TOOL_MESSAGE_TYPES.ELEMENT_PICKER_UI_UPDATE,
|
|
sessionId,
|
|
activeRequestId,
|
|
selections,
|
|
deadlineTs,
|
|
errorMessage: uiErrorMessage,
|
|
},
|
|
0, // Top frame only for UI
|
|
);
|
|
} catch {
|
|
uiAvailable = false;
|
|
}
|
|
};
|
|
|
|
// Set the active request and notify all frames + UI
|
|
const setActiveRequest = async (requestId: string | null): Promise<void> => {
|
|
activeRequestId = requestId;
|
|
await this.callPickerApi(tabId, 'setActiveRequest', {
|
|
sessionId,
|
|
activeRequestId: requestId,
|
|
});
|
|
await sendUiUpdate();
|
|
};
|
|
|
|
// Finish the tool execution
|
|
const finish = async (final: {
|
|
success: boolean;
|
|
cancelled?: boolean;
|
|
timedOut?: boolean;
|
|
}): Promise<void> => {
|
|
if (finished) return;
|
|
finished = true;
|
|
|
|
if (timer !== null) {
|
|
clearTimeout(timer);
|
|
timer = null;
|
|
}
|
|
|
|
chrome.runtime.onMessage.removeListener(onRuntimeMessage);
|
|
|
|
// Cleanup: stop picker in all frames and hide UI
|
|
await Promise.allSettled([
|
|
this.callPickerApi(tabId, 'stopSession', { sessionId }),
|
|
uiAvailable
|
|
? this.sendMessageToTab(
|
|
tabId,
|
|
{ action: TOOL_MESSAGE_TYPES.ELEMENT_PICKER_UI_HIDE, sessionId },
|
|
0,
|
|
)
|
|
: Promise.resolve(),
|
|
]);
|
|
|
|
const missing = listMissingRequestIds(requests, pickedById);
|
|
const result: ElementPickerResult = {
|
|
success: final.success,
|
|
sessionId,
|
|
timeoutMs,
|
|
cancelled: final.cancelled,
|
|
timedOut: final.timedOut,
|
|
missingRequestIds: missing.length > 0 ? missing : undefined,
|
|
results: buildResultItems(requests, pickedById),
|
|
};
|
|
|
|
resolveResult?.(result);
|
|
};
|
|
|
|
// Handle messages from content scripts
|
|
const onRuntimeMessage = (
|
|
message: unknown,
|
|
sender: chrome.runtime.MessageSender,
|
|
sendResponse: (response?: unknown) => void,
|
|
): boolean | void => {
|
|
const senderTabId = sender?.tab?.id;
|
|
if (senderTabId !== tabId) return;
|
|
|
|
const msg = message as Partial<PickerUiEvent & PickerFrameEvent> | undefined;
|
|
if (!msg || msg.sessionId !== sessionId) return;
|
|
|
|
// Handle frame events (element selection)
|
|
if (msg.type === BACKGROUND_MESSAGE_TYPES.ELEMENT_PICKER_FRAME_EVENT) {
|
|
if (msg.event === 'cancel') {
|
|
void finish({ success: false, cancelled: true });
|
|
sendResponse?.({ success: true });
|
|
return true;
|
|
}
|
|
|
|
if (msg.event === 'selected') {
|
|
const requestId = toTrimmedString(msg.requestId);
|
|
const frameId = typeof sender.frameId === 'number' ? sender.frameId : 0;
|
|
|
|
// Validate request ID
|
|
const reqExists = requestId && requests.some((r) => r.id === requestId);
|
|
if (!reqExists) {
|
|
sendResponse?.({ success: false, error: 'Unknown requestId' });
|
|
return true;
|
|
}
|
|
|
|
// Validate element data
|
|
const raw = (msg.element || {}) as Partial<Omit<PickedElement, 'frameId'>>;
|
|
const ref = toTrimmedString(raw.ref);
|
|
if (!ref) {
|
|
sendResponse?.({ success: false, error: 'Missing element.ref' });
|
|
return true;
|
|
}
|
|
|
|
// Build picked element with frameId
|
|
const selector = toTrimmedString(raw.selector);
|
|
const rect = raw.rect as PickedElement['rect'] | undefined;
|
|
const center = raw.center as PickedElement['center'] | undefined;
|
|
const picked: PickedElement = {
|
|
ref,
|
|
selector,
|
|
selectorType: 'css',
|
|
rect: rect && typeof rect === 'object' ? rect : { x: 0, y: 0, width: 0, height: 0 },
|
|
center: center && typeof center === 'object' ? center : { x: 0, y: 0 },
|
|
text: typeof raw.text === 'string' ? raw.text : undefined,
|
|
tagName: typeof raw.tagName === 'string' ? raw.tagName : undefined,
|
|
frameId,
|
|
};
|
|
|
|
pickedById.set(requestId, picked);
|
|
uiErrorMessage = null;
|
|
|
|
// Auto-advance to next missing request
|
|
const missing = listMissingRequestIds(requests, pickedById);
|
|
const next = missing.length > 0 ? missing[0] : null;
|
|
|
|
void (async () => {
|
|
try {
|
|
if (next) {
|
|
await setActiveRequest(next);
|
|
} else {
|
|
// All selected: update UI (user still needs to confirm)
|
|
await sendUiUpdate();
|
|
// If UI is unavailable, auto-confirm
|
|
if (!uiAvailable) {
|
|
await finish({ success: true });
|
|
}
|
|
}
|
|
} catch {
|
|
// Best-effort
|
|
}
|
|
})();
|
|
|
|
sendResponse?.({ success: true });
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Handle UI events (cancel, confirm, etc.)
|
|
if (msg.type === BACKGROUND_MESSAGE_TYPES.ELEMENT_PICKER_UI_EVENT) {
|
|
if (msg.event === 'cancel') {
|
|
void finish({ success: false, cancelled: true });
|
|
sendResponse?.({ success: true });
|
|
return true;
|
|
}
|
|
|
|
if (msg.event === 'confirm') {
|
|
const missing = listMissingRequestIds(requests, pickedById);
|
|
if (missing.length > 0) {
|
|
uiErrorMessage = `Please select all elements: missing ${missing.join(', ')}`;
|
|
void sendUiUpdate();
|
|
sendResponse?.({ success: false, error: 'missing_selections', missing });
|
|
return true;
|
|
}
|
|
void finish({ success: true });
|
|
sendResponse?.({ success: true });
|
|
return true;
|
|
}
|
|
|
|
if (msg.event === 'set_active_request') {
|
|
const requestId = toTrimmedString(msg.requestId);
|
|
if (!requestId || !requests.some((r) => r.id === requestId)) {
|
|
sendResponse?.({ success: false, error: 'Unknown requestId' });
|
|
return true;
|
|
}
|
|
void setActiveRequest(requestId);
|
|
sendResponse?.({ success: true });
|
|
return true;
|
|
}
|
|
|
|
if (msg.event === 'clear_selection') {
|
|
const requestId = toTrimmedString(msg.requestId);
|
|
if (!requestId || !requests.some((r) => r.id === requestId)) {
|
|
sendResponse?.({ success: false, error: 'Unknown requestId' });
|
|
return true;
|
|
}
|
|
pickedById.delete(requestId);
|
|
uiErrorMessage = null;
|
|
void setActiveRequest(requestId);
|
|
sendResponse?.({ success: true });
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return;
|
|
};
|
|
|
|
try {
|
|
// Step 1: Ensure UI content script is ready (ping + inject fallback)
|
|
const ensureUiReady = async (): Promise<boolean> => {
|
|
// Try to ping UI content script with retries
|
|
const pingWithTimeout = async (timeoutMs = 500): Promise<boolean> => {
|
|
try {
|
|
const resp = await Promise.race([
|
|
this.sendMessageToTab(
|
|
tabId,
|
|
{ action: TOOL_MESSAGE_TYPES.ELEMENT_PICKER_UI_PING },
|
|
0,
|
|
),
|
|
new Promise<null>((_, reject) =>
|
|
setTimeout(() => reject(new Error('Ping timeout')), timeoutMs),
|
|
),
|
|
]);
|
|
return resp?.success === true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
|
|
// First ping attempt (content script may already be loaded)
|
|
if (await pingWithTimeout()) return true;
|
|
|
|
// Try to inject UI content script as fallback
|
|
// Try multiple possible paths (production vs dev builds)
|
|
const possiblePaths = ['content-scripts/element-picker.js', 'element-picker.js'];
|
|
|
|
for (const path of possiblePaths) {
|
|
try {
|
|
await chrome.scripting.executeScript({
|
|
target: { tabId, frameIds: [0] },
|
|
files: [path],
|
|
injectImmediately: true,
|
|
} as any);
|
|
// Wait a bit for script to initialize
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
// Check if injection worked
|
|
if (await pingWithTimeout(300)) return true;
|
|
} catch (e) {
|
|
// Try next path
|
|
console.debug(`[ElementPicker] Path ${path} failed:`, e);
|
|
}
|
|
}
|
|
|
|
// Final attempt with longer timeout (in case of slow page)
|
|
return pingWithTimeout(1000);
|
|
};
|
|
|
|
const uiReady = await ensureUiReady();
|
|
if (!uiReady) {
|
|
console.error('[ElementPicker] UI not available after all attempts');
|
|
return createErrorResponse(
|
|
`${ERROR_MESSAGES.TOOL_EXECUTION_FAILED}: Element Picker UI is not available. This may happen if: (1) The page blocks content scripts, (2) You're using dev mode - try restarting the dev server or use production build, (3) The page needs to be refreshed.`,
|
|
);
|
|
}
|
|
|
|
// Step 2: Show UI in top frame (must receive success:true)
|
|
try {
|
|
const showResp = await this.sendMessageToTab(
|
|
tabId,
|
|
{
|
|
action: TOOL_MESSAGE_TYPES.ELEMENT_PICKER_UI_SHOW,
|
|
sessionId,
|
|
requests,
|
|
activeRequestId,
|
|
deadlineTs,
|
|
},
|
|
0,
|
|
);
|
|
if (showResp?.success !== true) {
|
|
throw new Error('UI did not acknowledge show message');
|
|
}
|
|
} catch (e) {
|
|
console.error('[ElementPicker] UI show failed:', e);
|
|
return createErrorResponse(
|
|
`${ERROR_MESSAGES.TOOL_EXECUTION_FAILED}: Failed to show Element Picker UI. Please refresh the page and try again.`,
|
|
);
|
|
}
|
|
|
|
// Step 3: Inject picker scripts and start selection engine in all frames
|
|
await this.injectPickerScripts(tabId);
|
|
await this.callPickerApi(tabId, 'startSession', { sessionId, activeRequestId });
|
|
|
|
// Register message listener
|
|
chrome.runtime.onMessage.addListener(onRuntimeMessage);
|
|
|
|
// Create result promise
|
|
const resultPromise = new Promise<ElementPickerResult>((resolve) => {
|
|
resolveResult = resolve;
|
|
});
|
|
|
|
// Set timeout
|
|
timer = setTimeout(() => {
|
|
void finish({ success: false, timedOut: true });
|
|
}, timeoutMs);
|
|
|
|
// Initial UI update
|
|
void sendUiUpdate();
|
|
|
|
// Wait for result
|
|
const result = await resultPromise;
|
|
return { content: [{ type: 'text', text: JSON.stringify(result) }], isError: false };
|
|
} catch (error) {
|
|
console.error('Error in element picker tool:', error);
|
|
// Cleanup on error
|
|
try {
|
|
await Promise.allSettled([
|
|
this.callPickerApi(tabId, 'stopSession', { sessionId }),
|
|
this.sendMessageToTab(
|
|
tabId,
|
|
{ action: TOOL_MESSAGE_TYPES.ELEMENT_PICKER_UI_HIDE, sessionId },
|
|
0,
|
|
),
|
|
]);
|
|
} catch {
|
|
// Best-effort cleanup
|
|
}
|
|
return createErrorResponse(
|
|
`${ERROR_MESSAGES.TOOL_EXECUTION_FAILED}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
export const elementPickerTool = new ElementPickerTool();
|