334 lines
11 KiB
TypeScript
334 lines
11 KiB
TypeScript
import path from "node:path";
|
|
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
|
import { captureScreenshot, snapshotAria } from "../cdp.js";
|
|
import {
|
|
DEFAULT_AI_SNAPSHOT_EFFICIENT_DEPTH,
|
|
DEFAULT_AI_SNAPSHOT_EFFICIENT_MAX_CHARS,
|
|
DEFAULT_AI_SNAPSHOT_MAX_CHARS,
|
|
} from "../constants.js";
|
|
import { withBrowserNavigationPolicy } from "../navigation-guard.js";
|
|
import {
|
|
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
|
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
|
normalizeBrowserScreenshot,
|
|
} from "../screenshot.js";
|
|
import type { BrowserRouteContext } from "../server-context.js";
|
|
import {
|
|
getPwAiModule,
|
|
handleRouteError,
|
|
readBody,
|
|
requirePwAi,
|
|
resolveProfileContext,
|
|
withPlaywrightRouteContext,
|
|
withRouteTabContext,
|
|
} from "./agent.shared.js";
|
|
import type { BrowserResponse, BrowserRouteRegistrar } from "./types.js";
|
|
import { jsonError, toBoolean, toNumber, toStringOrEmpty } from "./utils.js";
|
|
|
|
async function saveBrowserMediaResponse(params: {
|
|
res: BrowserResponse;
|
|
buffer: Buffer;
|
|
contentType: string;
|
|
maxBytes: number;
|
|
targetId: string;
|
|
url: string;
|
|
}) {
|
|
await ensureMediaDir();
|
|
const saved = await saveMediaBuffer(
|
|
params.buffer,
|
|
params.contentType,
|
|
"browser",
|
|
params.maxBytes,
|
|
);
|
|
params.res.json({
|
|
ok: true,
|
|
path: path.resolve(saved.path),
|
|
targetId: params.targetId,
|
|
url: params.url,
|
|
});
|
|
}
|
|
|
|
export function registerBrowserAgentSnapshotRoutes(
|
|
app: BrowserRouteRegistrar,
|
|
ctx: BrowserRouteContext,
|
|
) {
|
|
app.post("/navigate", async (req, res) => {
|
|
const body = readBody(req);
|
|
const url = toStringOrEmpty(body.url);
|
|
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
|
if (!url) {
|
|
return jsonError(res, 400, "url is required");
|
|
}
|
|
await withPlaywrightRouteContext({
|
|
req,
|
|
res,
|
|
ctx,
|
|
targetId,
|
|
feature: "navigate",
|
|
run: async ({ cdpUrl, tab, pw }) => {
|
|
const result = await pw.navigateViaPlaywright({
|
|
cdpUrl,
|
|
targetId: tab.targetId,
|
|
url,
|
|
...withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy),
|
|
});
|
|
res.json({ ok: true, targetId: tab.targetId, ...result });
|
|
},
|
|
});
|
|
});
|
|
|
|
app.post("/pdf", async (req, res) => {
|
|
const body = readBody(req);
|
|
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
|
await withPlaywrightRouteContext({
|
|
req,
|
|
res,
|
|
ctx,
|
|
targetId,
|
|
feature: "pdf",
|
|
run: async ({ cdpUrl, tab, pw }) => {
|
|
const pdf = await pw.pdfViaPlaywright({
|
|
cdpUrl,
|
|
targetId: tab.targetId,
|
|
});
|
|
await saveBrowserMediaResponse({
|
|
res,
|
|
buffer: pdf.buffer,
|
|
contentType: "application/pdf",
|
|
maxBytes: pdf.buffer.byteLength,
|
|
targetId: tab.targetId,
|
|
url: tab.url,
|
|
});
|
|
},
|
|
});
|
|
});
|
|
|
|
app.post("/screenshot", async (req, res) => {
|
|
const body = readBody(req);
|
|
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
|
const fullPage = toBoolean(body.fullPage) ?? false;
|
|
const ref = toStringOrEmpty(body.ref) || undefined;
|
|
const element = toStringOrEmpty(body.element) || undefined;
|
|
const type = body.type === "jpeg" ? "jpeg" : "png";
|
|
|
|
if (fullPage && (ref || element)) {
|
|
return jsonError(res, 400, "fullPage is not supported for element screenshots");
|
|
}
|
|
|
|
await withRouteTabContext({
|
|
req,
|
|
res,
|
|
ctx,
|
|
targetId,
|
|
run: async ({ profileCtx, tab, cdpUrl }) => {
|
|
let buffer: Buffer;
|
|
const shouldUsePlaywright =
|
|
profileCtx.profile.driver === "extension" ||
|
|
!tab.wsUrl ||
|
|
Boolean(ref) ||
|
|
Boolean(element);
|
|
if (shouldUsePlaywright) {
|
|
const pw = await requirePwAi(res, "screenshot");
|
|
if (!pw) {
|
|
return;
|
|
}
|
|
const snap = await pw.takeScreenshotViaPlaywright({
|
|
cdpUrl,
|
|
targetId: tab.targetId,
|
|
ref,
|
|
element,
|
|
fullPage,
|
|
type,
|
|
});
|
|
buffer = snap.buffer;
|
|
} else {
|
|
buffer = await captureScreenshot({
|
|
wsUrl: tab.wsUrl ?? "",
|
|
fullPage,
|
|
format: type,
|
|
quality: type === "jpeg" ? 85 : undefined,
|
|
});
|
|
}
|
|
|
|
const normalized = await normalizeBrowserScreenshot(buffer, {
|
|
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
|
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
|
});
|
|
await saveBrowserMediaResponse({
|
|
res,
|
|
buffer: normalized.buffer,
|
|
contentType: normalized.contentType ?? `image/${type}`,
|
|
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
|
targetId: tab.targetId,
|
|
url: tab.url,
|
|
});
|
|
},
|
|
});
|
|
});
|
|
|
|
app.get("/snapshot", async (req, res) => {
|
|
const profileCtx = resolveProfileContext(req, res, ctx);
|
|
if (!profileCtx) {
|
|
return;
|
|
}
|
|
const targetId = typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
|
|
const mode = req.query.mode === "efficient" ? "efficient" : undefined;
|
|
const labels = toBoolean(req.query.labels) ?? undefined;
|
|
const explicitFormat =
|
|
req.query.format === "aria" ? "aria" : req.query.format === "ai" ? "ai" : undefined;
|
|
const format = explicitFormat ?? (mode ? "ai" : (await getPwAiModule()) ? "ai" : "aria");
|
|
const limitRaw = typeof req.query.limit === "string" ? Number(req.query.limit) : undefined;
|
|
const hasMaxChars = Object.hasOwn(req.query, "maxChars");
|
|
const maxCharsRaw =
|
|
typeof req.query.maxChars === "string" ? Number(req.query.maxChars) : undefined;
|
|
const limit = Number.isFinite(limitRaw) ? limitRaw : undefined;
|
|
const maxChars =
|
|
typeof maxCharsRaw === "number" && Number.isFinite(maxCharsRaw) && maxCharsRaw > 0
|
|
? Math.floor(maxCharsRaw)
|
|
: undefined;
|
|
const resolvedMaxChars =
|
|
format === "ai"
|
|
? hasMaxChars
|
|
? maxChars
|
|
: mode === "efficient"
|
|
? DEFAULT_AI_SNAPSHOT_EFFICIENT_MAX_CHARS
|
|
: DEFAULT_AI_SNAPSHOT_MAX_CHARS
|
|
: undefined;
|
|
const interactiveRaw = toBoolean(req.query.interactive);
|
|
const compactRaw = toBoolean(req.query.compact);
|
|
const depthRaw = toNumber(req.query.depth);
|
|
const refsModeRaw = toStringOrEmpty(req.query.refs).trim();
|
|
const refsMode: "aria" | "role" | undefined =
|
|
refsModeRaw === "aria" ? "aria" : refsModeRaw === "role" ? "role" : undefined;
|
|
const interactive = interactiveRaw ?? (mode === "efficient" ? true : undefined);
|
|
const compact = compactRaw ?? (mode === "efficient" ? true : undefined);
|
|
const depth =
|
|
depthRaw ?? (mode === "efficient" ? DEFAULT_AI_SNAPSHOT_EFFICIENT_DEPTH : undefined);
|
|
const selector = toStringOrEmpty(req.query.selector);
|
|
const frameSelector = toStringOrEmpty(req.query.frame);
|
|
const selectorValue = selector.trim() || undefined;
|
|
const frameSelectorValue = frameSelector.trim() || undefined;
|
|
|
|
try {
|
|
const tab = await profileCtx.ensureTabAvailable(targetId || undefined);
|
|
if ((labels || mode === "efficient") && format === "aria") {
|
|
return jsonError(res, 400, "labels/mode=efficient require format=ai");
|
|
}
|
|
if (format === "ai") {
|
|
const pw = await requirePwAi(res, "ai snapshot");
|
|
if (!pw) {
|
|
return;
|
|
}
|
|
const wantsRoleSnapshot =
|
|
labels === true ||
|
|
mode === "efficient" ||
|
|
interactive === true ||
|
|
compact === true ||
|
|
depth !== undefined ||
|
|
Boolean(selectorValue) ||
|
|
Boolean(frameSelectorValue);
|
|
const roleSnapshotArgs = {
|
|
cdpUrl: profileCtx.profile.cdpUrl,
|
|
targetId: tab.targetId,
|
|
selector: selectorValue,
|
|
frameSelector: frameSelectorValue,
|
|
refsMode,
|
|
options: {
|
|
interactive: interactive ?? undefined,
|
|
compact: compact ?? undefined,
|
|
maxDepth: depth ?? undefined,
|
|
},
|
|
};
|
|
|
|
const snap = wantsRoleSnapshot
|
|
? await pw.snapshotRoleViaPlaywright(roleSnapshotArgs)
|
|
: await pw
|
|
.snapshotAiViaPlaywright({
|
|
cdpUrl: profileCtx.profile.cdpUrl,
|
|
targetId: tab.targetId,
|
|
...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
|
|
})
|
|
.catch(async (err) => {
|
|
// Public-API fallback when Playwright's private _snapshotForAI is missing.
|
|
if (String(err).toLowerCase().includes("_snapshotforai")) {
|
|
return await pw.snapshotRoleViaPlaywright(roleSnapshotArgs);
|
|
}
|
|
throw err;
|
|
});
|
|
if (labels) {
|
|
const labeled = await pw.screenshotWithLabelsViaPlaywright({
|
|
cdpUrl: profileCtx.profile.cdpUrl,
|
|
targetId: tab.targetId,
|
|
refs: "refs" in snap ? snap.refs : {},
|
|
type: "png",
|
|
});
|
|
const normalized = await normalizeBrowserScreenshot(labeled.buffer, {
|
|
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
|
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
|
});
|
|
await ensureMediaDir();
|
|
const saved = await saveMediaBuffer(
|
|
normalized.buffer,
|
|
normalized.contentType ?? "image/png",
|
|
"browser",
|
|
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
|
);
|
|
const imageType = normalized.contentType?.includes("jpeg") ? "jpeg" : "png";
|
|
return res.json({
|
|
ok: true,
|
|
format,
|
|
targetId: tab.targetId,
|
|
url: tab.url,
|
|
labels: true,
|
|
labelsCount: labeled.labels,
|
|
labelsSkipped: labeled.skipped,
|
|
imagePath: path.resolve(saved.path),
|
|
imageType,
|
|
...snap,
|
|
});
|
|
}
|
|
|
|
return res.json({
|
|
ok: true,
|
|
format,
|
|
targetId: tab.targetId,
|
|
url: tab.url,
|
|
...snap,
|
|
});
|
|
}
|
|
|
|
const snap =
|
|
profileCtx.profile.driver === "extension" || !tab.wsUrl
|
|
? (() => {
|
|
// Extension relay doesn't expose per-page WS URLs; run AX snapshot via Playwright CDP session.
|
|
// Also covers cases where wsUrl is missing/unusable.
|
|
return requirePwAi(res, "aria snapshot").then(async (pw) => {
|
|
if (!pw) {
|
|
return null;
|
|
}
|
|
return await pw.snapshotAriaViaPlaywright({
|
|
cdpUrl: profileCtx.profile.cdpUrl,
|
|
targetId: tab.targetId,
|
|
limit,
|
|
});
|
|
});
|
|
})()
|
|
: snapshotAria({ wsUrl: tab.wsUrl ?? "", limit });
|
|
|
|
const resolved = await Promise.resolve(snap);
|
|
if (!resolved) {
|
|
return;
|
|
}
|
|
return res.json({
|
|
ok: true,
|
|
format,
|
|
targetId: tab.targetId,
|
|
url: tab.url,
|
|
...resolved,
|
|
});
|
|
} catch (err) {
|
|
handleRouteError(ctx, res, err);
|
|
}
|
|
});
|
|
}
|