Include full contents of all nested repositories

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

openclaw/extensions/diagnostics-otel/index.ts

@@ -0,0 +1,15 @@
+import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
+import { emptyPluginConfigSchema } from "openclaw/plugin-sdk";
+import { createDiagnosticsOtelService } from "./src/service.js";
+
+const plugin = {
+  id: "diagnostics-otel",
+  name: "Diagnostics OpenTelemetry",
+  description: "Export diagnostics events to OpenTelemetry",
+  configSchema: emptyPluginConfigSchema(),
+  register(api: OpenClawPluginApi) {
+    api.registerService(createDiagnosticsOtelService());
+  },
+};
+
+export default plugin;

openclaw/extensions/diagnostics-otel/openclaw.plugin.json

@@ -0,0 +1,8 @@
+{
+  "id": "diagnostics-otel",
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {}
+  }
+}

openclaw/extensions/diagnostics-otel/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@openclaw/diagnostics-otel",
+  "version": "2026.2.26",
+  "description": "OpenClaw diagnostics OpenTelemetry exporter",
+  "type": "module",
+  "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/api-logs": "^0.212.0",
+    "@opentelemetry/exporter-logs-otlp-proto": "^0.212.0",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.212.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.212.0",
+    "@opentelemetry/resources": "^2.5.1",
+    "@opentelemetry/sdk-logs": "^0.212.0",
+    "@opentelemetry/sdk-metrics": "^2.5.1",
+    "@opentelemetry/sdk-node": "^0.212.0",
+    "@opentelemetry/sdk-trace-base": "^2.5.1",
+    "@opentelemetry/semantic-conventions": "^1.39.0"
+  },
+  "openclaw": {
+    "extensions": [
+      "./index.ts"
+    ]
+  }
+}

openclaw/extensions/diagnostics-otel/src/service.test.ts

@@ -0,0 +1,367 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+
+const registerLogTransportMock = vi.hoisted(() => vi.fn());
+
+const telemetryState = vi.hoisted(() => {
+  const counters = new Map<string, { add: ReturnType<typeof vi.fn> }>();
+  const histograms = new Map<string, { record: ReturnType<typeof vi.fn> }>();
+  const tracer = {
+    startSpan: vi.fn((_name: string, _opts?: unknown) => ({
+      end: vi.fn(),
+      setStatus: vi.fn(),
+    })),
+  };
+  const meter = {
+    createCounter: vi.fn((name: string) => {
+      const counter = { add: vi.fn() };
+      counters.set(name, counter);
+      return counter;
+    }),
+    createHistogram: vi.fn((name: string) => {
+      const histogram = { record: vi.fn() };
+      histograms.set(name, histogram);
+      return histogram;
+    }),
+  };
+  return { counters, histograms, tracer, meter };
+});
+
+const sdkStart = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+const sdkShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+const logEmit = vi.hoisted(() => vi.fn());
+const logShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+const traceExporterCtor = vi.hoisted(() => vi.fn());
+
+vi.mock("@opentelemetry/api", () => ({
+  metrics: {
+    getMeter: () => telemetryState.meter,
+  },
+  trace: {
+    getTracer: () => telemetryState.tracer,
+  },
+  SpanStatusCode: {
+    ERROR: 2,
+  },
+}));
+
+vi.mock("@opentelemetry/sdk-node", () => ({
+  NodeSDK: class {
+    start = sdkStart;
+    shutdown = sdkShutdown;
+  },
+}));
+
+vi.mock("@opentelemetry/exporter-metrics-otlp-proto", () => ({
+  OTLPMetricExporter: class {},
+}));
+
+vi.mock("@opentelemetry/exporter-trace-otlp-proto", () => ({
+  OTLPTraceExporter: class {
+    constructor(options?: unknown) {
+      traceExporterCtor(options);
+    }
+  },
+}));
+
+vi.mock("@opentelemetry/exporter-logs-otlp-proto", () => ({
+  OTLPLogExporter: class {},
+}));
+
+vi.mock("@opentelemetry/sdk-logs", () => ({
+  BatchLogRecordProcessor: class {},
+  LoggerProvider: class {
+    getLogger = vi.fn(() => ({
+      emit: logEmit,
+    }));
+    shutdown = logShutdown;
+  },
+}));
+
+vi.mock("@opentelemetry/sdk-metrics", () => ({
+  PeriodicExportingMetricReader: class {},
+}));
+
+vi.mock("@opentelemetry/sdk-trace-base", () => ({
+  ParentBasedSampler: class {},
+  TraceIdRatioBasedSampler: class {},
+}));
+
+vi.mock("@opentelemetry/resources", () => ({
+  resourceFromAttributes: vi.fn((attrs: Record<string, unknown>) => attrs),
+  Resource: class {
+    // eslint-disable-next-line @typescript-eslint/no-useless-constructor
+    constructor(_value?: unknown) {}
+  },
+}));
+
+vi.mock("@opentelemetry/semantic-conventions", () => ({
+  ATTR_SERVICE_NAME: "service.name",
+}));
+
+vi.mock("openclaw/plugin-sdk", async () => {
+  const actual = await vi.importActual<typeof import("openclaw/plugin-sdk")>("openclaw/plugin-sdk");
+  return {
+    ...actual,
+    registerLogTransport: registerLogTransportMock,
+  };
+});
+
+import type { OpenClawPluginServiceContext } from "openclaw/plugin-sdk";
+import { emitDiagnosticEvent } from "openclaw/plugin-sdk";
+import { createDiagnosticsOtelService } from "./service.js";
+
+const OTEL_TEST_STATE_DIR = "/tmp/openclaw-diagnostics-otel-test";
+const OTEL_TEST_ENDPOINT = "http://otel-collector:4318";
+const OTEL_TEST_PROTOCOL = "http/protobuf";
+
+function createLogger() {
+  return {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    debug: vi.fn(),
+  };
+}
+
+type OtelContextFlags = {
+  traces?: boolean;
+  metrics?: boolean;
+  logs?: boolean;
+};
+function createOtelContext(
+  endpoint: string,
+  { traces = false, metrics = false, logs = false }: OtelContextFlags = {},
+): OpenClawPluginServiceContext {
+  return {
+    config: {
+      diagnostics: {
+        enabled: true,
+        otel: {
+          enabled: true,
+          endpoint,
+          protocol: OTEL_TEST_PROTOCOL,
+          traces,
+          metrics,
+          logs,
+        },
+      },
+    },
+    logger: createLogger(),
+    stateDir: OTEL_TEST_STATE_DIR,
+  };
+}
+
+function createTraceOnlyContext(endpoint: string): OpenClawPluginServiceContext {
+  return createOtelContext(endpoint, { traces: true });
+}
+
+type RegisteredLogTransport = (logObj: Record<string, unknown>) => void;
+function setupRegisteredTransports() {
+  const registeredTransports: RegisteredLogTransport[] = [];
+  const stopTransport = vi.fn();
+  registerLogTransportMock.mockImplementation((transport) => {
+    registeredTransports.push(transport);
+    return stopTransport;
+  });
+  return { registeredTransports, stopTransport };
+}
+
+async function emitAndCaptureLog(logObj: Record<string, unknown>) {
+  const { registeredTransports } = setupRegisteredTransports();
+  const service = createDiagnosticsOtelService();
+  const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { logs: true });
+  await service.start(ctx);
+  expect(registeredTransports).toHaveLength(1);
+  registeredTransports[0]?.(logObj);
+  expect(logEmit).toHaveBeenCalled();
+  const emitCall = logEmit.mock.calls[0]?.[0];
+  await service.stop?.(ctx);
+  return emitCall;
+}
+
+describe("diagnostics-otel service", () => {
+  beforeEach(() => {
+    telemetryState.counters.clear();
+    telemetryState.histograms.clear();
+    telemetryState.tracer.startSpan.mockClear();
+    telemetryState.meter.createCounter.mockClear();
+    telemetryState.meter.createHistogram.mockClear();
+    sdkStart.mockClear();
+    sdkShutdown.mockClear();
+    logEmit.mockClear();
+    logShutdown.mockClear();
+    traceExporterCtor.mockClear();
+    registerLogTransportMock.mockReset();
+  });
+
+  test("records message-flow metrics and spans", async () => {
+    const { registeredTransports } = setupRegisteredTransports();
+
+    const service = createDiagnosticsOtelService();
+    const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true, logs: true });
+    await service.start(ctx);
+
+    emitDiagnosticEvent({
+      type: "webhook.received",
+      channel: "telegram",
+      updateType: "telegram-post",
+    });
+    emitDiagnosticEvent({
+      type: "webhook.processed",
+      channel: "telegram",
+      updateType: "telegram-post",
+      durationMs: 120,
+    });
+    emitDiagnosticEvent({
+      type: "message.queued",
+      channel: "telegram",
+      source: "telegram",
+      queueDepth: 2,
+    });
+    emitDiagnosticEvent({
+      type: "message.processed",
+      channel: "telegram",
+      outcome: "completed",
+      durationMs: 55,
+    });
+    emitDiagnosticEvent({
+      type: "queue.lane.dequeue",
+      lane: "main",
+      queueSize: 3,
+      waitMs: 10,
+    });
+    emitDiagnosticEvent({
+      type: "session.stuck",
+      state: "processing",
+      ageMs: 125_000,
+    });
+    emitDiagnosticEvent({
+      type: "run.attempt",
+      runId: "run-1",
+      attempt: 2,
+    });
+
+    expect(telemetryState.counters.get("openclaw.webhook.received")?.add).toHaveBeenCalled();
+    expect(
+      telemetryState.histograms.get("openclaw.webhook.duration_ms")?.record,
+    ).toHaveBeenCalled();
+    expect(telemetryState.counters.get("openclaw.message.queued")?.add).toHaveBeenCalled();
+    expect(telemetryState.counters.get("openclaw.message.processed")?.add).toHaveBeenCalled();
+    expect(
+      telemetryState.histograms.get("openclaw.message.duration_ms")?.record,
+    ).toHaveBeenCalled();
+    expect(telemetryState.histograms.get("openclaw.queue.wait_ms")?.record).toHaveBeenCalled();
+    expect(telemetryState.counters.get("openclaw.session.stuck")?.add).toHaveBeenCalled();
+    expect(
+      telemetryState.histograms.get("openclaw.session.stuck_age_ms")?.record,
+    ).toHaveBeenCalled();
+    expect(telemetryState.counters.get("openclaw.run.attempt")?.add).toHaveBeenCalled();
+
+    const spanNames = telemetryState.tracer.startSpan.mock.calls.map((call) => call[0]);
+    expect(spanNames).toContain("openclaw.webhook.processed");
+    expect(spanNames).toContain("openclaw.message.processed");
+    expect(spanNames).toContain("openclaw.session.stuck");
+
+    expect(registerLogTransportMock).toHaveBeenCalledTimes(1);
+    expect(registeredTransports).toHaveLength(1);
+    registeredTransports[0]?.({
+      0: '{"subsystem":"diagnostic"}',
+      1: "hello",
+      _meta: { logLevelName: "INFO", date: new Date() },
+    });
+    expect(logEmit).toHaveBeenCalled();
+
+    await service.stop?.(ctx);
+  });
+
+  test("appends signal path when endpoint contains non-signal /v1 segment", async () => {
+    const service = createDiagnosticsOtelService();
+    const ctx = createTraceOnlyContext("https://www.comet.com/opik/api/v1/private/otel");
+    await service.start(ctx);
+
+    const options = traceExporterCtor.mock.calls[0]?.[0] as { url?: string } | undefined;
+    expect(options?.url).toBe("https://www.comet.com/opik/api/v1/private/otel/v1/traces");
+    await service.stop?.(ctx);
+  });
+
+  test("keeps already signal-qualified endpoint unchanged", async () => {
+    const service = createDiagnosticsOtelService();
+    const ctx = createTraceOnlyContext("https://collector.example.com/v1/traces");
+    await service.start(ctx);
+
+    const options = traceExporterCtor.mock.calls[0]?.[0] as { url?: string } | undefined;
+    expect(options?.url).toBe("https://collector.example.com/v1/traces");
+    await service.stop?.(ctx);
+  });
+
+  test("keeps signal-qualified endpoint unchanged when it has query params", async () => {
+    const service = createDiagnosticsOtelService();
+    const ctx = createTraceOnlyContext("https://collector.example.com/v1/traces?timeout=30s");
+    await service.start(ctx);
+
+    const options = traceExporterCtor.mock.calls[0]?.[0] as { url?: string } | undefined;
+    expect(options?.url).toBe("https://collector.example.com/v1/traces?timeout=30s");
+    await service.stop?.(ctx);
+  });
+
+  test("keeps signal-qualified endpoint unchanged when signal path casing differs", async () => {
+    const service = createDiagnosticsOtelService();
+    const ctx = createTraceOnlyContext("https://collector.example.com/v1/Traces");
+    await service.start(ctx);
+
+    const options = traceExporterCtor.mock.calls[0]?.[0] as { url?: string } | undefined;
+    expect(options?.url).toBe("https://collector.example.com/v1/Traces");
+    await service.stop?.(ctx);
+  });
+
+  test("redacts sensitive data from log messages before export", async () => {
+    const emitCall = await emitAndCaptureLog({
+      0: "Using API key sk-1234567890abcdef1234567890abcdef",
+      _meta: { logLevelName: "INFO", date: new Date() },
+    });
+
+    expect(emitCall?.body).not.toContain("sk-1234567890abcdef1234567890abcdef");
+    expect(emitCall?.body).toContain("sk-123");
+    expect(emitCall?.body).toContain("…");
+  });
+
+  test("redacts sensitive data from log attributes before export", async () => {
+    const emitCall = await emitAndCaptureLog({
+      0: '{"token":"ghp_abcdefghijklmnopqrstuvwxyz123456"}',
+      1: "auth configured",
+      _meta: { logLevelName: "DEBUG", date: new Date() },
+    });
+
+    const tokenAttr = emitCall?.attributes?.["openclaw.token"];
+    expect(tokenAttr).not.toBe("ghp_abcdefghijklmnopqrstuvwxyz123456");
+    if (typeof tokenAttr === "string") {
+      expect(tokenAttr).toContain("…");
+    }
+  });
+
+  test("redacts sensitive reason in session.state metric attributes", async () => {
+    const service = createDiagnosticsOtelService();
+    const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { metrics: true });
+    await service.start(ctx);
+
+    emitDiagnosticEvent({
+      type: "session.state",
+      state: "waiting",
+      reason: "token=ghp_abcdefghijklmnopqrstuvwxyz123456",
+    });
+
+    const sessionCounter = telemetryState.counters.get("openclaw.session.state");
+    expect(sessionCounter?.add).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        "openclaw.reason": expect.stringContaining("…"),
+      }),
+    );
+    const attrs = sessionCounter?.add.mock.calls[0]?.[1] as Record<string, unknown> | undefined;
+    expect(typeof attrs?.["openclaw.reason"]).toBe("string");
+    expect(String(attrs?.["openclaw.reason"])).not.toContain(
+      "ghp_abcdefghijklmnopqrstuvwxyz123456",
+    );
+    await service.stop?.(ctx);
+  });
+});

openclaw/extensions/diagnostics-otel/src/service.ts

@@ -0,0 +1,678 @@
+import { metrics, trace, SpanStatusCode } from "@opentelemetry/api";
+import type { SeverityNumber } from "@opentelemetry/api-logs";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { BatchLogRecordProcessor, LoggerProvider } from "@opentelemetry/sdk-logs";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import type { DiagnosticEventPayload, OpenClawPluginService } from "openclaw/plugin-sdk";
+import { onDiagnosticEvent, redactSensitiveText, registerLogTransport } from "openclaw/plugin-sdk";
+
+const DEFAULT_SERVICE_NAME = "openclaw";
+
+function normalizeEndpoint(endpoint?: string): string | undefined {
+  const trimmed = endpoint?.trim();
+  return trimmed ? trimmed.replace(/\/+$/, "") : undefined;
+}
+
+function resolveOtelUrl(endpoint: string | undefined, path: string): string | undefined {
+  if (!endpoint) {
+    return undefined;
+  }
+  const endpointWithoutQueryOrFragment = endpoint.split(/[?#]/, 1)[0] ?? endpoint;
+  if (/\/v1\/(?:traces|metrics|logs)$/i.test(endpointWithoutQueryOrFragment)) {
+    return endpoint;
+  }
+  return `${endpoint}/${path}`;
+}
+
+function resolveSampleRate(value: number | undefined): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return undefined;
+  }
+  if (value < 0 || value > 1) {
+    return undefined;
+  }
+  return value;
+}
+
+function formatError(err: unknown): string {
+  if (err instanceof Error) {
+    return err.stack ?? err.message;
+  }
+  if (typeof err === "string") {
+    return err;
+  }
+  try {
+    return JSON.stringify(err);
+  } catch {
+    return String(err);
+  }
+}
+
+function redactOtelAttributes(attributes: Record<string, string | number | boolean>) {
+  const redactedAttributes: Record<string, string | number | boolean> = {};
+  for (const [key, value] of Object.entries(attributes)) {
+    redactedAttributes[key] = typeof value === "string" ? redactSensitiveText(value) : value;
+  }
+  return redactedAttributes;
+}
+
+export function createDiagnosticsOtelService(): OpenClawPluginService {
+  let sdk: NodeSDK | null = null;
+  let logProvider: LoggerProvider | null = null;
+  let stopLogTransport: (() => void) | null = null;
+  let unsubscribe: (() => void) | null = null;
+
+  return {
+    id: "diagnostics-otel",
+    async start(ctx) {
+      const cfg = ctx.config.diagnostics;
+      const otel = cfg?.otel;
+      if (!cfg?.enabled || !otel?.enabled) {
+        return;
+      }
+
+      const protocol = otel.protocol ?? process.env.OTEL_EXPORTER_OTLP_PROTOCOL ?? "http/protobuf";
+      if (protocol !== "http/protobuf") {
+        ctx.logger.warn(`diagnostics-otel: unsupported protocol ${protocol}`);
+        return;
+      }
+
+      const endpoint = normalizeEndpoint(otel.endpoint ?? process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+      const headers = otel.headers ?? undefined;
+      const serviceName =
+        otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE_NAME;
+      const sampleRate = resolveSampleRate(otel.sampleRate);
+
+      const tracesEnabled = otel.traces !== false;
+      const metricsEnabled = otel.metrics !== false;
+      const logsEnabled = otel.logs === true;
+      if (!tracesEnabled && !metricsEnabled && !logsEnabled) {
+        return;
+      }
+
+      const resource = resourceFromAttributes({
+        [ATTR_SERVICE_NAME]: serviceName,
+      });
+
+      const traceUrl = resolveOtelUrl(endpoint, "v1/traces");
+      const metricUrl = resolveOtelUrl(endpoint, "v1/metrics");
+      const logUrl = resolveOtelUrl(endpoint, "v1/logs");
+      const traceExporter = tracesEnabled
+        ? new OTLPTraceExporter({
+            ...(traceUrl ? { url: traceUrl } : {}),
+            ...(headers ? { headers } : {}),
+          })
+        : undefined;
+
+      const metricExporter = metricsEnabled
+        ? new OTLPMetricExporter({
+            ...(metricUrl ? { url: metricUrl } : {}),
+            ...(headers ? { headers } : {}),
+          })
+        : undefined;
+
+      const metricReader = metricExporter
+        ? new PeriodicExportingMetricReader({
+            exporter: metricExporter,
+            ...(typeof otel.flushIntervalMs === "number"
+              ? { exportIntervalMillis: Math.max(1000, otel.flushIntervalMs) }
+              : {}),
+          })
+        : undefined;
+
+      if (tracesEnabled || metricsEnabled) {
+        sdk = new NodeSDK({
+          resource,
+          ...(traceExporter ? { traceExporter } : {}),
+          ...(metricReader ? { metricReader } : {}),
+          ...(sampleRate !== undefined
+            ? {
+                sampler: new ParentBasedSampler({
+                  root: new TraceIdRatioBasedSampler(sampleRate),
+                }),
+              }
+            : {}),
+        });
+
+        try {
+          await sdk.start();
+        } catch (err) {
+          ctx.logger.error(`diagnostics-otel: failed to start SDK: ${formatError(err)}`);
+          throw err;
+        }
+      }
+
+      const logSeverityMap: Record<string, SeverityNumber> = {
+        TRACE: 1 as SeverityNumber,
+        DEBUG: 5 as SeverityNumber,
+        INFO: 9 as SeverityNumber,
+        WARN: 13 as SeverityNumber,
+        ERROR: 17 as SeverityNumber,
+        FATAL: 21 as SeverityNumber,
+      };
+
+      const meter = metrics.getMeter("openclaw");
+      const tracer = trace.getTracer("openclaw");
+
+      const tokensCounter = meter.createCounter("openclaw.tokens", {
+        unit: "1",
+        description: "Token usage by type",
+      });
+      const costCounter = meter.createCounter("openclaw.cost.usd", {
+        unit: "1",
+        description: "Estimated model cost (USD)",
+      });
+      const durationHistogram = meter.createHistogram("openclaw.run.duration_ms", {
+        unit: "ms",
+        description: "Agent run duration",
+      });
+      const contextHistogram = meter.createHistogram("openclaw.context.tokens", {
+        unit: "1",
+        description: "Context window size and usage",
+      });
+      const webhookReceivedCounter = meter.createCounter("openclaw.webhook.received", {
+        unit: "1",
+        description: "Webhook requests received",
+      });
+      const webhookErrorCounter = meter.createCounter("openclaw.webhook.error", {
+        unit: "1",
+        description: "Webhook processing errors",
+      });
+      const webhookDurationHistogram = meter.createHistogram("openclaw.webhook.duration_ms", {
+        unit: "ms",
+        description: "Webhook processing duration",
+      });
+      const messageQueuedCounter = meter.createCounter("openclaw.message.queued", {
+        unit: "1",
+        description: "Messages queued for processing",
+      });
+      const messageProcessedCounter = meter.createCounter("openclaw.message.processed", {
+        unit: "1",
+        description: "Messages processed by outcome",
+      });
+      const messageDurationHistogram = meter.createHistogram("openclaw.message.duration_ms", {
+        unit: "ms",
+        description: "Message processing duration",
+      });
+      const queueDepthHistogram = meter.createHistogram("openclaw.queue.depth", {
+        unit: "1",
+        description: "Queue depth on enqueue/dequeue",
+      });
+      const queueWaitHistogram = meter.createHistogram("openclaw.queue.wait_ms", {
+        unit: "ms",
+        description: "Queue wait time before execution",
+      });
+      const laneEnqueueCounter = meter.createCounter("openclaw.queue.lane.enqueue", {
+        unit: "1",
+        description: "Command queue lane enqueue events",
+      });
+      const laneDequeueCounter = meter.createCounter("openclaw.queue.lane.dequeue", {
+        unit: "1",
+        description: "Command queue lane dequeue events",
+      });
+      const sessionStateCounter = meter.createCounter("openclaw.session.state", {
+        unit: "1",
+        description: "Session state transitions",
+      });
+      const sessionStuckCounter = meter.createCounter("openclaw.session.stuck", {
+        unit: "1",
+        description: "Sessions stuck in processing",
+      });
+      const sessionStuckAgeHistogram = meter.createHistogram("openclaw.session.stuck_age_ms", {
+        unit: "ms",
+        description: "Age of stuck sessions",
+      });
+      const runAttemptCounter = meter.createCounter("openclaw.run.attempt", {
+        unit: "1",
+        description: "Run attempts",
+      });
+
+      if (logsEnabled) {
+        const logExporter = new OTLPLogExporter({
+          ...(logUrl ? { url: logUrl } : {}),
+          ...(headers ? { headers } : {}),
+        });
+        const logProcessor = new BatchLogRecordProcessor(
+          logExporter,
+          typeof otel.flushIntervalMs === "number"
+            ? { scheduledDelayMillis: Math.max(1000, otel.flushIntervalMs) }
+            : {},
+        );
+        logProvider = new LoggerProvider({
+          resource,
+          processors: [logProcessor],
+        });
+        const otelLogger = logProvider.getLogger("openclaw");
+
+        stopLogTransport = registerLogTransport((logObj) => {
+          try {
+            const safeStringify = (value: unknown) => {
+              try {
+                return JSON.stringify(value);
+              } catch {
+                return String(value);
+              }
+            };
+            const meta = (logObj as Record<string, unknown>)._meta as
+              | {
+                  logLevelName?: string;
+                  date?: Date;
+                  name?: string;
+                  parentNames?: string[];
+                  path?: {
+                    filePath?: string;
+                    fileLine?: string;
+                    fileColumn?: string;
+                    filePathWithLine?: string;
+                    method?: string;
+                  };
+                }
+              | undefined;
+            const logLevelName = meta?.logLevelName ?? "INFO";
+            const severityNumber = logSeverityMap[logLevelName] ?? (9 as SeverityNumber);
+
+            const numericArgs = Object.entries(logObj)
+              .filter(([key]) => /^\d+$/.test(key))
+              .toSorted((a, b) => Number(a[0]) - Number(b[0]))
+              .map(([, value]) => value);
+
+            let bindings: Record<string, unknown> | undefined;
+            if (typeof numericArgs[0] === "string" && numericArgs[0].trim().startsWith("{")) {
+              try {
+                const parsed = JSON.parse(numericArgs[0]);
+                if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                  bindings = parsed as Record<string, unknown>;
+                  numericArgs.shift();
+                }
+              } catch {
+                // ignore malformed json bindings
+              }
+            }
+
+            let message = "";
+            if (numericArgs.length > 0 && typeof numericArgs[numericArgs.length - 1] === "string") {
+              message = String(numericArgs.pop());
+            } else if (numericArgs.length === 1) {
+              message = safeStringify(numericArgs[0]);
+              numericArgs.length = 0;
+            }
+            if (!message) {
+              message = "log";
+            }
+
+            const attributes: Record<string, string | number | boolean> = {
+              "openclaw.log.level": logLevelName,
+            };
+            if (meta?.name) {
+              attributes["openclaw.logger"] = meta.name;
+            }
+            if (meta?.parentNames?.length) {
+              attributes["openclaw.logger.parents"] = meta.parentNames.join(".");
+            }
+            if (bindings) {
+              for (const [key, value] of Object.entries(bindings)) {
+                if (
+                  typeof value === "string" ||
+                  typeof value === "number" ||
+                  typeof value === "boolean"
+                ) {
+                  attributes[`openclaw.${key}`] = value;
+                } else if (value != null) {
+                  attributes[`openclaw.${key}`] = safeStringify(value);
+                }
+              }
+            }
+            if (numericArgs.length > 0) {
+              attributes["openclaw.log.args"] = safeStringify(numericArgs);
+            }
+            if (meta?.path?.filePath) {
+              attributes["code.filepath"] = meta.path.filePath;
+            }
+            if (meta?.path?.fileLine) {
+              attributes["code.lineno"] = Number(meta.path.fileLine);
+            }
+            if (meta?.path?.method) {
+              attributes["code.function"] = meta.path.method;
+            }
+            if (meta?.path?.filePathWithLine) {
+              attributes["openclaw.code.location"] = meta.path.filePathWithLine;
+            }
+
+            // OTLP can leave the host boundary, so redact string fields before export.
+            otelLogger.emit({
+              body: redactSensitiveText(message),
+              severityText: logLevelName,
+              severityNumber,
+              attributes: redactOtelAttributes(attributes),
+              timestamp: meta?.date ?? new Date(),
+            });
+          } catch (err) {
+            ctx.logger.error(`diagnostics-otel: log transport failed: ${formatError(err)}`);
+          }
+        });
+      }
+
+      const spanWithDuration = (
+        name: string,
+        attributes: Record<string, string | number>,
+        durationMs?: number,
+      ) => {
+        const startTime =
+          typeof durationMs === "number" ? Date.now() - Math.max(0, durationMs) : undefined;
+        const span = tracer.startSpan(name, {
+          attributes,
+          ...(startTime ? { startTime } : {}),
+        });
+        return span;
+      };
+
+      const recordModelUsage = (evt: Extract<DiagnosticEventPayload, { type: "model.usage" }>) => {
+        const attrs = {
+          "openclaw.channel": evt.channel ?? "unknown",
+          "openclaw.provider": evt.provider ?? "unknown",
+          "openclaw.model": evt.model ?? "unknown",
+        };
+
+        const usage = evt.usage;
+        if (usage.input) {
+          tokensCounter.add(usage.input, { ...attrs, "openclaw.token": "input" });
+        }
+        if (usage.output) {
+          tokensCounter.add(usage.output, { ...attrs, "openclaw.token": "output" });
+        }
+        if (usage.cacheRead) {
+          tokensCounter.add(usage.cacheRead, { ...attrs, "openclaw.token": "cache_read" });
+        }
+        if (usage.cacheWrite) {
+          tokensCounter.add(usage.cacheWrite, { ...attrs, "openclaw.token": "cache_write" });
+        }
+        if (usage.promptTokens) {
+          tokensCounter.add(usage.promptTokens, { ...attrs, "openclaw.token": "prompt" });
+        }
+        if (usage.total) {
+          tokensCounter.add(usage.total, { ...attrs, "openclaw.token": "total" });
+        }
+
+        if (evt.costUsd) {
+          costCounter.add(evt.costUsd, attrs);
+        }
+        if (evt.durationMs) {
+          durationHistogram.record(evt.durationMs, attrs);
+        }
+        if (evt.context?.limit) {
+          contextHistogram.record(evt.context.limit, {
+            ...attrs,
+            "openclaw.context": "limit",
+          });
+        }
+        if (evt.context?.used) {
+          contextHistogram.record(evt.context.used, {
+            ...attrs,
+            "openclaw.context": "used",
+          });
+        }
+
+        if (!tracesEnabled) {
+          return;
+        }
+        const spanAttrs: Record<string, string | number> = {
+          ...attrs,
+          "openclaw.sessionKey": evt.sessionKey ?? "",
+          "openclaw.sessionId": evt.sessionId ?? "",
+          "openclaw.tokens.input": usage.input ?? 0,
+          "openclaw.tokens.output": usage.output ?? 0,
+          "openclaw.tokens.cache_read": usage.cacheRead ?? 0,
+          "openclaw.tokens.cache_write": usage.cacheWrite ?? 0,
+          "openclaw.tokens.total": usage.total ?? 0,
+        };
+
+        const span = spanWithDuration("openclaw.model.usage", spanAttrs, evt.durationMs);
+        span.end();
+      };
+
+      const recordWebhookReceived = (
+        evt: Extract<DiagnosticEventPayload, { type: "webhook.received" }>,
+      ) => {
+        const attrs = {
+          "openclaw.channel": evt.channel ?? "unknown",
+          "openclaw.webhook": evt.updateType ?? "unknown",
+        };
+        webhookReceivedCounter.add(1, attrs);
+      };
+
+      const recordWebhookProcessed = (
+        evt: Extract<DiagnosticEventPayload, { type: "webhook.processed" }>,
+      ) => {
+        const attrs = {
+          "openclaw.channel": evt.channel ?? "unknown",
+          "openclaw.webhook": evt.updateType ?? "unknown",
+        };
+        if (typeof evt.durationMs === "number") {
+          webhookDurationHistogram.record(evt.durationMs, attrs);
+        }
+        if (!tracesEnabled) {
+          return;
+        }
+        const spanAttrs: Record<string, string | number> = { ...attrs };
+        if (evt.chatId !== undefined) {
+          spanAttrs["openclaw.chatId"] = String(evt.chatId);
+        }
+        const span = spanWithDuration("openclaw.webhook.processed", spanAttrs, evt.durationMs);
+        span.end();
+      };
+
+      const recordWebhookError = (
+        evt: Extract<DiagnosticEventPayload, { type: "webhook.error" }>,
+      ) => {
+        const attrs = {
+          "openclaw.channel": evt.channel ?? "unknown",
+          "openclaw.webhook": evt.updateType ?? "unknown",
+        };
+        webhookErrorCounter.add(1, attrs);
+        if (!tracesEnabled) {
+          return;
+        }
+        const redactedError = redactSensitiveText(evt.error);
+        const spanAttrs: Record<string, string | number> = {
+          ...attrs,
+          "openclaw.error": redactedError,
+        };
+        if (evt.chatId !== undefined) {
+          spanAttrs["openclaw.chatId"] = String(evt.chatId);
+        }
+        const span = tracer.startSpan("openclaw.webhook.error", {
+          attributes: spanAttrs,
+        });
+        span.setStatus({ code: SpanStatusCode.ERROR, message: redactedError });
+        span.end();
+      };
+
+      const recordMessageQueued = (
+        evt: Extract<DiagnosticEventPayload, { type: "message.queued" }>,
+      ) => {
+        const attrs = {
+          "openclaw.channel": evt.channel ?? "unknown",
+          "openclaw.source": evt.source ?? "unknown",
+        };
+        messageQueuedCounter.add(1, attrs);
+        if (typeof evt.queueDepth === "number") {
+          queueDepthHistogram.record(evt.queueDepth, attrs);
+        }
+      };
+
+      const addSessionIdentityAttrs = (
+        spanAttrs: Record<string, string | number>,
+        evt: { sessionKey?: string; sessionId?: string },
+      ) => {
+        if (evt.sessionKey) {
+          spanAttrs["openclaw.sessionKey"] = evt.sessionKey;
+        }
+        if (evt.sessionId) {
+          spanAttrs["openclaw.sessionId"] = evt.sessionId;
+        }
+      };
+
+      const recordMessageProcessed = (
+        evt: Extract<DiagnosticEventPayload, { type: "message.processed" }>,
+      ) => {
+        const attrs = {
+          "openclaw.channel": evt.channel ?? "unknown",
+          "openclaw.outcome": evt.outcome ?? "unknown",
+        };
+        messageProcessedCounter.add(1, attrs);
+        if (typeof evt.durationMs === "number") {
+          messageDurationHistogram.record(evt.durationMs, attrs);
+        }
+        if (!tracesEnabled) {
+          return;
+        }
+        const spanAttrs: Record<string, string | number> = { ...attrs };
+        addSessionIdentityAttrs(spanAttrs, evt);
+        if (evt.chatId !== undefined) {
+          spanAttrs["openclaw.chatId"] = String(evt.chatId);
+        }
+        if (evt.messageId !== undefined) {
+          spanAttrs["openclaw.messageId"] = String(evt.messageId);
+        }
+        if (evt.reason) {
+          spanAttrs["openclaw.reason"] = redactSensitiveText(evt.reason);
+        }
+        const span = spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs);
+        if (evt.outcome === "error" && evt.error) {
+          span.setStatus({ code: SpanStatusCode.ERROR, message: redactSensitiveText(evt.error) });
+        }
+        span.end();
+      };
+
+      const recordLaneEnqueue = (
+        evt: Extract<DiagnosticEventPayload, { type: "queue.lane.enqueue" }>,
+      ) => {
+        const attrs = { "openclaw.lane": evt.lane };
+        laneEnqueueCounter.add(1, attrs);
+        queueDepthHistogram.record(evt.queueSize, attrs);
+      };
+
+      const recordLaneDequeue = (
+        evt: Extract<DiagnosticEventPayload, { type: "queue.lane.dequeue" }>,
+      ) => {
+        const attrs = { "openclaw.lane": evt.lane };
+        laneDequeueCounter.add(1, attrs);
+        queueDepthHistogram.record(evt.queueSize, attrs);
+        if (typeof evt.waitMs === "number") {
+          queueWaitHistogram.record(evt.waitMs, attrs);
+        }
+      };
+
+      const recordSessionState = (
+        evt: Extract<DiagnosticEventPayload, { type: "session.state" }>,
+      ) => {
+        const attrs: Record<string, string> = { "openclaw.state": evt.state };
+        if (evt.reason) {
+          attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
+        }
+        sessionStateCounter.add(1, attrs);
+      };
+
+      const recordSessionStuck = (
+        evt: Extract<DiagnosticEventPayload, { type: "session.stuck" }>,
+      ) => {
+        const attrs: Record<string, string> = { "openclaw.state": evt.state };
+        sessionStuckCounter.add(1, attrs);
+        if (typeof evt.ageMs === "number") {
+          sessionStuckAgeHistogram.record(evt.ageMs, attrs);
+        }
+        if (!tracesEnabled) {
+          return;
+        }
+        const spanAttrs: Record<string, string | number> = { ...attrs };
+        addSessionIdentityAttrs(spanAttrs, evt);
+        spanAttrs["openclaw.queueDepth"] = evt.queueDepth ?? 0;
+        spanAttrs["openclaw.ageMs"] = evt.ageMs;
+        const span = tracer.startSpan("openclaw.session.stuck", { attributes: spanAttrs });
+        span.setStatus({ code: SpanStatusCode.ERROR, message: "session stuck" });
+        span.end();
+      };
+
+      const recordRunAttempt = (evt: Extract<DiagnosticEventPayload, { type: "run.attempt" }>) => {
+        runAttemptCounter.add(1, { "openclaw.attempt": evt.attempt });
+      };
+
+      const recordHeartbeat = (
+        evt: Extract<DiagnosticEventPayload, { type: "diagnostic.heartbeat" }>,
+      ) => {
+        queueDepthHistogram.record(evt.queued, { "openclaw.channel": "heartbeat" });
+      };
+
+      unsubscribe = onDiagnosticEvent((evt: DiagnosticEventPayload) => {
+        try {
+          switch (evt.type) {
+            case "model.usage":
+              recordModelUsage(evt);
+              return;
+            case "webhook.received":
+              recordWebhookReceived(evt);
+              return;
+            case "webhook.processed":
+              recordWebhookProcessed(evt);
+              return;
+            case "webhook.error":
+              recordWebhookError(evt);
+              return;
+            case "message.queued":
+              recordMessageQueued(evt);
+              return;
+            case "message.processed":
+              recordMessageProcessed(evt);
+              return;
+            case "queue.lane.enqueue":
+              recordLaneEnqueue(evt);
+              return;
+            case "queue.lane.dequeue":
+              recordLaneDequeue(evt);
+              return;
+            case "session.state":
+              recordSessionState(evt);
+              return;
+            case "session.stuck":
+              recordSessionStuck(evt);
+              return;
+            case "run.attempt":
+              recordRunAttempt(evt);
+              return;
+            case "diagnostic.heartbeat":
+              recordHeartbeat(evt);
+              return;
+          }
+        } catch (err) {
+          ctx.logger.error(
+            `diagnostics-otel: event handler failed (${evt.type}): ${formatError(err)}`,
+          );
+        }
+      });
+
+      if (logsEnabled) {
+        ctx.logger.info("diagnostics-otel: logs exporter enabled (OTLP/Protobuf)");
+      }
+    },
+    async stop() {
+      unsubscribe?.();
+      unsubscribe = null;
+      stopLogTransport?.();
+      stopLogTransport = null;
+      if (logProvider) {
+        await logProvider.shutdown().catch(() => undefined);
+        logProvider = null;
+      }
+      if (sdk) {
+        await sdk.shutdown().catch(() => undefined);
+        sdk = null;
+      }
+    },
+  } satisfies OpenClawPluginService;
+}