Skip to content

Rollout JSONL grew to 98 GB from inline Base64 PNG payloads #40111

Open

Description

@marylglez

What version of Codex CLI is running?

codex-cli 0.148.0

What subscription do you have?

ChatGPT Pro Lite

Which model were you using?

gpt-5.6-sol (reasoning medium)

What platform is your computer?

Darwin 25.5.0 arm64 arm

What terminal emulator and version are you using (if applicable)?

Orca 1.4.186 embedded terminal, zsh, no terminal multiplexer

Codex doctor report

{
  "schemaVersion": 1,
  "generatedAt": "1787405798s since unix epoch",
  "overallStatus": "warning",
  "codexVersion": "0.148.0",
  "checks": {
    "app_server.status": {
      "id": "app_server.status",
      "category": "app-server",
      "status": "ok",
      "summary": "background server is not running",
      "details": {
        "control socket": "/Users/<redacted>/.codex/app-server-control/app-server-control.sock",
        "daemon state dir": "/Users/<redacted>/.codex/app-server-daemon",
        "mode": "ephemeral",
        "pid file": "/Users/<redacted>/.codex/app-server-daemon/app-server.pid (missing)",
        "settings": "/Users/<redacted>/.codex/app-server-daemon/settings.json (missing)",
        "status": "not running",
        "update-loop pid file": "/Users/<redacted>/.codex/app-server-daemon/app-server-updater.pid (missing)"
      },
      "remediation": null,
      "durationMs": 0
    },
    "auth.credentials": {
      "id": "auth.credentials",
      "category": "auth",
      "status": "ok",
      "summary": "auth is configured",
      "details": {
        "auth file": "/Users/<redacted>/.codex/auth.json",
        "auth storage mode": "File",
        "stored API key": "false",
        "stored ChatGPT tokens": "true",
        "stored agent identity": "false",
        "stored auth mode": "chatgpt"
      },
      "remediation": null,
      "durationMs": 0
    },
    "config.load": {
      "id": "config.load",
      "category": "config",
      "status": "ok",
      "summary": "config loaded",
      "details": {
        "CODEX_HOME": "/Users/<redacted>/.codex",
        "config.toml": "/Users/<redacted>/.codex/config.toml",
        "config.toml parse": "ok",
        "cwd": "/Users/<redacted>/project",
        "enabled feature flags": "shell_tool, view_image, unified_exec, shell_snapshot, code_mode_host, terminal_resize_reflow, sqlite, hooks, enable_request_compression, unbounded_connection_retries, multi_agent, apps, tool_search_always_defer_mcp_tools, tool_suggest, plugins, in_app_browser, in_app_updates, browser_use, browser_use_full_cdp_access, browser_use_external, computer_use, remote_plugin, plugin_sharing, image_generation, resize_all_images, item_ids, skill_mcp_dependency_install, skill_search, mentions_v2, steer, guardian_approval, goals, collaboration_modes, tool_call_mcp_elicitation, auth_elicitation, personality, fast_mode, tui_app_server, remote_compaction_v2, workspace_dependencies",
        "feature flag overrides": "none",
        "feature flags enabled": "40",
        "log dir": "/Users/<redacted>/.codex/log",
        "mcp servers": "3",
        "model": "gpt-5.6-sol",
        "model provider": "openai",
        "sqlite home": "/Users/<redacted>/.codex"
      },
      "remediation": null,
      "durationMs": 0
    },
    "git.environment": {
      "id": "git.environment",
      "category": "git",
      "status": "ok",
      "summary": "git version 2.38.1",
      "details": {
        ".git entry": "file -> /Users/<redacted>/project/.git/worktrees/loach",
        "PATH git #1": "/opt/homebrew/bin/git",
        "PATH git #2": "/usr/bin/git",
        "PATH git entries": "2",
        "git branch": "<redacted>",
        "git build options": "git version 2.38.1; cpu: arm64; no commit associated with this build; sizeof-long: 8; sizeof-size_t: 8; shell-path: /bin/sh; feature: fsmonitor--daemon",
        "git exec path": "/opt/homebrew/Cellar/git/2.38.1/libexec/git-core",
        "git version": "git version 2.38.1",
        "repo detected": "true",
        "repo root": "/Users/<redacted>/project",
        "selected git": "/opt/homebrew/bin/git"
      },
      "remediation": null,
      "durationMs": 149
    },
    "installation": {
      "id": "installation",
      "category": "install",
      "status": "ok",
      "summary": "installation looks consistent",
      "details": {
        "PATH codex #1": "/Users/<redacted>/.nvm/versions/node/v20.0.0/bin/codex",
        "PATH codex #2": "/Users/<redacted>/.nvm/versions/node/v20.0.0/bin/codex",
        "PATH codex #3": "/Users/<redacted>/.nvm/versions/node/v22.22.2/bin/codex",
        "PATH codex entries": "3",
        "current executable": "/Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin/codex",
        "install context": "npm (package /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin, bin /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin, resources /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/codex-resources, path /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/codex-path)",
        "managed by bun": "false",
        "managed by npm": "true",
        "managed by pnpm": "false",
        "managed package root": "/Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex",
        "npm update target": "/Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex"
      },
      "remediation": null,
      "durationMs": 298
    },
    "mcp.config": {
      "id": "mcp.config",
      "category": "mcp",
      "status": "ok",
      "summary": "MCP configuration is locally consistent",
      "details": {
        "configured servers": "3",
        "disabled servers": "1",
        "stdio servers": "3"
      },
      "remediation": null,
      "durationMs": 0
    },
    "network.env": {
      "id": "network.env",
      "category": "network",
      "status": "ok",
      "summary": "network-related environment looks readable",
      "details": {
        "proxy env vars": "none"
      },
      "remediation": null,
      "durationMs": 0
    },
    "network.provider_reachability": {
      "id": "network.provider_reachability",
      "category": "reachability",
      "status": "ok",
      "summary": "active provider endpoints are reachable over HTTP",
      "details": {
        "ChatGPT base URL": "https://chatgpt.com/backend-api/ reachable (HTTP 404)",
        "reachability mode": "ChatGPT auth"
      },
      "remediation": null,
      "durationMs": 259
    },
    "network.websocket_reachability": {
      "id": "network.websocket_reachability",
      "category": "websocket",
      "status": "ok",
      "summary": "Responses WebSocket handshake succeeded",
      "details": {
        "DNS": "2 IPv4, 2 IPv6, first IPv6",
        "auth mode": "chatgpt",
        "connect timeout": "15000 ms",
        "endpoint": "wss://chatgpt.com/backend-api/<redacted>",
        "handshake result": "HTTP 101 Switching Protocols",
        "model provider": "openai",
        "provider name": "OpenAI",
        "proxy env vars": "none",
        "reasoning header": "false",
        "server model present": "false",
        "supports websockets": "true",
        "wire API": "responses"
      },
      "remediation": null,
      "durationMs": 872
    },
    "runtime.provenance": {
      "id": "runtime.provenance",
      "category": "runtime",
      "status": "ok",
      "summary": "running npm on macos-aarch64",
      "details": {
        "commit": "unknown",
        "current executable": "/Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin/codex",
        "install method": "npm (package /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin, bin /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin, resources /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/codex-resources, path /Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/codex-path)",
        "platform": "macos-aarch64",
        "version": "0.148.0"
      },
      "remediation": null,
      "durationMs": 0
    },
    "runtime.search": {
      "id": "runtime.search",
      "category": "search",
      "status": "ok",
      "summary": "search is OK (bundled)",
      "details": {
        "search command": "/Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/codex-path/rg",
        "search command readiness": "file exists",
        "search provider": "bundled"
      },
      "remediation": null,
      "durationMs": 0
    },
    "sandbox.helpers": {
      "id": "sandbox.helpers",
      "category": "sandbox",
      "status": "ok",
      "summary": "sandbox configuration is readable",
      "details": {
        "approval policy": "OnRequest",
        "codex-linux-sandbox helper": "none",
        "execve wrapper helper": "/Users/<redacted>/.codex/tmp/arg0/codex-arg0djet4M/codex-execve-wrapper",
        "filesystem sandbox": "restricted",
        "network sandbox": "restricted"
      },
      "remediation": null,
      "durationMs": 0
    },
    "state.paths": {
      "id": "state.paths",
      "category": "state",
      "status": "ok",
      "summary": "state paths and databases are inspectable",
      "details": {
        "CODEX_HOME": "/Users/<redacted>/.codex (dir)",
        "active rollout files": "21 files, 548428704 total bytes, 26115652 average bytes",
        "archived rollout files": "1 files, 100604 total bytes, 100604 average bytes",
        "goals DB": "/Users/<redacted>/.codex/goals_1.sqlite (file)",
        "goals DB integrity": "ok",
        "log DB": "/Users/<redacted>/.codex/logs_2.sqlite (file)",
        "log DB integrity": "ok",
        "log dir": "/Users/<redacted>/.codex/log (missing)",
        "memories DB": "/Users/<redacted>/.codex/memories_1.sqlite (file)",
        "memories DB integrity": "ok",
        "queue DB": "/Users/<redacted>/.codex/queue_1.sqlite (file)",
        "queue DB integrity": "ok",
        "sqlite home": "/Users/<redacted>/.codex (dir)",
        "state DB": "/Users/<redacted>/.codex/state_5.sqlite (file)",
        "state DB integrity": "ok",
        "thread history DB": "/Users/<redacted>/.codex/thread_history_1.sqlite (file)",
        "thread history DB integrity": "ok"
      },
      "remediation": null,
      "durationMs": 328
    },
    "state.rollout_db_parity": {
      "id": "state.rollout_db_parity",
      "category": "threads",
      "status": "warning",
      "summary": "rollout files and state DB thread inventory differ",
      "details": {
        "default model provider": "openai",
        "rollout DB active files": "21",
        "rollout DB active rows": "22",
        "rollout DB archive mismatches": "0",
        "rollout DB archived files": "1",
        "rollout DB archived rows": "1",
        "rollout DB duplicate DB paths": "0",
        "rollout DB duplicate rollout thread ids": "0",
        "rollout DB malformed file names": "0",
        "rollout DB missing active rows": "0",
        "rollout DB missing archived rows": "0",
        "rollout DB model providers": "openai=23",
        "rollout DB rows": "23",
        "rollout DB scan cap reached": "false",
        "rollout DB scan errors": "0",
        "rollout DB sources": "cli=10, vscode=8, subagent:other=5",
        "rollout DB stale row sample": "/Users/<redacted>/.codex/sessions/2026/08/18/rollout-2026-08-18T20-16-19-01a01797-3108-77d3-bbcb-9f385c32e5b2.jsonl",
        "rollout DB stale rows": "1"
      },
      "issues": [
        {
          "severity": "warning",
          "cause": "state DB rows point at missing or unusable rollout files",
          "measured": "1 stale rows",
          "expected": "every state DB rollout path is a file on disk",
          "remedy": null,
          "fields": []
        }
      ],
      "remediation": null,
      "durationMs": 190
    },
    "system.disk": {
      "id": "system.disk",
      "category": "disk",
      "status": "ok",
      "summary": "sufficient free disk space (122.7 GiB)",
      "details": {
        "CODEX_HOME available": "122.7 GiB",
        "failure threshold": "1.0 GiB",
        "warning threshold": "5.0 GiB",
        "worktree available": "122.7 GiB"
      },
      "remediation": null,
      "durationMs": 0
    },
    "system.environment": {
      "id": "system.environment",
      "category": "system",
      "status": "ok",
      "summary": "OS language en-MX",
      "details": {
        "EDITOR": "not set",
        "GH_PAGER": "set",
        "GIT_PAGER": "set",
        "LANG": "en_US.UTF-8",
        "LC_ALL": "C.UTF-8",
        "LC_CTYPE": "C.UTF-8",
        "PAGER": "set",
        "VISUAL": "not set",
        "os": "Mac OS 26.5.0 [64-bit]",
        "os language": "en-MX",
        "os type": "Mac OS",
        "os version": "26.5.0"
      },
      "remediation": null,
      "durationMs": 2
    },
    "terminal.env": {
      "id": "terminal.env",
      "category": "terminal",
      "status": "ok",
      "summary": "terminal metadata was detected",
      "details": {
        "COLORTERM": "truecolor",
        "NO_COLOR": "1",
        "TERM_PROGRAM": "Orca",
        "color output": "disabled (NO_COLOR)",
        "effective locale": "C.UTF-8",
        "stderr is terminal": "false",
        "stdin is terminal": "false",
        "stdout is terminal": "false",
        "terminal": "unknown",
        "terminal size": "80x24",
        "terminal version": "1.4.186"
      },
      "remediation": null,
      "durationMs": 6
    },
    "terminal.title": {
      "id": "terminal.title",
      "category": "title",
      "status": "ok",
      "summary": "terminal title default",
      "details": {
        "terminal title activity": "true",
        "terminal title items": "activity, project-name",
        "terminal title project source": "git repo root",
        "terminal title project value": "<redacted>",
        "terminal title source": "default"
      },
      "remediation": null,
      "durationMs": 0
    },
    "updates.status": {
      "id": "updates.status",
      "category": "updates",
      "status": "ok",
      "summary": "update configuration is locally consistent",
      "details": {
        "cached latest version": "0.149.0",
        "check for update on startup": "true",
        "dismissed version": "0.149.0",
        "last checked at": "2026-08-22T12:53:23.571309Z",
        "latest version": "0.149.0",
        "latest version status": "newer version is available",
        "npm update target": "/Users/<redacted>/.nvm/versions/node/v20.0.0/lib/node_modules/@openai/codex",
        "update action": "npm install -g @openai/codex",
        "version cache": "/Users/<redacted>/.codex/version.json"
      },
      "remediation": null,
      "durationMs": 463
    }
  }
}

What issue are you seeing?

Codex version

0.148.0

Environment

  • macOS 26.5 arm64
  • Codex CLI
  • Image-heavy design/PDF workflow
  • Session persisted from August 18 through August 22

Problem

A single rollout JSONL under ~/.codex/sessions/ grew to exactly 98,277,570,375 bytes (~91.5 GiB). occupied approximately 92 GB. The disk reached 97–98% capacity, leaving only 12–13 GiB free.

Inspection showed complete PNG payloads stored inline in session records:

{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/
png;base64,iVBORw0KGgo...<redacted>"}]}}
png;base64,iVBORw0KGgo...<redacted>"}]}}

Early individual image-bearing records were approximately 5.15 MB, 7.76 MB, and 6.76 MB. Image payloads continued accumulating throughout the rollout.

### Impact

- Approximately 92 GB consumed by one session log
- Disk nearly exhausted
- Risk of application or system failure/data loss
- File had to be permanently deleted
- Deleting it increased available disk space from approximately 13 GB
  to 105 GB

### Expected behavior

- Image payloads should be referenced or deduplicated instead of repeatedly
  embedded in rollout history.

- Session logs should have rotation or maximum-size limits.
- Codex should warn or stop before a session log exhausts the disk.
- Old or redundant image payloads should be compacted or pruned safely.

### Additional information

The file was deleted after confirming that no process had it open, because
the machine was critically low on disk space. OpenAI Support has also noted
the incident in a support conversation.

### What steps can reproduce the bug?

I have not attempted to reproduce the full 98 GB growth because doing so could exhaust the disk again.

Observed workflow:

1. Start Codex CLI 0.148.0 in a project.
2. Keep the same session active/resumable across multiple days.
3. Attach PNG moodboard/reference images.
4. Perform an image-heavy design and PDF workflow involving rendered pages and image-returning tool calls.
5. Inspect the session rollout under `~/.codex/sessions/`.

The rollout contained complete PNG payloads inline in both user-image and tool-output records:

```json
{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/
png;base64,iVBORw0KGgo...<redacted>"}]}}

{"type":"response_item","payload":{"type":"custom_tool_call_output","output":[{"type":"input_image","image_url":"data:image/
png;base64,iVBORw0KGgo...<redacted>"}]}}

Early individual image-bearing records were approximately 5.15 MB, 7.76 MB, and 6.76 MB. These payloads continued accumulating as the session was used.

The resulting rollout reached exactly 98,277,570,375 bytes (~91.5 GiB).

### What is the expected behavior?

```markdown
Image payloads should be referenced, deduplicated, compressed, or pruned instead of allowing a session rollout to grow without bounds.

Codex should enforce a reasonable maximum size or rotation policy for rollout files and warn the user before local session data can exhaust the
disk.

Resuming an image-heavy session should not repeatedly persist unnecessary copies of full Base64 image payloads.

### Additional information

The project itself occupied only 584 MB, while this single rollout consumed approximately 92 GB.

The disk reached 97–98% capacity, leaving only 12–13 GiB free. After confirming that no process had the rollout open, I permanently deleted it. Available disk space increased from approximately 13 GB to 105 GB.

`codex doctor --json` now reports a stale state-database row pointing to the deleted rollout:

- `state.rollout_db_parity`: warning
- Cause: state DB rows point at missing or unusable rollout files
- Measured: 1 stale row

The original rollout could not be retained or attached because the machine was critically low on disk space. OpenAI Support has also recorded the incident in a support conversation.

Activity

  1. added
    CLIIssues related to the Codex CLI
    sessionIssues involving session (thread) management, resuming, forking, naming, archiving
    on Aug 22, 2026
  2. github-actions commented on Aug 22, 2026

    @github-actions
    Contributor
  3. gunsnmotorcycles commented on Aug 28, 2026

    @gunsnmotorcycles

    Additional Windows/VS Code evidence from Codex desktop and extension:

    • Windows 11, Codex desktop 26.820.10647.0, VS Code extension 26.825.32147.
    • ~/.codex/sessions reached 72.9 GB. Sixteen old rollout JSONLs over 100 MB totaled 69.06 GB; the largest were 37.91 GB and 16.93 GB.
    • Random 64 KiB samples through the 37.91 GB file were 95.65-100% Base64-alphabet bytes and contained input_image records, confirming inline image payloads rather than repository data.
    • Starting the VS Code extension app-server while these files were active caused its bundled codex.exe to grow to 11.7 GB private/working set while idle, pushing system commit from about 53% to 80% and making the extension unusable.
    • Moving only the old oversized rollouts out of the active sessions tree immediately prevented that replay path; active session data is now about 1.0 GB and the files remain recoverable.

    This shows the impact is not only disk exhaustion: app-server/session discovery can turn oversized image rollouts into a reproducible Windows memory/OOM problem. A useful guard would combine externalized/deduplicated images, rollout size limits, and bounded/lazy history loading in app-server clients.

  4. mellowmarshall commented on Aug 29, 2026

    @mellowmarshall

    I traced a remaining durable-storage path that is not covered by the compaction image budget in #40280 or the notification filter in #41416.

    On current main, LiveThread::persist_appended_items computes items through persisted_rollout_items(...), but the durable append receives a fresh clone of the unprojected batch:

    let (items, measurement) = /* persistence projection */;
    self.thread_store
        .append_items(AppendThreadItemsParams {
            thread_id: self.thread_id,
            items: raw_items.to_vec(),
        })
        .await?;

    Source:

    async fn persist_appended_items(
    &self,
    raw_items: &[RolloutItem],
    ) -> ThreadStoreResult<Vec<RolloutItem>> {
    // Empty appends are intentionally ignored rather than represented as zero-sized batches.
    if raw_items.is_empty() {
    return Ok(Vec::new());
    }
    let (items, measurement) = if self.persistence_telemetry.is_enabled() {
    let (items, measurement) =
    measure_and_filter_rollout_items(raw_items, self.history_mode);
    (items, Some(measurement))
    } else {
    (persisted_rollout_items(raw_items, self.history_mode), None)
    };
    self.thread_store
    .append_items(AppendThreadItemsParams {
    thread_id: self.thread_id,
    items: raw_items.to_vec(),
    })
    .await?;
    if let Some(measurement) = measurement.as_ref() {
    self.persistence_telemetry
    .record_batch(raw_items, measurement);
    }
    Ok(items)

    The returned items are used for metadata synchronization, not for that append. This permits full inline tool media and image-generation results to remain in durable rollout records. It also makes an additional full allocation of the raw media batch before the lower store processes it.

    I implemented and tested a focused reference patch here:

    mellowmarshall@d7de404

    The patch:

    • makes persisted_rollout_item the single durable projection boundary;
    • replaces inline data:image/ and data:audio/ tool-output payloads with small placeholders;
    • clears image-generation result bytes while retaining status, prompt, failure, and saved-path metadata;
    • applies the same rule to compacted replacement history, legacy events, dynamic tool results, imports, migration canonicalization, and persistence metrics;
    • passes the projected batch across the thread-store seam instead of cloning raw_items;
    • leaves live model input unchanged; only durable history is bounded;
    • adds regression coverage for each media-bearing representation and the migration path.

    This complements the existing compaction and notification fixes. It addresses the separate case where tool-returned or generated media remains durable and is cloned during append. The commit is offered as a tested implementation reference because this repository does not accept external code pull requests.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    CLIIssues related to the Codex CLIbugSomething isn't workingsessionIssues involving session (thread) management, resuming, forking, naming, archiving

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions