Skip to content

Set provider import policy defaults #95

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
luca-ctx merged 1 commit into from
Jul 7, 2026
Merged

Set provider import policy defaults #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions crates/ctx-cli/src/commands/import.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use std::{
env, fs,
fs,
io::{Cursor, Read},
path::{Path, PathBuf},
sync::{Arc, Mutex},
Expand Down Expand Up @@ -32,17 +32,17 @@ use ctx_history_capture::{
stable_capture_uuid, validate_custom_history_jsonl_v1, validate_custom_history_jsonl_v1_reader,
AntigravityCliImportOptions, AstrBotSqliteImportOptions, AuggieImportOptions, CatalogSummary,
ClaudeProjectsImportOptions, ClineTaskJsonImportOptions, CodeBuddyImportOptions,
CodexEventImportMode, CodexHistoryImportOptions, CodexSessionCatalogOptions,
CodexSessionImportOptions, CodexSessionImportProgressCallback, CodexToolOutputMode,
ContinueCliImportOptions, CopilotCliImportOptions, CrushSqliteImportOptions,
CursorNativeImportOptions, CustomHistoryJsonlV1ImportOptions, DeepAgentsSqliteImportOptions,
FactoryAiDroidImportOptions, FirebenderSqliteImportOptions, ForgeCodeSqliteImportOptions,
GeminiCliImportOptions, GooseSessionsSqliteImportOptions, HermesSqliteImportOptions,
JunieImportOptions, KiloSqliteImportOptions, KimiCodeCliImportOptions, KiroSqliteImportOptions,
LingmaSqliteImportOptions, MistralVibeImportOptions, MuxImportOptions, NanoClawImportOptions,
OpenClawImportOptions, OpenCodeSqliteImportOptions, OpenHandsImportOptions,
PiSessionImportOptions, ProviderImportSummary, ProviderImportSupport, ProviderSourceStatus,
QoderImportOptions, QwenCodeImportOptions, RooTaskJsonImportOptions, RovoDevImportOptions,
CodexHistoryImportOptions, CodexSessionCatalogOptions, CodexSessionImportOptions,
CodexSessionImportProgressCallback, ContinueCliImportOptions, CopilotCliImportOptions,
CrushSqliteImportOptions, CursorNativeImportOptions, CustomHistoryJsonlV1ImportOptions,
DeepAgentsSqliteImportOptions, FactoryAiDroidImportOptions, FirebenderSqliteImportOptions,
ForgeCodeSqliteImportOptions, GeminiCliImportOptions, GooseSessionsSqliteImportOptions,
HermesSqliteImportOptions, JunieImportOptions, KiloSqliteImportOptions,
KimiCodeCliImportOptions, KiroSqliteImportOptions, LingmaSqliteImportOptions,
MistralVibeImportOptions, MuxImportOptions, NanoClawImportOptions, OpenClawImportOptions,
OpenCodeSqliteImportOptions, OpenHandsImportOptions, PiSessionImportOptions,
ProviderImportSummary, ProviderImportSupport, ProviderSourceStatus, QoderImportOptions,
QwenCodeImportOptions, RooTaskJsonImportOptions, RovoDevImportOptions,
ShelleySqliteImportOptions, TabnineCliImportOptions, TraeImportOptions,
WarpSqliteImportOptions, WindsurfCascadeHookImportOptions, ZedThreadsSqliteImportOptions,
};
Expand Down
46 changes: 0 additions & 46 deletions crates/ctx-cli/src/commands/import/catalog.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ pub(crate) fn import_incremental_codex_session_tree(
store: &mut Store,
source: &SourceInfo,
record_id: Uuid,
tool_output_mode: CodexToolOutputMode,
event_mode: CodexEventImportMode,
include_notices: bool,
progress: Option<CodexSessionImportProgressCallback>,
allow_partial_failures: bool,
catalog_before_import: bool,
Expand Down Expand Up @@ -70,9 +67,6 @@ pub(crate) fn import_incremental_codex_session_tree(
source_path: Some(source.path.clone()),
history_record_id: Some(record_id),
allow_partial_failures,
tool_output_mode,
event_mode,
include_notices,
progress: progress.clone(),
..CodexSessionImportOptions::default()
},
Expand Down Expand Up @@ -132,9 +126,6 @@ pub(crate) fn import_incremental_codex_session_tree(
source_path: Some(source.path.clone()),
history_record_id: Some(record_id),
allow_partial_failures,
tool_output_mode,
event_mode,
include_notices,
progress,
..CodexSessionImportOptions::default()
},
Expand Down Expand Up @@ -295,43 +286,6 @@ pub(crate) fn source_uses_incremental_event_search(source: &SourceInfo) -> bool
)
}

pub(crate) fn codex_tool_output_mode() -> Result<CodexToolOutputMode> {
if let Some(raw) = env::var_os("CTX_CODEX_TOOL_OUTPUT_MODE") {
let raw = raw.to_string_lossy();
return match raw.as_ref() {
"full" => Ok(CodexToolOutputMode::Full),
"metadata" => Ok(CodexToolOutputMode::Metadata),
"failures" | "failure" | "errors" | "error" => Ok(CodexToolOutputMode::Failures),
"skip" => Ok(CodexToolOutputMode::Skip),
other => Err(anyhow!(
"unsupported CTX_CODEX_TOOL_OUTPUT_MODE={other:?}; expected full, metadata, failures, or skip"
)),
};
}
if env::var_os("CTX_EXPERIMENTAL_SKIP_TOOL_OUTPUTS").is_some() {
return Ok(CodexToolOutputMode::Skip);
}
Ok(CodexToolOutputMode::Skip)
}

pub(crate) fn codex_event_import_mode() -> Result<CodexEventImportMode> {
if let Some(raw) = env::var_os("CTX_CODEX_EVENT_MODE") {
let raw = raw.to_string_lossy();
return match raw.as_ref() {
"search" | "message" | "messages" => Ok(CodexEventImportMode::Search),
"rich" | "full" => Ok(CodexEventImportMode::Rich),
other => Err(anyhow!(
"unsupported CTX_CODEX_EVENT_MODE={other:?}; expected search or rich"
)),
};
}
Ok(CodexEventImportMode::Search)
}

pub(crate) fn codex_include_notices() -> bool {
env::var_os("CTX_CODEX_INCLUDE_NOTICES").is_some()
}

pub(crate) fn source_stats(path: &Path) -> Result<SourceStats> {
let metadata = fs::symlink_metadata(path)
.with_context(|| format!("stat import source {}", path.display()))?;
Expand Down
25 changes: 1 addition & 24 deletions crates/ctx-cli/src/commands/import/native.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
use std::collections::BTreeMap;

use super::*;
use crate::commands::import::catalog::{
codex_event_import_mode, codex_include_notices, codex_tool_output_mode, system_time_ms,
};
use crate::commands::import::catalog::system_time_ms;

pub(crate) fn validate_source_import_supported(source: &SourceInfo) -> Result<()> {
match source.import_support {
Expand Down Expand Up @@ -71,9 +69,6 @@ pub(crate) fn import_one_source_inner(
allow_partial_failures: bool,
preinventory: &SourcePreinventory,
) -> Result<ProviderImportSummary> {
let tool_output_mode = codex_tool_output_mode()?;
let event_mode = codex_event_import_mode()?;
let include_notices = codex_include_notices();
let record = import_record_for_source(source);
let record_id = record.id;
store.upsert_record(&record)?;
Expand All @@ -82,9 +77,6 @@ pub(crate) fn import_one_source_inner(
store,
source,
record_id,
tool_output_mode,
event_mode,
include_notices,
progress,
allow_partial_failures,
preinventory.source_import_files(),
Expand All @@ -101,9 +93,6 @@ pub(crate) fn import_one_source_inner(
source_path: Some(source.path.clone()),
history_record_id: Some(record_id),
allow_partial_failures,
tool_output_mode,
event_mode,
include_notices,
progress: progress.clone(),
..CodexSessionImportOptions::default()
},
Expand All @@ -114,9 +103,6 @@ pub(crate) fn import_one_source_inner(
store,
source,
record_id,
tool_output_mode,
event_mode,
include_notices,
progress.clone(),
allow_partial_failures,
!preinventory.codex_session_tree_cataloged(),
Expand Down Expand Up @@ -147,9 +133,6 @@ pub(crate) fn import_one_source_inner(
source_path: Some(source.path.clone()),
history_record_id: Some(record_id),
allow_partial_failures,
tool_output_mode,
event_mode,
include_notices,
progress,
..CodexSessionImportOptions::default()
},
Expand Down Expand Up @@ -682,9 +665,6 @@ pub(crate) fn import_manifested_source(
store: &mut Store,
source: &SourceInfo,
record_id: Uuid,
tool_output_mode: CodexToolOutputMode,
event_mode: CodexEventImportMode,
include_notices: bool,
progress: Option<CodexSessionImportProgressCallback>,
allow_partial_failures: bool,
preinventoried_files: Option<&[SourceImportFile]>,
Expand Down Expand Up @@ -798,9 +778,6 @@ pub(crate) fn import_manifested_source(
}

let _ = record_id;
let _ = tool_output_mode;
let _ = event_mode;
let _ = include_notices;
Ok(summary)
}

Expand Down
9 changes: 9 additions & 0 deletions crates/ctx-cli/src/docs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,15 @@ const TOPICS: &[DocTopic] = &[
source_path: "docs/provider-support.md",
body: include_str!("../../../docs/provider-support.md"),
},
DocTopic {
id: "provider-import-policy",
title: "Provider Import Policy",
audience: "integrator-agent",
summary: "Native provider content policy, storage families, and fixture expectations.",
tags: &["providers", "imports", "policy", "testing"],
source_path: "docs/provider-import-policy.md",
body: include_str!("../../../docs/provider-import-policy.md"),
},
DocTopic {
id: "troubleshooting",
title: "Troubleshooting",
Expand Down
100 changes: 95 additions & 5 deletions crates/ctx-cli/tests/native_providers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,96 @@ fn native_provider_cli_flow_imports_supported_provider_paths() {
assert_search_provider_oracle(&search, stored_provider, &query, 1, "message");
}
}

#[test]
fn native_provider_cli_policy_excludes_success_tool_outputs_from_search_and_payloads() {
let temp = tempdir();
let qoder_query = "qoder-policy-real-message-oracle";
let qoder_path = write_native_qoder_fixture(&temp, qoder_query);
let openhands_query = "openhands-policy-real-message-oracle";
let openhands_path = write_native_openhands_fixture(&temp, openhands_query);
let continue_query = "continue-policy-real-message-oracle";
let continue_path = write_native_continue_fixture(&temp, continue_query);

for (provider, path, query) in [
("qoder", qoder_path.as_str(), qoder_query),
("openhands", openhands_path.as_str(), openhands_query),
("continue", continue_path.as_str(), continue_query),
] {
let imported = json_output(ctx(&temp).args([
"import",
"--provider",
provider,
"--path",
path,
"--json",
"--progress",
"none",
]));
assert_eq!(imported["totals"]["failed"], 0, "{imported:#}");

let search = json_output(ctx(&temp).args([
"search",
query,
"--provider",
provider,
"--refresh",
"off",
"--json",
]));
assert_search_provider_oracle(&search, provider, query, 1, "message");
}

for (provider, sentinel) in [
("qoder", "native qoder fixture result"),
("openhands", "Edited openhands-cli-native-oracle.txt"),
("continue", "Continue tool output marker"),
] {
let search = json_output(ctx(&temp).args([
"search",
sentinel,
"--provider",
provider,
"--refresh",
"off",
"--json",
]));
assert!(
search["results"].as_array().unwrap().is_empty(),
"{provider} success tool output leaked into search: {search:#}"
);
}

let conn = Connection::open(temp.path().join("work.sqlite")).unwrap();
assert_eq!(
sqlite_count(
&conn,
"SELECT COUNT(*) FROM events WHERE payload_json LIKE '%native qoder fixture result%'",
),
0
);
assert_eq!(
sqlite_count(
&conn,
"SELECT COUNT(*) FROM events WHERE payload_json LIKE '%Edited openhands-cli-native-oracle.txt%'",
),
0
);
assert_eq!(
sqlite_count(
&conn,
"SELECT COUNT(*) FROM events WHERE payload_json LIKE '%Continue tool output marker%'",
),
0
);
assert!(
sqlite_count(
&conn,
"SELECT COUNT(*) FROM files_touched WHERE path = 'openhands-cli-native-oracle.txt'",
) > 0
);
}

#[test]
fn trae_cli_imports_explicit_workspace_storage_with_default_discovery() {
let temp = tempdir();
Expand Down Expand Up @@ -888,8 +978,8 @@ fn sqlite_cli_imports_crush_goose_zed_kiro_and_forgecode_and_searches() {
"crush_sqlite",
"crush/v1/crush.db",
"crush oracle",
2,
4,
1,
3,
),
(
"goose",
Expand Down Expand Up @@ -1311,9 +1401,9 @@ fn antigravity_cli_imports_native_transcript_tree() {
imported["sources"][0]["source_format"],
"antigravity_cli_transcript_jsonl_tree"
);
assert_eq!(imported["totals"]["imported_sessions"], 4);
assert_eq!(imported["totals"]["imported_events"], 11);
assert_eq!(imported["totals"]["failed"], 1);
assert_eq!(imported["totals"]["imported_sessions"], 3);
assert_eq!(imported["totals"]["imported_events"], 9);
assert_eq!(imported["totals"]["failed"], 2);

let search = json_output(ctx(&temp).args([
"search",
Expand Down
18 changes: 7 additions & 11 deletions crates/ctx-history-capture/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@ pub(crate) const FORGECODE_SQLITE_SOURCE_FORMAT: &str = "forgecode_sqlite";
pub(crate) const DEEPAGENTS_SQLITE_SOURCE_FORMAT: &str = "deepagents_sessions_sqlite";
pub(crate) const MISTRAL_VIBE_SOURCE_FORMAT: &str = "mistral_vibe_session_jsonl";
pub(crate) const MUX_SOURCE_FORMAT: &str = "mux_session_jsonl";
pub(crate) const CODEX_MAX_TEXT_CHARS: usize = 16_000;
pub(crate) const CODEX_MAX_METADATA_TEXT_CHARS: usize = 4_000;
pub(crate) const CODEX_MAX_OUTPUT_PREVIEW_CHARS: usize = 4_000;
pub(crate) const PROVIDER_MAX_TEXT_CHARS: usize = 16_000;
pub(crate) const PROVIDER_MAX_PREVIEW_CHARS: usize = 4_000;
pub(crate) const CODEX_FAST_IMPORT_TRANSACTION_FILES: usize = 512;
Expand All @@ -72,14 +69,13 @@ mod options;
pub use options::{
AntigravityCliImportOptions, AstrBotSqliteImportOptions, AuggieImportOptions,
ClaudeProjectsImportOptions, ClineTaskJsonImportOptions, CodeBuddyImportOptions,
CodexEventImportMode, CodexHistoryImportOptions, CodexSessionCatalogOptions,
CodexSessionImportOptions, CodexSessionImportProgress, CodexSessionImportProgressCallback,
CodexToolOutputMode, ContinueCliImportOptions, CopilotCliImportOptions,
CrushSqliteImportOptions, CursorNativeImportOptions, CustomHistoryJsonlV1ImportOptions,
DeepAgentsSqliteImportOptions, FactoryAiDroidImportOptions, FirebenderSqliteImportOptions,
FixtureOptions, ForgeCodeSqliteImportOptions, GeminiCliImportOptions,
GooseSessionsSqliteImportOptions, HermesSqliteImportOptions, JunieImportOptions,
KiloSqliteImportOptions, KimiCodeCliImportOptions, KiroSqliteImportOptions,
CodexHistoryImportOptions, CodexSessionCatalogOptions, CodexSessionImportOptions,
CodexSessionImportProgress, CodexSessionImportProgressCallback, ContinueCliImportOptions,
CopilotCliImportOptions, CrushSqliteImportOptions, CursorNativeImportOptions,
CustomHistoryJsonlV1ImportOptions, DeepAgentsSqliteImportOptions, FactoryAiDroidImportOptions,
FirebenderSqliteImportOptions, FixtureOptions, ForgeCodeSqliteImportOptions,
GeminiCliImportOptions, GooseSessionsSqliteImportOptions, HermesSqliteImportOptions,
JunieImportOptions, KiloSqliteImportOptions, KimiCodeCliImportOptions, KiroSqliteImportOptions,
LingmaSqliteImportOptions, MistralVibeImportOptions, MuxImportOptions, NanoClawImportOptions,
OpenClawImportOptions, OpenCodeSqliteImportOptions, OpenHandsImportOptions,
PiSessionImportOptions, ProviderFixtureImportOptions, QoderImportOptions,
Expand Down
Loading
Loading