| 1 | //! Additive migration: source files are never rewritten, renamed, or deleted. |
| 2 | //! Imported notes are candidates. Export imports receive new local memory IDs. |
| 3 | use crate::{ |
| 4 | Access, Draft, Evidence, Memory, MemoryBackend, Result, Scope, SourceKind, Store, policy, |
| 5 | }; |
| 6 | use serde::{Deserialize, Serialize}; |
| 7 | |
| 8 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 9 | pub struct ImportIssue { |
| 10 | pub line: usize, |
| 11 | pub code: String, |
| 12 | } |
| 13 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 14 | pub struct ImportReport { |
| 15 | pub created: usize, |
| 16 | pub reused: usize, |
| 17 | pub rejected: Vec<ImportIssue>, |
| 18 | pub source_sha256: String, |
| 19 | } |
| 20 | |
| 21 | /// Native MEMORY.md stores bullet notes. Paragraph-form legacy notes also work. |
| 22 | /// Headings and fenced code are skipped; no conversation/thinking dump is ingested. |
| 23 | pub fn parse_markdown(text: &str) -> Vec<(usize, String)> { |
| 24 | let mut notes = Vec::new(); |
| 25 | let mut current = String::new(); |
| 26 | let mut start = 0; |
| 27 | let mut fenced = false; |
| 28 | let flush = |current: &mut String, start: usize, notes: &mut Vec<(usize, String)>| { |
| 29 | if !current.trim().is_empty() { |
| 30 | notes.push((start, current.trim().to_owned())); |
| 31 | } |
| 32 | current.clear(); |
| 33 | }; |
| 34 | for (i, line) in text.lines().enumerate() { |
| 35 | let trimmed = line.trim(); |
| 36 | if trimmed.starts_with("```") || trimmed.starts_with("~~~") { |
| 37 | flush(&mut current, start, &mut notes); |
| 38 | fenced = !fenced; |
| 39 | continue; |
| 40 | } |
| 41 | if fenced { |
| 42 | continue; |
| 43 | } |
| 44 | if trimmed.is_empty() || trimmed.starts_with('#') || trimmed == "---" { |
| 45 | flush(&mut current, start, &mut notes); |
| 46 | continue; |
| 47 | } |
| 48 | if let Some(body) = trimmed |
| 49 | .strip_prefix("- ") |
| 50 | .or_else(|| trimmed.strip_prefix("* ")) |
| 51 | { |
| 52 | flush(&mut current, start, &mut notes); |
| 53 | start = i + 1; |
| 54 | current.push_str(body); |
| 55 | } else { |
| 56 | if current.is_empty() { |
| 57 | start = i + 1; |
| 58 | } else { |
| 59 | current.push('\n'); |
| 60 | } |
| 61 | current.push_str(trimmed); |
| 62 | } |
| 63 | } |
| 64 | flush(&mut current, start, &mut notes); |
| 65 | notes |
| 66 | } |
| 67 | pub fn markdown( |
| 68 | store: &mut Store, |
| 69 | access: &Access, |
| 70 | scope: &Scope, |
| 71 | source_uri: &str, |
| 72 | text: &str, |
| 73 | ) -> Result<ImportReport> { |
| 74 | policy::bounded(text, "Markdown import", 1024 * 1024, false)?; |
| 75 | policy::bounded(source_uri, "import URI", 1024, true)?; |
| 76 | let mut report = ImportReport { |
| 77 | created: 0, |
| 78 | reused: 0, |
| 79 | rejected: Vec::new(), |
| 80 | source_sha256: policy::sha256(text.as_bytes()), |
| 81 | }; |
| 82 | for (line, note) in parse_markdown(text) { |
| 83 | let digest = policy::sha256(note.as_bytes()); |
| 84 | let evidence = Evidence { |
| 85 | kind: SourceKind::Import, |
| 86 | uri: source_uri.into(), |
| 87 | locator: format!("line:{line}"), |
| 88 | sha256: Some(digest.clone()), |
| 89 | observed_at: 0, |
| 90 | }; |
| 91 | let mut draft = Draft::note(scope.clone(), policy::excerpt(¬e, 80), note, evidence); |
| 92 | draft.confidence = 0.25; |
| 93 | let request = format!("markdown:{source_uri}:{line}:{digest}"); |
| 94 | match store.capture(access, &request, draft) { |
| 95 | Ok(r) if r.created => report.created += 1, |
| 96 | Ok(_) => report.reused += 1, |
| 97 | Err(e) => report.rejected.push(ImportIssue { |
| 98 | line, |
| 99 | code: e.code().into(), |
| 100 | }), |
| 101 | } |
| 102 | } |
| 103 | Ok(report) |
| 104 | } |
| 105 | #[derive(Deserialize)] |
| 106 | #[serde(deny_unknown_fields)] |
| 107 | struct ExportRow { |
| 108 | schema: String, |
| 109 | memory: Memory, |
| 110 | } |
| 111 | /// Portable export import, NOT a restore of trusted status or a DB backup. |
| 112 | /// Review, graph edges, grants and original local IDs are intentionally not restored. |
| 113 | pub fn jsonl( |
| 114 | store: &mut Store, |
| 115 | access: &Access, |
| 116 | scope: &Scope, |
| 117 | text: &str, |
| 118 | ) -> Result<ImportReport> { |
| 119 | policy::bounded(text, "JSONL import", 16 * 1024 * 1024, false)?; |
| 120 | let mut report = ImportReport { |
| 121 | created: 0, |
| 122 | reused: 0, |
| 123 | rejected: vec![], |
| 124 | source_sha256: policy::sha256(text.as_bytes()), |
| 125 | }; |
| 126 | for (index, line) in text.lines().enumerate() { |
| 127 | if line.trim().is_empty() { |
| 128 | continue; |
| 129 | } |
| 130 | let result = (|| -> Result<bool> { |
| 131 | let row: ExportRow = serde_json::from_str(line)?; |
| 132 | if row.schema != "codewhale.memory.export.v1" { |
| 133 | return Err(crate::Error::Invalid("unsupported export schema".into())); |
| 134 | } |
| 135 | let source_id = row.memory.id; |
| 136 | let source_hash = row.memory.content_hash; |
| 137 | let mut draft = row.memory.draft; |
| 138 | draft.scope = scope.clone(); |
| 139 | draft.parent_ids.clear(); |
| 140 | draft.evidence = vec![Evidence { |
| 141 | kind: SourceKind::Import, |
| 142 | uri: format!("codewhale-export:{source_id}"), |
| 143 | locator: "imported candidate; prior authority not retained".into(), |
| 144 | sha256: Some(source_hash.clone()), |
| 145 | observed_at: 0, |
| 146 | }]; |
| 147 | Ok(store |
| 148 | .capture(access, &format!("export:{source_id}:{source_hash}"), draft)? |
| 149 | .created) |
| 150 | })(); |
| 151 | match result { |
| 152 | Ok(true) => report.created += 1, |
| 153 | Ok(false) => report.reused += 1, |
| 154 | Err(e) => report.rejected.push(ImportIssue { |
| 155 | line: index + 1, |
| 156 | code: e.code().into(), |
| 157 | }), |
| 158 | } |
| 159 | } |
| 160 | Ok(report) |
| 161 | } |
| 162 |