diff --git a/README.md b/README.md index 1a21255..d4713a6 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,53 @@ To dump all embedded file in example.one into output_dir and add .bin to the end pyonenote -f example.one -o output_dir -e bin ``` +## Bulk markdown export + +To extract the page text of a section to clean markdown (embedded null bytes +stripped, historical page revisions de-duplicated, one `##` heading per page): +``` +pyonenote -m -f example.one -o output_dir +``` + +`-f` may also point at a directory, in which case every `.one` file in it is +exported to its own `
.md` inside `output_dir` — useful for dumping a +whole notebook's worth of sections in one pass for scripted diffing or pattern +extraction: +``` +pyonenote -m -f "/path/to/OneNote Backup/MyNotebook" -o markdown_out +``` + +### Redacting secrets + +Add `-r` / `--redact-secrets` to replace credentials with visible +`[redacted:*]` markers — useful when output will be shared for review: +``` +pyonenote -r -f "MyNotebook/Section.one" # default dump +pyonenote -r -j -f "MyNotebook/Section.one" # JSON +pyonenote -r -m -f "MyNotebook" -o markdown_out # markdown +``` +`-r` applies to **every output mode** — the default dump, JSON (`-j`), markdown +(`-m`), and any text attachment written to `--output-dir`. Binary attachments +(images, Office docs) contain no text to scrub and are written unchanged. + +Redaction matches only high-confidence, structurally-anchored shapes (see +`pyOneNote/redact.py`), so ordinary note text is left untouched: +- **Vendor sentinels**: prefix/structure-keyed API keys and tokens — + AWS/GitHub/GitLab/OpenAI/Anthropic/Slack/Stripe/Google/npm/PyPI/SendGrid/ + Twilio keys, JWTs, and `Authorization: Bearer` tokens. +- **PEM private keys**: `PRIVATE KEY` blocks (RSA/EC/DSA/OPENSSH/ENCRYPTED/PGP). + Public certificates, public keys, and CSRs are meant to be shared and are left + intact. + +There is deliberately no generic entropy, keyword-proximity, or base64/blob +detection, so ordinary text and embedded data (images, encoded content) are left +untouched and the pass is effectively free of false positives. + +A per-run summary reports how many credentials of each type were redacted. + # Command Line ``` -usage: pyonenote [-h] -f FILE [-o OUTPUT_DIR] [-e EXTENSION] +usage: pyonenote [-h] -f FILE [-o OUTPUT_DIR] [-e EXTENSION] [-j] [-m] [-r] ``` Note: pyOneNote is under active development diff --git a/pyOneNote/FileNode.py b/pyOneNote/FileNode.py index a5f04d0..c9368a3 100644 --- a/pyOneNote/FileNode.py +++ b/pyOneNote/FileNode.py @@ -143,10 +143,27 @@ def __init__(self, file, document): file.seek(self.data.ref.stp) self.propertySet = ObjectSpaceObjectPropSet(file, document) file.seek(current_offset) + elif self.file_node_header.file_node_type == "ObjectDeclaration2LargeRefCountFND": + self.data = ObjectDeclaration2LargeRefCountFND(file, self.document, self.file_node_header) + current_offset = file.tell() + if self.data.body.jcid.IsPropertySet: + file.seek(self.data.ref.stp) + self.propertySet = ObjectSpaceObjectPropSet(file, document) + file.seek(current_offset) elif self.file_node_header.file_node_type == "ReadOnlyObjectDeclaration2LargeRefCountFND": self.data = ReadOnlyObjectDeclaration2LargeRefCountFND(file, self.document, self.file_node_header) + current_offset = file.tell() + if self.data.base.body.jcid.IsPropertySet: + file.seek(self.data.base.ref.stp) + self.propertySet = ObjectSpaceObjectPropSet(file, document) + file.seek(current_offset) elif self.file_node_header.file_node_type == "ReadOnlyObjectDeclaration2RefCountFND": self.data = ReadOnlyObjectDeclaration2RefCountFND(file, self.document, self.file_node_header) + current_offset = file.tell() + if self.data.base.body.jcid.IsPropertySet: + file.seek(self.data.base.ref.stp) + self.propertySet = ObjectSpaceObjectPropSet(file, document) + file.seek(current_offset) elif self.file_node_header.file_node_type == "FileDataStoreListReferenceFND": self.data = FileDataStoreListReferenceFND(file, self.file_node_header) elif self.file_node_header.file_node_type == "FileDataStoreObjectReferenceFND": @@ -174,10 +191,10 @@ def __init__(self, file, document): # no data part self.data = None else: - p = 1 + self.data = None current_offset = file.tell() - if self.file_node_header.baseType == 2: + if self.file_node_header.baseType == 2 and self.data is not None: self.children.append(FileNodeList(file, self.document, self.data.ref)) file.seek(current_offset) @@ -638,9 +655,19 @@ def __init__(self, file, OIDs, OSIDs, ContextIDs, document): count, = struct.unpack(' 0 a prid PropertyID + # (4 bytes, always type 0x11/PropertySet) followed by that + # many nested PropertySet structures. + count, = struct.unpack(' 0: + PropertyID(file) # prid header, type is always 0x11 here + for _ in range(count): + array_data.append(PropertySet(file, OIDs, OSIDs, ContextIDs, document)) + self.rgData.append(array_data) elif type == 0x11: - self.rgData.append(PropertySet(file)) + self.rgData.append(PropertySet(file, OIDs, OSIDs, ContextIDs, document)) else: raise ValueError('rgPrids[i].type is not valid') @@ -664,6 +691,12 @@ def get_properties(self): if isinstance(self.rgData[i], PrtFourBytesOfLengthFollowedByData): if 'guid' in propertyName.lower(): propertyVal = uuid.UUID(bytes_le=self.rgData[i].Data).hex + elif propertyName == 'TextExtendedAscii': + # OneNote for Mac stores plain-text runs as single-byte + # (latin-1) text under this property, instead of the + # UTF-16 RichEditTextUnicode Windows OneNote uses. + # Decoding it as utf-16 either garbles it or throws. + propertyVal = self.rgData[i].Data.decode('latin-1') else: try: propertyVal = self.rgData[i].Data.decode('utf-16') diff --git a/pyOneNote/Main.py b/pyOneNote/Main.py index 6189d2e..cd5d703 100644 --- a/pyOneNote/Main.py +++ b/pyOneNote/Main.py @@ -7,6 +7,7 @@ import logging import argparse import json +from collections import Counter log = logging.getLogger() @@ -19,7 +20,7 @@ def check_valid(file): return False -def process_onenote_file(file, output_dir, extension, json_output): +def process_onenote_file(file, output_dir, extension, json_output, redact=False): if not check_valid(file): log.error("please provide valid One file") exit() @@ -27,6 +28,27 @@ def process_onenote_file(file, output_dir, extension, json_output): file.seek(0) document = OneDocment(file) data = document.get_json() + + if redact: + from pyOneNote.redact import redact_tree, redact_bytes + counts = Counter() + for section in ("headers", "properties"): + data[section], section_counts = redact_tree(data[section]) + counts.update(section_counts) + # Embedded attachments that are really text (a pasted .pem/.env/config) + # are scrubbed too; binary attachments have no text to redact. Mutating + # the document's file cache also covers the extraction loop below. + for key, embedded in document.get_files().items(): + content, file_counts = redact_bytes(embedded["content"]) + embedded["content"] = content + data["files"][key]["content"] = content.hex() + counts.update(file_counts) + n = sum(counts.values()) + # stderr, so it can never corrupt the JSON on stdout + print("Redacted {} credential(s){}".format( + n, ": " + ", ".join("{} {}".format(v, k) for k, v in counts.most_common()) if n else ""), + file=sys.stderr) + if not json_output: print('Headers\n####################################################################') indent = '\t' @@ -75,6 +97,57 @@ def process_onenote_file(file, output_dir, extension, json_output): return json.dumps(data) +def export_markdown(file, output_path, section_title, redact=False): + if not check_valid(file): + log.error("skipping invalid .one file: %s", section_title) + return None, None + file.seek(0) + document = OneDocment(file) + result = document.get_markdown(section_title, redact=redact) + markdown, redactions = result if redact else (result, None) + with open(output_path, "w", encoding="utf-8") as out: + out.write(markdown) + page_count = len(document.get_pages()) + log.info("wrote %d pages -> %s", page_count, output_path) + return page_count, redactions + + +def bulk_export_markdown(input_path, output_dir, redact=False): + """Export one .one file, or every .one file in a directory, to markdown.""" + os.makedirs(output_dir, exist_ok=True) + if os.path.isdir(input_path): + one_files = sorted( + os.path.join(input_path, name) + for name in os.listdir(input_path) + if name.lower().endswith(".one") + ) + else: + one_files = [input_path] + + if not one_files: + sys.exit("No .one files found in: %s" % input_path) + + total_pages = 0 + total_redactions = Counter() + for one_file in one_files: + section_title = os.path.splitext(os.path.basename(one_file))[0] + output_path = os.path.join(output_dir, section_title + ".md") + with open(one_file, "rb") as fh: + pages, redactions = export_markdown(fh, output_path, section_title, redact=redact) + if pages: + total_pages += pages + note = "" + if redactions: + total_redactions.update(redactions) + note = " [{} secrets redacted]".format(sum(redactions.values())) + print("{}: {} pages -> {}{}".format(section_title, pages, output_path, note)) + print("\nDone. {} file(s), {} unique pages total.".format(len(one_files), total_pages)) + if redact: + n = sum(total_redactions.values()) + print("Redacted {} credential(s) total{}".format( + n, ": " + ", ".join("{} {}".format(v, k) for k, v in total_redactions.most_common()) if n else "")) + + def get_hex_format(hex_str, col, indent): res = '' chars = (col*2) @@ -90,14 +163,21 @@ def main(): p.add_argument("-o", "--output-dir", action="store", default="./", help="Path where store extracted files") p.add_argument("-e", "--extension", action="store", default="", help="Append this extension to extracted file(s)") p.add_argument("-j", "--json", action="store_true", default=False, help="Generate JSON output only, no dumps or prints") + p.add_argument("-m", "--markdown", action="store_true", default=False, help="Bulk-export page text to clean markdown. -f may be a single .one file or a directory of them; one .md is written per section into --output-dir") + p.add_argument("-r", "--redact-secrets", action="store_true", default=False, help="Replace high-confidence credential shapes (AWS/GitHub/Slack/JWT/private keys, etc.) with [redacted:*] markers. Applies to every output mode: the default dump, JSON (-j), markdown (-m), and text attachments written to --output-dir") args = p.parse_args() if not os.path.exists(args.file): sys.exit("File: %s doesn't exist", args.file) + if args.markdown: + logging.basicConfig(level=logging.INFO, format="%(message)s") + bulk_export_markdown(args.file, args.output_dir, redact=args.redact_secrets) + return + with open(args.file, "rb") as file: - process_onenote_file(file, args.output_dir, args.extension, args.json) + process_onenote_file(file, args.output_dir, args.extension, args.json, redact=args.redact_secrets) if __name__ == "__main__": diff --git a/pyOneNote/OneDocument.py b/pyOneNote/OneDocument.py index 906a606..38ef546 100644 --- a/pyOneNote/OneDocument.py +++ b/pyOneNote/OneDocument.py @@ -24,7 +24,18 @@ def get_properties(self): if self._properties: return self._properties nodes = [] - filters = ['ObjectDeclaration2RefCountFND'] + # ObjectDeclaration2LargeRefCountFND is used for objects whose + # reference count doesn't fit in one byte (e.g. more heavily-shared + # or larger page content), and the ReadOnly* variants wrap either + # declaration with an extra md5Hash. All four carry the same + # propertySet-bearing body and were previously excluded here, which + # silently dropped any text stored under them. + filters = [ + 'ObjectDeclaration2RefCountFND', + 'ObjectDeclaration2LargeRefCountFND', + 'ReadOnlyObjectDeclaration2RefCountFND', + 'ReadOnlyObjectDeclaration2LargeRefCountFND', + ] self._properties = [] @@ -32,7 +43,9 @@ def get_properties(self): for node in nodes: if hasattr(node, 'propertySet'): node.propertySet.body.indent= '\t\t' - self._properties.append({'type': str(node.data.body.jcid), 'identity':str(node.data.body.oid), 'val':node.propertySet.body.get_properties()}) + # ReadOnly* variants wrap the declaration in `.base`. + decl_body = node.data.base.body if hasattr(node.data, 'base') else node.data.body + self._properties.append({'type': str(decl_body.jcid), 'identity':str(decl_body.oid), 'val':node.propertySet.body.get_properties()}) return self._properties @@ -66,6 +79,106 @@ def get_files(self): def get_global_identification_table(self): return self._global_identification_table + # jcids that carry displayable text runs, in the order OneNote lays them out. + _TEXT_JCIDS = ('jcidRichTextOENode', 'jcidTitleNode', 'jcidNumberListNode') + + @staticmethod + def _run_text(val): + # A single text run stores its content under one of these keys depending + # on platform: RichEditTextUnicode (Windows, UTF-16) or TextExtendedAscii + # (OneNote for Mac, latin-1). Strip embedded null bytes either way. + text = val.get('RichEditTextUnicode') or val.get('TextExtendedAscii') or '' + return text.replace('\x00', '').replace('\r\n', '\n').replace('\r', '\n').rstrip() + + def _toc_titles(self): + # OneNote maintains a CachedTitleString for every page (used to render + # the section's page tabs). Collecting them gives the authoritative set + # of real page names, which we use to distinguish a page's title run + # from its body runs. + titles = set() + for prop in self.get_properties(): + cached = prop['val'].get('CachedTitleString') + if cached: + cached = cached.replace('\x00', '').strip() + if cached: + titles.add(cached) + return titles + + def get_pages(self): + """Reconstruct de-duplicated pages from the property stream. + + OneNote keeps every historical revision of a page in the same file, so + the raw stream repeats each page 2-20x. We segment on jcidPageNode, pick + the page's title as the first run matching a known CachedTitleString, + then key pages by that title. Revisions of the same page collapse into + one entry (identical runs de-duplicated), while genuinely distinct pages + that happen to share text are preserved because de-dup is scoped per + title. Pages are returned in first-seen order. + """ + toc = self._toc_titles() + + # segment the flat stream into per-revision run lists + raw_pages = [] + current = None + for prop in self.get_properties(): + ptype = prop['type'] + if ptype == 'jcidPageNode': + if current is not None: + raw_pages.append(current) + current = [] + continue + if current is None: + continue + if ptype in self._TEXT_JCIDS: + text = self._run_text(prop['val']) + if text: + current.append(text) + if current is not None: + raw_pages.append(current) + + pages = [] + by_title = {} + for runs in raw_pages: + if not runs: + continue + title = next((r for r in runs if r in toc), runs[0]) + if title not in by_title: + entry = {'title': title, 'lines': [], '_seen': set()} + by_title[title] = entry + pages.append(entry) + entry = by_title[title] + for run in runs: + if run == title or run in entry['_seen']: + continue + entry['_seen'].add(run) + entry['lines'].append(run) + + for entry in pages: + del entry['_seen'] + return pages + + def get_markdown(self, section_title=None, redact=False): + """Render every unique page in this document as a single markdown string. + + When redact is True, high-confidence credential shapes are replaced with + visible [redacted:*] markers (see pyOneNote.redact). Returns the markdown + string, or (string, Counter) of redaction counts when redact is True. + """ + out = [] + if section_title: + out.append('# {}\n'.format(section_title)) + for page in self.get_pages(): + title = page['title'].strip() or 'Untitled' + out.append('## {}\n'.format(title)) + body = '\n'.join(page['lines']).strip() + if body: + out.append(body + '\n') + markdown = '\n'.join(out).rstrip() + '\n' + if redact: + from pyOneNote.redact import redact_secrets + return redact_secrets(markdown) + return markdown + def get_json(self): files_in_hex = {} for key, file in self.get_files().items(): diff --git a/pyOneNote/redact.py b/pyOneNote/redact.py new file mode 100644 index 0000000..1f39a75 --- /dev/null +++ b/pyOneNote/redact.py @@ -0,0 +1,145 @@ +"""Credential redaction for markdown export. + +Only PREFIX/STRUCTURE-KEYED credential shapes are matched -- vendor sentinels +like AKIA / ghp_ / xoxb- / sk-ant- whose anchor essentially cannot occur in +prose by accident -- plus PEM PRIVATE KEY blocks. There is deliberately NO +generic entropy, keyword-proximity, or base64/blob detection, so ordinary note +text, code, identifiers, public certificates, and embedded data are left intact +and the pass is effectively free of false positives. + +Redaction replaces the credential with a visible marker (e.g. "[redacted:jwt]") +rather than deleting it: "a credential was here" is useful context for whoever +reads the exported markdown; the credential value itself is not. +""" + +import re +from collections import Counter + +# Vendor-sentinel-keyed credential shapes. (compiled pattern, replacement) +_SECRET_PATTERNS = ( + # AWS access key IDs (AKIA=long-lived, ASIA=STS, ABIA/ACCA variants). + (re.compile(r"\b(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b"), "[redacted:aws-key-id]"), + # GitHub tokens: classic (ghp_/gho_/ghu_/ghs_/ghr_) and fine-grained. + (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,255}\b"), "[redacted:github-token]"), + (re.compile(r"\bgithub_pat_[A-Za-z0-9_]{22,255}\b"), "[redacted:github-token]"), + # GitLab personal access tokens. + (re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}\b"), "[redacted:gitlab-token]"), + # OpenAI / Anthropic style keys. Two branches so each is captured as one + # full span (a single prefix-loop pattern locks its tail onto an interior + # 32-char run and leaves the rest of a base64url key un-redacted): + # - OpenAI (sk- / sk-proj-): dashless base62 tail, so short kebab-case + # slugs like "sk-loading-spinner-dark" can't match. + # - Anthropic (sk-ant--): covers console API keys + # (sk-ant-api03-) and Claude.ai OAuth tokens (sk-ant-oat01-). The type + # segment is [a-z]+ followed by REQUIRED version digits (\d{2,}), which + # future-proofs new versions (api04, oat02, ...) and, crucially, is what + # separates a real key from an Ant-Design-style kebab class such as + # "sk-ant-design-system-..." (no fused digits -> no match). Body uses + # the base64url charset with a modest floor since the sk-ant-NN- + # anchor is already highly specific. + ( + re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9]{32,}\b|\bsk-ant-[a-z]{2,12}\d{2,}-[A-Za-z0-9_-]{20,}\b"), + "[redacted:api-key]", + ), + # Slack tokens + incoming-webhook paths. + (re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"), "[redacted:slack-token]"), + ( + re.compile(r"hooks\.slack\.com/services/T[A-Za-z0-9]+/B[A-Za-z0-9]+/[A-Za-z0-9]+"), + "hooks.slack.com/services/[redacted:slack-webhook]", + ), + # Stripe secret/restricted keys (live AND test -- test keys still + # grant API access to the test-mode account). + (re.compile(r"\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}\b"), "[redacted:stripe-key]"), + # Google API keys. + (re.compile(r"\bAIza[A-Za-z0-9_-]{35}\b"), "[redacted:google-api-key]"), + # npm automation tokens. + (re.compile(r"\bnpm_[A-Za-z0-9]{36}\b"), "[redacted:npm-token]"), + # PyPI upload tokens (macaroon with fixed b64 "pypi.org" prefix). + (re.compile(r"\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{20,}\b"), "[redacted:pypi-token]"), + # SendGrid API keys (SG.<22>.<43>). + (re.compile(r"\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b"), "[redacted:sendgrid-key]"), + # Twilio API key SIDs (SK + 32 hex). + (re.compile(r"\bSK[a-f0-9]{32}\b"), "[redacted:twilio-key]"), + # JWTs -- three dot-joined base64url segments, first two decoding + # from the literal '{"' ("eyJ"). + ( + re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"), + "[redacted:jwt]", + ), + # Authorization: Bearer header captures. {30,} floor keeps + # doc placeholders (YOUR_TOKEN, $TOKEN, ) out of scope. + (re.compile(r"(?i)\bBearer\s+[A-Za-z0-9_\-.=~+/]{30,}"), "Bearer [redacted:bearer-token]"), +) + +# PEM PRIVATE KEY blocks (RSA/EC/DSA/OPENSSH/ENCRYPTED/PGP). Public +# certificates, public keys, and CSRs are meant to be shared and carry no +# secret, so they are deliberately left intact. No generic base64/blob rule +# is applied -- that would redact non-secret embedded images and encoded data. +_ARMOR_PATTERNS = ( + ( + re.compile( + r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----.*?" + r"-----END [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----", + re.DOTALL, + ), + "[redacted:private-key]", + ), +) + + +def redact_secrets(text): + """Redact credentials, returning (redacted_text, Counter of marker -> count). + + Applies the vendor-sentinel screen (API keys/tokens with recognizable + prefixes) and PEM PRIVATE KEY blocks. Only high-confidence, structurally- + anchored shapes are touched; public certs and embedded data are left intact. + """ + counts = Counter() + for pat, repl in _ARMOR_PATTERNS + _SECRET_PATTERNS: + text, n = pat.subn(repl, text) + if n: + counts[repl.strip()] += n + return text, counts + + +def redact_tree(obj): + """Recursively redact every string *value* in a nested dict/list structure. + + Returns (new_obj, Counter). Dictionary keys are property names, not secrets, + so they are left alone. Used to scrub the parsed document (headers, + property sets) that backs the default dump and the JSON output. + """ + counts = Counter() + + def walk(node): + if isinstance(node, str): + red, c = redact_secrets(node) + counts.update(c) + return red + if isinstance(node, dict): + return {k: walk(v) for k, v in node.items()} + if isinstance(node, list): + return [walk(v) for v in node] + if isinstance(node, tuple): + return tuple(walk(v) for v in node) + return node + + return walk(obj), counts + + +def redact_bytes(data): + """Redact an embedded file's contents when it is UTF-8 text. + + Covers attachments that are really text (a pasted .pem, .env, config, or + script). Binary attachments (images, Office docs) fail the strict decode and + are returned untouched -- there is no text there to scrub, and rewriting + their bytes could only corrupt them. Returns (bytes, Counter). + """ + try: + text = data.decode("utf-8") + except (UnicodeDecodeError, AttributeError): + return data, Counter() + red, counts = redact_secrets(text) + if not counts: + return data, counts + return red.encode("utf-8"), counts diff --git a/tests/test_redact.py b/tests/test_redact.py new file mode 100644 index 0000000..6df3be6 --- /dev/null +++ b/tests/test_redact.py @@ -0,0 +1,189 @@ +"""Regression tests for pyOneNote.redact secret patterns. + +Run with: python3 -m unittest discover -s tests +(stdlib unittest only -- no third-party test runner required.) + +The sk- (OpenAI/Anthropic) rule is the sharp edge: an earlier single +prefix-loop pattern would lock its {32,} tail onto an interior run of a +base64url Anthropic key and redact only that chunk, silently leaving the +rest of the live key in the output. These tests assert FULL-SPAN redaction +(the whole key token is gone), not merely that a match occurred. +""" +import json +import os +import random +import string +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from pyOneNote.redact import redact_secrets, redact_tree, redact_bytes # noqa: E402 + +MARKER = "[redacted:api-key]" +B64URL = string.ascii_letters + string.digits + "-_" +BASE62 = string.ascii_letters + string.digits + + +def _redacted_line(secret): + """Redact a line that is exactly the secret; return the stripped result.""" + out, counts = redact_secrets(secret) + return out.strip(), counts + + +class SkKeyRedaction(unittest.TestCase): + def test_openai_classic_full_span(self): + key = "sk-" + "".join(random.Random(1).choice(BASE62) for _ in range(48)) + out, counts = _redacted_line(key) + self.assertEqual(out, MARKER) # whole token replaced + self.assertNotIn("sk-", out) # no residual key material + self.assertEqual(counts[MARKER], 1) + + def test_openai_project_full_span(self): + key = "sk-proj-" + "".join(random.Random(2).choice(BASE62) for _ in range(48)) + out, _ = _redacted_line(key) + self.assertEqual(out, MARKER) + + def test_anthropic_base64url_full_span_bulk(self): + # The regression: across many realistic keys, EVERY key must be fully + # redacted with no leaked tail. The old pattern managed ~21% full. + # Cover both documented token types (console api03, OAuth oat01) and + # future version digits (api04, oat02, ...). Bodies end alphanumeric, + # as real keys do (…AA), so no trailing-dash trim artifact. + rng = random.Random(7) + for _ in range(500): + kind = rng.choice(("api", "oat", "admin")) + ver = "0%d" % rng.randint(1, 9) + length = rng.randint(20, 95) + body = "".join(rng.choice(B64URL) for _ in range(length)) + body = body.rstrip("-") + rng.choice(BASE62) # real keys end alnum + key = "sk-ant-%s%s-%s" % (kind, ver, body) + out, counts = _redacted_line(key) + self.assertEqual(out, MARKER, "partial/missed redaction leaks key material") + self.assertEqual(counts[MARKER], 1) + + def test_anthropic_oauth_and_future_versions(self): + for key in ( + "sk-ant-oat01-" + "".join(random.Random(3).choice(B64URL) for _ in range(60)) + "Az", + "sk-ant-api04-" + "".join(random.Random(4).choice(B64URL) for _ in range(80)) + "Az", + "sk-ant-api100-" + "".join(random.Random(5).choice(B64URL) for _ in range(80)) + "Az", + ): + out, _ = _redacted_line(key) + self.assertEqual(out, MARKER) + + def test_anthropic_embedded_in_prose(self): + rng = random.Random(11) + body = "".join(rng.choice(B64URL) for _ in range(93)) + "AA" + key = "sk-ant-api03-" + body + line = "ANTHROPIC_API_KEY=" + key + " # rotate me" + out, _ = redact_secrets(line) + self.assertIn(MARKER, out) + self.assertNotIn(key, out) + # no non-trivial fragment of the key survives + self.assertNotIn(body[:40], out) + + +class SkFalsePositives(unittest.TestCase): + def test_kebab_css_class_not_redacted(self): + s = 'class="sk-loading-spinner-overlay-fullscreen-darkmode"' + out, counts = redact_secrets(s) + self.assertEqual(out, s) + self.assertNotIn(MARKER, out) + + def test_short_sk_identifier_not_redacted(self): + s = "sk-loader" + out, _ = redact_secrets(s) + self.assertEqual(out, s) + + def test_ant_design_kebab_class_not_redacted(self): + # "ant" collides with Ant Design's class prefix; the required fused + # version digits (api03) are what keep these kebab classes out. + for s in ( + "class='sk-ant-design-system-really-long-classname-wrapper'", + "sk-ant-col-24-responsive-grid-column-layout-helper-xl", + "sk-ant-btn", + ): + out, counts = redact_secrets(s) + self.assertEqual(out, s) + self.assertNotIn(MARKER, out) + + +class PemArmor(unittest.TestCase): + def _block(self, label): + body = "\n".join("MIID" + "A" * 60 for _ in range(20)) + return "-----BEGIN %s-----\n%s\n-----END %s-----" % (label, body, label) + + def test_private_keys_redacted(self): + for label in ( + "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY", + "OPENSSH PRIVATE KEY", "ENCRYPTED PRIVATE KEY", "PGP PRIVATE KEY BLOCK", + ): + out, counts = redact_secrets(self._block(label)) + self.assertEqual(counts["[redacted:private-key]"], 1, label) + self.assertNotIn("MIID" + "A" * 60, out) + + def test_public_material_left_intact(self): + # Public certs / keys / CSRs are shareable and must NOT be redacted, + # and with no generic base64 rule their bodies survive too. + for label in ("CERTIFICATE", "PUBLIC KEY", "CERTIFICATE REQUEST"): + block = self._block(label) + out, counts = redact_secrets(block) + self.assertEqual(out, block, label) + self.assertEqual(len(counts), 0, label) + + def test_embedded_base64_not_redacted(self): + # A long base64 blob (e.g. an embedded image) is not a credential. + blob = "data = '" + "QUJD" * 200 + "'" + out, _ = redact_secrets(blob) + self.assertEqual(out, blob) + + +class TreeAndBytesRedaction(unittest.TestCase): + """Redaction must cover every output mode, not just markdown: the default + dump and JSON both render the parsed tree, and attachments are written to + disk. Partial coverage would leak in the modes that aren't scrubbed.""" + + def test_redact_tree_scrubs_nested_string_values(self): + key = "sk-" + "A1b2C3d4" * 6 # 48-char base62 tail + tree = { + "headers": {"guidFile": "1234-abcd", "count": 7}, + "properties": [ + {"type": "jcidRichTextOENode", + "val": {"RichEditTextUnicode": "aws key AKIAABCDEFGHIJKLMNOP here"}}, + {"type": "jcidTitleNode", "val": {"RichEditTextUnicode": "token " + key}}, + ], + } + out, counts = redact_tree(tree) + blob = json.dumps(out) + self.assertNotIn("AKIAABCDEFGHIJKLMNOP", blob) + self.assertNotIn(key, blob) + self.assertIn("[redacted:aws-key-id]", blob) + self.assertIn(MARKER, blob) + self.assertEqual(sum(counts.values()), 2) + # non-string values and dict keys survive untouched + self.assertEqual(out["headers"]["count"], 7) + self.assertEqual(out["properties"][0]["type"], "jcidRichTextOENode") + + def test_redact_tree_leaves_clean_text_alone(self): + tree = {"properties": [{"val": {"RichEditTextUnicode": "just some notes"}}]} + out, counts = redact_tree(tree) + self.assertEqual(out, tree) + self.assertEqual(sum(counts.values()), 0) + + def test_redact_bytes_scrubs_text_attachment(self): + pem = (b"-----BEGIN RSA PRIVATE KEY-----\n" + + b"MIIB" + b"B" * 60 + b"\n-----END RSA PRIVATE KEY-----\n") + out, counts = redact_bytes(pem) + self.assertIn(b"[redacted:private-key]", out) + self.assertNotIn(b"MIIB" + b"B" * 60, out) + self.assertEqual(counts["[redacted:private-key]"], 1) + + def test_redact_bytes_leaves_binary_untouched(self): + # A PNG header is not valid UTF-8; it must pass through byte-identical. + png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\xff\xfe\xfd" + out, counts = redact_bytes(png) + self.assertEqual(out, png) + self.assertEqual(sum(counts.values()), 0) + + +if __name__ == "__main__": + unittest.main(verbosity=2)