Add credential redaction and markdown export

README.md

@@ -24,9 +24,53 @@ To dump all embedded file in example.one into output_dir and add .bin to the end
 pyonenote -f example.one -o output_dir -e bin
 ```
 
+## Bulk markdown export
+
+To extract the page text of a section to clean markdown (embedded null bytes
+stripped, historical page revisions de-duplicated, one `##` heading per page):
+```
+pyonenote -m -f example.one -o output_dir
+```
+
+`-f` may also point at a directory, in which case every `.one` file in it is
+exported to its own `<section>.md` inside `output_dir` — useful for dumping a
+whole notebook's worth of sections in one pass for scripted diffing or pattern
+extraction:
+```
+pyonenote -m -f "/path/to/OneNote Backup/MyNotebook" -o markdown_out
+```
+
+### Redacting secrets
+
+Add `-r` / `--redact-secrets` to replace credentials with visible
+`[redacted:*]` markers — useful when output will be shared for review:
+```
+pyonenote -r    -f "MyNotebook/Section.one"                 # default dump
+pyonenote -r -j -f "MyNotebook/Section.one"                 # JSON
+pyonenote -r -m -f "MyNotebook" -o markdown_out             # markdown
+```
+`-r` applies to **every output mode** — the default dump, JSON (`-j`), markdown
+(`-m`), and any text attachment written to `--output-dir`. Binary attachments
+(images, Office docs) contain no text to scrub and are written unchanged.
+
+Redaction matches only high-confidence, structurally-anchored shapes (see
+`pyOneNote/redact.py`), so ordinary note text is left untouched:
+- **Vendor sentinels**: prefix/structure-keyed API keys and tokens —
+  AWS/GitHub/GitLab/OpenAI/Anthropic/Slack/Stripe/Google/npm/PyPI/SendGrid/
+  Twilio keys, JWTs, and `Authorization: Bearer` tokens.
+- **PEM private keys**: `PRIVATE KEY` blocks (RSA/EC/DSA/OPENSSH/ENCRYPTED/PGP).
+  Public certificates, public keys, and CSRs are meant to be shared and are left
+  intact.
+
+There is deliberately no generic entropy, keyword-proximity, or base64/blob
+detection, so ordinary text and embedded data (images, encoded content) are left
+untouched and the pass is effectively free of false positives.
+
+A per-run summary reports how many credentials of each type were redacted.
+
 # Command Line
 ```
-usage: pyonenote [-h] -f FILE [-o OUTPUT_DIR] [-e EXTENSION]
+usage: pyonenote [-h] -f FILE [-o OUTPUT_DIR] [-e EXTENSION] [-j] [-m] [-r]
 ```
 
 Note: pyOneNote is under active development

pyOneNote/FileNode.py

@@ -143,10 +143,27 @@ def __init__(self, file, document):
                 file.seek(self.data.ref.stp)
                 self.propertySet = ObjectSpaceObjectPropSet(file, document)
             file.seek(current_offset)
+        elif self.file_node_header.file_node_type == "ObjectDeclaration2LargeRefCountFND":
+            self.data = ObjectDeclaration2LargeRefCountFND(file, self.document, self.file_node_header)
+            current_offset = file.tell()
+            if self.data.body.jcid.IsPropertySet:
+                file.seek(self.data.ref.stp)
+                self.propertySet = ObjectSpaceObjectPropSet(file, document)
+            file.seek(current_offset)
         elif self.file_node_header.file_node_type == "ReadOnlyObjectDeclaration2LargeRefCountFND":
             self.data = ReadOnlyObjectDeclaration2LargeRefCountFND(file, self.document, self.file_node_header)
+            current_offset = file.tell()
+            if self.data.base.body.jcid.IsPropertySet:
+                file.seek(self.data.base.ref.stp)
+                self.propertySet = ObjectSpaceObjectPropSet(file, document)
+            file.seek(current_offset)
         elif self.file_node_header.file_node_type == "ReadOnlyObjectDeclaration2RefCountFND":
             self.data = ReadOnlyObjectDeclaration2RefCountFND(file, self.document, self.file_node_header)
+            current_offset = file.tell()
+            if self.data.base.body.jcid.IsPropertySet:
+                file.seek(self.data.base.ref.stp)
+                self.propertySet = ObjectSpaceObjectPropSet(file, document)
+            file.seek(current_offset)
         elif self.file_node_header.file_node_type == "FileDataStoreListReferenceFND":
             self.data = FileDataStoreListReferenceFND(file, self.file_node_header)
         elif self.file_node_header.file_node_type == "FileDataStoreObjectReferenceFND":
@@ -174,10 +191,10 @@ def __init__(self, file, document):
             # no data part
             self.data = None
         else:
-            p = 1
+            self.data = None
 
         current_offset = file.tell()
-        if self.file_node_header.baseType == 2:
+        if self.file_node_header.baseType == 2 and self.data is not None:
             self.children.append(FileNodeList(file, self.document, self.data.ref))
         file.seek(current_offset)
 
@@ -638,9 +655,19 @@ def __init__(self, file, OIDs, OSIDs, ContextIDs, document):
                     count, = struct.unpack('<I', file.read(4))
                 self.rgData.append(self.get_compact_ids(ContextIDs, count))
             elif type == 0x10:
-                raise NotImplementedError('ArrayOfPropertyValues is not implement')
+                # prtArrayOfPropertyValues ([MS-ONESTORE] 2.6.9): cProperties
+                # (4 bytes), then if cProperties > 0 a prid PropertyID
+                # (4 bytes, always type 0x11/PropertySet) followed by that
+                # many nested PropertySet structures.
+                count, = struct.unpack('<I', file.read(4))
+                array_data = []
+                if count > 0:
+                    PropertyID(file)  # prid header, type is always 0x11 here
+                    for _ in range(count):
+                        array_data.append(PropertySet(file, OIDs, OSIDs, ContextIDs, document))
+                self.rgData.append(array_data)
             elif type == 0x11:
-                self.rgData.append(PropertySet(file))
+                self.rgData.append(PropertySet(file, OIDs, OSIDs, ContextIDs, document))
             else:
                 raise ValueError('rgPrids[i].type is not valid')
 
@@ -664,6 +691,12 @@ def get_properties(self):
                 if isinstance(self.rgData[i], PrtFourBytesOfLengthFollowedByData):
                     if 'guid' in propertyName.lower():
                         propertyVal = uuid.UUID(bytes_le=self.rgData[i].Data).hex
+                    elif propertyName == 'TextExtendedAscii':
+                        # OneNote for Mac stores plain-text runs as single-byte
+                        # (latin-1) text under this property, instead of the
+                        # UTF-16 RichEditTextUnicode Windows OneNote uses.
+                        # Decoding it as utf-16 either garbles it or throws.
+                        propertyVal = self.rgData[i].Data.decode('latin-1')
                     else:
                         try:
                             propertyVal = self.rgData[i].Data.decode('utf-16')

pyOneNote/Main.py

@@ -7,6 +7,7 @@
 import logging
 import argparse
 import json
+from collections import Counter
 
 log = logging.getLogger()
 
@@ -19,14 +20,35 @@ def check_valid(file):
     return False
 
 
-def process_onenote_file(file, output_dir, extension, json_output):
+def process_onenote_file(file, output_dir, extension, json_output, redact=False):
     if not check_valid(file):
         log.error("please provide valid One file")
         exit()
 
     file.seek(0)
     document = OneDocment(file)
     data = document.get_json()
+
+    if redact:
+        from pyOneNote.redact import redact_tree, redact_bytes
+        counts = Counter()
+        for section in ("headers", "properties"):
+            data[section], section_counts = redact_tree(data[section])
+            counts.update(section_counts)
+        # Embedded attachments that are really text (a pasted .pem/.env/config)
+        # are scrubbed too; binary attachments have no text to redact. Mutating
+        # the document's file cache also covers the extraction loop below.
+        for key, embedded in document.get_files().items():
+            content, file_counts = redact_bytes(embedded["content"])
+            embedded["content"] = content
+            data["files"][key]["content"] = content.hex()
+            counts.update(file_counts)
+        n = sum(counts.values())
+        # stderr, so it can never corrupt the JSON on stdout
+        print("Redacted {} credential(s){}".format(
+            n, ": " + ", ".join("{} {}".format(v, k) for k, v in counts.most_common()) if n else ""),
+            file=sys.stderr)
+
     if not json_output:
         print('Headers\n####################################################################')
         indent = '\t'
@@ -75,6 +97,57 @@ def process_onenote_file(file, output_dir, extension, json_output):
     return json.dumps(data)
 
 
+def export_markdown(file, output_path, section_title, redact=False):
+    if not check_valid(file):
+        log.error("skipping invalid .one file: %s", section_title)
+        return None, None
+    file.seek(0)
+    document = OneDocment(file)
+    result = document.get_markdown(section_title, redact=redact)
+    markdown, redactions = result if redact else (result, None)
+    with open(output_path, "w", encoding="utf-8") as out:
+        out.write(markdown)
+    page_count = len(document.get_pages())
+    log.info("wrote %d pages -> %s", page_count, output_path)
+    return page_count, redactions
+
+
+def bulk_export_markdown(input_path, output_dir, redact=False):
+    """Export one .one file, or every .one file in a directory, to markdown."""
+    os.makedirs(output_dir, exist_ok=True)
+    if os.path.isdir(input_path):
+        one_files = sorted(
+            os.path.join(input_path, name)
+            for name in os.listdir(input_path)
+            if name.lower().endswith(".one")
+        )
+    else:
+        one_files = [input_path]
+
+    if not one_files:
+        sys.exit("No .one files found in: %s" % input_path)
+
+    total_pages = 0
+    total_redactions = Counter()
+    for one_file in one_files:
+        section_title = os.path.splitext(os.path.basename(one_file))[0]
+        output_path = os.path.join(output_dir, section_title + ".md")
+        with open(one_file, "rb") as fh:
+            pages, redactions = export_markdown(fh, output_path, section_title, redact=redact)
+        if pages:
+            total_pages += pages
+            note = ""
+            if redactions:
+                total_redactions.update(redactions)
+                note = "  [{} secrets redacted]".format(sum(redactions.values()))
+            print("{}: {} pages -> {}{}".format(section_title, pages, output_path, note))
+    print("\nDone. {} file(s), {} unique pages total.".format(len(one_files), total_pages))
+    if redact:
+        n = sum(total_redactions.values())
+        print("Redacted {} credential(s) total{}".format(
+            n, ": " + ", ".join("{} {}".format(v, k) for k, v in total_redactions.most_common()) if n else ""))
+
+
 def get_hex_format(hex_str, col, indent):
     res = ''
     chars = (col*2)
@@ -90,14 +163,21 @@ def main():
     p.add_argument("-o", "--output-dir", action="store", default="./", help="Path where store extracted files")
     p.add_argument("-e", "--extension", action="store", default="", help="Append this extension to extracted file(s)")
     p.add_argument("-j", "--json", action="store_true", default=False, help="Generate JSON output only, no dumps or prints")
+    p.add_argument("-m", "--markdown", action="store_true", default=False, help="Bulk-export page text to clean markdown. -f may be a single .one file or a directory of them; one .md is written per section into --output-dir")
+    p.add_argument("-r", "--redact-secrets", action="store_true", default=False, help="Replace high-confidence credential shapes (AWS/GitHub/Slack/JWT/private keys, etc.) with [redacted:*] markers. Applies to every output mode: the default dump, JSON (-j), markdown (-m), and text attachments written to --output-dir")
 
     args = p.parse_args()
 
     if not os.path.exists(args.file):
         sys.exit("File: %s doesn't exist", args.file)
 
+    if args.markdown:
+        logging.basicConfig(level=logging.INFO, format="%(message)s")
+        bulk_export_markdown(args.file, args.output_dir, redact=args.redact_secrets)
+        return
+
     with open(args.file, "rb") as file:
-        process_onenote_file(file, args.output_dir, args.extension, args.json)
+        process_onenote_file(file, args.output_dir, args.extension, args.json, redact=args.redact_secrets)
 
 
 if __name__ == "__main__":

pyOneNote/OneDocument.py

@@ -24,15 +24,28 @@ def get_properties(self):
         if self._properties:
             return self._properties
         nodes = []
-        filters = ['ObjectDeclaration2RefCountFND']
+        # ObjectDeclaration2LargeRefCountFND is used for objects whose
+        # reference count doesn't fit in one byte (e.g. more heavily-shared
+        # or larger page content), and the ReadOnly* variants wrap either
+        # declaration with an extra md5Hash. All four carry the same
+        # propertySet-bearing body and were previously excluded here, which
+        # silently dropped any text stored under them.
+        filters = [
+            'ObjectDeclaration2RefCountFND',
+            'ObjectDeclaration2LargeRefCountFND',
+            'ReadOnlyObjectDeclaration2RefCountFND',
+            'ReadOnlyObjectDeclaration2LargeRefCountFND',
+        ]
 
         self._properties = []
 
         OneDocment.traverse_nodes(self.root_file_node_list, nodes, filters)
         for node in nodes:
             if hasattr(node, 'propertySet'):
                 node.propertySet.body.indent= '\t\t'
-                self._properties.append({'type': str(node.data.body.jcid), 'identity':str(node.data.body.oid), 'val':node.propertySet.body.get_properties()})
+                # ReadOnly* variants wrap the declaration in `.base`.
+                decl_body = node.data.base.body if hasattr(node.data, 'base') else node.data.body
+                self._properties.append({'type': str(decl_body.jcid), 'identity':str(decl_body.oid), 'val':node.propertySet.body.get_properties()})
 
         return  self._properties
 
@@ -66,6 +79,106 @@ def get_files(self):
     def get_global_identification_table(self):
         return self._global_identification_table
 
+    # jcids that carry displayable text runs, in the order OneNote lays them out.
+    _TEXT_JCIDS = ('jcidRichTextOENode', 'jcidTitleNode', 'jcidNumberListNode')
+
+    @staticmethod
+    def _run_text(val):
+        # A single text run stores its content under one of these keys depending
+        # on platform: RichEditTextUnicode (Windows, UTF-16) or TextExtendedAscii
+        # (OneNote for Mac, latin-1). Strip embedded null bytes either way.
+        text = val.get('RichEditTextUnicode') or val.get('TextExtendedAscii') or ''
+        return text.replace('\x00', '').replace('\r\n', '\n').replace('\r', '\n').rstrip()
+
+    def _toc_titles(self):
+        # OneNote maintains a CachedTitleString for every page (used to render
+        # the section's page tabs). Collecting them gives the authoritative set
+        # of real page names, which we use to distinguish a page's title run
+        # from its body runs.
+        titles = set()
+        for prop in self.get_properties():
+            cached = prop['val'].get('CachedTitleString')
+            if cached:
+                cached = cached.replace('\x00', '').strip()
+                if cached:
+                    titles.add(cached)
+        return titles
+
+    def get_pages(self):
+        """Reconstruct de-duplicated pages from the property stream.
+
+        OneNote keeps every historical revision of a page in the same file, so
+        the raw stream repeats each page 2-20x. We segment on jcidPageNode, pick
+        the page's title as the first run matching a known CachedTitleString,
+        then key pages by that title. Revisions of the same page collapse into
+        one entry (identical runs de-duplicated), while genuinely distinct pages
+        that happen to share text are preserved because de-dup is scoped per
+        title. Pages are returned in first-seen order.
+        """
+        toc = self._toc_titles()
+
+        # segment the flat stream into per-revision run lists
+        raw_pages = []
+        current = None
+        for prop in self.get_properties():
+            ptype = prop['type']
+            if ptype == 'jcidPageNode':
+                if current is not None:
+                    raw_pages.append(current)
+                current = []
+                continue
+            if current is None:
+                continue
+            if ptype in self._TEXT_JCIDS:
+                text = self._run_text(prop['val'])
+                if text:
+                    current.append(text)
+        if current is not None:
+            raw_pages.append(current)
+
+        pages = []
+        by_title = {}
+        for runs in raw_pages:
+            if not runs:
+                continue
+            title = next((r for r in runs if r in toc), runs[0])
+            if title not in by_title:
+                entry = {'title': title, 'lines': [], '_seen': set()}
+                by_title[title] = entry
+                pages.append(entry)
+            entry = by_title[title]
+            for run in runs:
+                if run == title or run in entry['_seen']:
+                    continue
+                entry['_seen'].add(run)
+                entry['lines'].append(run)
+
+        for entry in pages:
+            del entry['_seen']
+        return pages
+
+    def get_markdown(self, section_title=None, redact=False):
+        """Render every unique page in this document as a single markdown string.
+
+        When redact is True, high-confidence credential shapes are replaced with
+        visible [redacted:*] markers (see pyOneNote.redact). Returns the markdown
+        string, or (string, Counter) of redaction counts when redact is True.
+        """
+        out = []
+        if section_title:
+            out.append('# {}\n'.format(section_title))
+        for page in self.get_pages():
+            title = page['title'].strip() or 'Untitled'
+            out.append('## {}\n'.format(title))
+            body = '\n'.join(page['lines']).strip()
+            if body:
+                out.append(body + '\n')
+        markdown = '\n'.join(out).rstrip() + '\n'
+        if redact:
+            from pyOneNote.redact import redact_secrets
+            return redact_secrets(markdown)
+        return markdown
+
     def get_json(self):
         files_in_hex = {}
         for key, file in self.get_files().items():