=== upload dir ===
total 7523476
drwxr-xr-x  4 root root       4096 Sep 11 00:37 .
drwxrwxrwx 12 root root       4096 Sep 10 16:02 ..
-rw-r--r--  1 root root      10398 Sep  9 19:32 _dd_err_fulltest.tar.gz
drwxr-xr-x  2 root root       4096 Sep  9 20:34 _dd_err_fulltest.tar.gz.x
-rw-r--r--  1 root root      10368 Sep  9 19:32 _dd_err_test.tar.gz
drwxr-xr-x  2 root root       4096 Sep  9 20:34 _dd_err_test.tar.gz.x
-rw-r--r--  1 root root          4 Sep  9 19:31 _test.txt
-rw-r--r--  1 root root  279747409 Sep 10 20:50 azul_pws_cobranded_dedup.jsonl.tar.gz
-rw-r--r--  1 root root  318546677 Sep 10 19:55 azul_pws_cobranded_dedup.tsv.tar.gz
-rw-r--r--  1 root root 1517147394 Sep 10 17:31 azul_sys_audit.jsonl.tar.gz
-rw-r--r--  1 root root 2837424574 Sep 10 14:33 azul_sys_email_log_shards.tar.gz
-rw-r--r--  1 root root  781714497 Sep  9 23:55 companydump__companies.tar.gz
-rw-r--r--  1 root root 1790340300 Sep  9 22:48 companydump__intel.tar.gz
-rw-r--r--  1 root root  179044352 Sep 11 00:52 <tool>.bak_20260831_031023.tar.gz.f68c0e1b.partial
=== indexfeed logs ===
total 700
drwxrwxrwx 2 root root   4096 Sep 13 09:26 .
drwxrwxrwx 9 root root   4096 Sep 10 13:56 ..
-rw-r--r-- 1 root root  11815 Sep 10 06:40 detect.log
-rwxrwxrwx 1 root root  94090 Sep  9 22:31 feed.log
-rw-r--r-- 1 root root  76703 Sep 10 05:52 ingest_loop.log
-rw-r--r-- 1 root root  31873 Sep 10 05:52 ingest_loop.out
-rw-r--r-- 1 root root   4984 Sep  9 20:40 parts.log
-rw-r--r-- 1 root root   4984 Sep  9 20:40 parts.out
-rwxrwxrwx 1 root root  25356 Sep  9 22:31 run_all.log
-rwxrwxrwx 1 root root      0 Sep  9 18:16 run_all.out
-rw-r--r-- 1 root root 282469 Sep 13 15:06 scan_loop.log
-rw-r--r-- 1 root root  34483 Sep 13 15:06 scan_loop.out
-rw-r--r-- 1 root root  95971 Sep 10 19:51 th_scan.log
-- /srv/data2/indexfeed/logs/detect.log (11815 bytes)
-- /srv/data2/indexfeed/logs/feed.log (94090 bytes)
-- /srv/data2/indexfeed/logs/ingest_loop.log (76703 bytes)
-- /srv/data2/indexfeed/logs/parts.log (4984 bytes)
-- /srv/data2/indexfeed/logs/run_all.log (25356 bytes)
-- /srv/data2/indexfeed/logs/scan_loop.log (282469 bytes)
-- /srv/data2/indexfeed/logs/th_scan.log (95971 bytes)
=== done-filter / dedupe secrets scripts ===
#!/usr/bin/env python3
"""dedupe_secrets.py — collapse secret_hits to one row per unique
(provider_type, detected_value), preserving every location in a new
secret_provenance table. Resets verification to 'unverified' (the previous
'verified' came from format-only detectors = not real validation).

Idempotent: safe to re-run.
"""
import sqlite3
import sys

DB = sys.argv[1] if len(sys.argv) > 1 else "/data/datacatalog/catalog.db"
c = sqlite3.connect(DB)
cur = c.cursor()

# 1. provenance table (all locations, duplicates kept for provenance)
cur.execute(
    """
    CREATE TABLE IF NOT EXISTS secret_provenance (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        provider_type TEXT,
        detected_value TEXT,
        source TEXT,
        file_path TEXT,
        context TEXT,
        seen_at TEXT DEFAULT (datetime('now'))
    )
    """
)
cur.execute("CREATE INDEX IF NOT EXISTS idx_prov_val ON secret_provenance(provider_type, detected_value)")
cur.execute("CREATE INDEX IF NOT EXISTS idx_prov_file ON secret_provenance(file_path)")

# 2. backfill provenance from current secret_hits (once)
have = cur.execute("SELECT COUNT(*) FROM secret_provenance").fetchone()[0]
if have == 0:
    cur.execute(
        """
        INSERT INTO secret_provenance (provider_type, detected_value, source, file_path, context)
        SELECT provider_type, detected_value, source, file_path, context FROM secret_hits
        """
=== ocr_batches naming ===
0.tar
10000.tar
10400.tar
10800.tar
11200.tar
153
DONE