From 1bc7d89ce9c6eb7d60867416cb344300f14f9b8b Mon Sep 17 00:00:00 2001
From: Michael Treadgold <michael-treadgold@quietidiot.com>
Date: Thu, 18 Jun 2026 20:27:15 +1200
Subject: [PATCH] Add debug output to preprocessor to diagnose 0-row issue

---
 preprocess_dataset.py | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/preprocess_dataset.py b/preprocess_dataset.py
index 3d72fd7..92ba2c9 100644
--- a/preprocess_dataset.py
+++ b/preprocess_dataset.py
@@ -133,15 +133,25 @@ def process_hf_dataset(
 
     print(f"Loading HF dataset: {dataset_path}  subset={subset}  split={split}")
     if split:
-        ds = load_dataset(dataset_path, subset, split=split, trust_remote_code=True)
+        ds = load_dataset(dataset_path, subset, split=split)
     else:
-        ds_dict = load_dataset(dataset_path, subset, trust_remote_code=True)
+        ds_dict = load_dataset(dataset_path, subset)
         splits = list(ds_dict.keys())
         print(f"Available splits: {splits}")
-        # Use first train split, or first available
         preferred = [s for s in splits if "train" in s.lower()]
         ds = ds_dict[preferred[0] if preferred else splits[0]]
 
+    # Debug: print first row to see available columns
+    first = next(iter(ds))
+    print(f"Dataset columns: {list(first.keys())}")
+    for k, v in first.items():
+        if isinstance(v, dict):
+            print(f"  {k}: dict with keys {list(v.keys())}")
+        elif isinstance(v, str):
+            print(f"  {k}: str ({len(v)} chars) = {v[:80]!r}")
+        else:
+            print(f"  {k}: {type(v).__name__}")
+
     import tempfile
 
     mel_frontend = MelFrontend(HifiGanConfig(variant="v2plus"))
@@ -152,16 +162,25 @@ def process_hf_dataset(
     print(f"Audio tmp dir: {audio_tmp}")
 
     count = 0
+    skip_no_text = 0
+    skip_no_audio = 0
+    skip_bad_ids = 0
     try:
         with output_jsonl.open("w", encoding="utf-8") as f:
             for i, row in enumerate(ds):
                 text = str(row.get(text_key, "")).strip()
                 if not text:
+                    skip_no_text += 1
+                    if skip_no_text <= 3:
+                        print(f"  Row {i}: no text (keys={list(row.keys())[:5]})")
                     continue
 
                 # Get audio path or array
                 audio_info = row.get("audio", row.get("file", None))
                 if audio_info is None:
+                    skip_no_audio += 1
+                    if skip_no_audio <= 3:
+                        print(f"  Row {i}: no audio/file key (keys={list(row.keys())[:5]})")
                     continue
 
                 if isinstance(audio_info, dict):
@@ -184,7 +203,9 @@ def process_hf_dataset(
                 try:
                     phone_ids, tone_ids, lang_ids = text_to_ids(text)
                 except Exception as e:
-                    print(f"  Skipping row {i}: text-to-ids failed: {e}")
+                    skip_bad_ids += 1
+                    if skip_bad_ids <= 3:
+                        print(f"  Row {i}: text-to-ids failed: {e}  text={text[:60]!r}")
                     continue
 
                 if not phone_ids:
@@ -216,6 +237,7 @@ def process_hf_dataset(
         pass
 
     print(f"Wrote {count} rows to {output_jsonl}")
+    print(f"Skipped: no_text={skip_no_text}  no_audio={skip_no_audio}  bad_ids={skip_bad_ids}")
     return count