remove stray comment (#1831)

sanderland · web-flow · commit c5eb93f4430e · 2025-08-29T11:08:59.000+02:00
diff --git a/tokenizers/src/models/bpe/trainer.rs b/tokenizers/src/models/bpe/trainer.rs
@@ -499,12 +499,9 @@ impl BpeTrainer {
                     part_b = rest;
                 }
             }
-            let new_token = format!("{part_a}{part_b}");
-            // implement sentencepiece-like merge.
-            // if this code were to be merged, integrate a way in the python bindings to communicate this variable
-            // default should be 0/None to maintain previous behavior. 16 is the spm default.
 
             // Insert new token if it does not already exist
+            let new_token = format!("{part_a}{part_b}");
             let new_token_id = word_to_id
                 .get(&CompactString::from(&new_token))
                 .copied()

Original file line number	Diff line number	Diff line change
`@@ -499,12 +499,9 @@ impl BpeTrainer {`
`499`	`499`	`part_b = rest;`
`500`	`500`	`}`
`501`	`501`	`}`
`502`		`- let new_token = format!("{part_a}{part_b}");`
`503`		`- // implement sentencepiece-like merge.`
`504`		`- // if this code were to be merged, integrate a way in the python bindings to communicate this variable`
`505`		`- // default should be 0/None to maintain previous behavior. 16 is the spm default.`
`506`	`502`
`507`	`503`	`// Insert new token if it does not already exist`
	`504`	`+ let new_token = format!("{part_a}{part_b}");`
`508`	`505`	`let new_token_id = word_to_id`
`509`	`506`	`.get(&CompactString::from(&new_token))`
`510`	`507`	`.copied()`