Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
tokenizaiton progress bar
HenryNdubuakubf380943bf27
diff --git a/src/data.py b/src/data.pyindex 0a895b6..fffca9f 100644--- a/src/data.py+++ b/src/data.py@@ -500,24 +500,24 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_chunk_size = max(1, len(enc_texts) // (num_workers * 4))enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]- print(f"Tokenizing encoder inputs ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,initargs=(model_path, max_enc_len)) as pool:- enc_results = pool.map(_tokenize_chunk, enc_chunks)+ enc_results = list(tqdm(pool.imap(_tokenize_chunk, enc_chunks),+ total=len(enc_chunks), desc="Tokenizing queries"))all_enc_tokens = [tok for chunk in enc_results for tok in chunk]tools_chunks = [tools_texts[i:i + chunk_size] for i in range(0, len(tools_texts), chunk_size)]- print(f"Tokenizing tools ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,initargs=(model_path, max_enc_len)) as pool:- tools_results = pool.map(_tokenize_chunk, tools_chunks)+ tools_results = list(tqdm(pool.imap(_tokenize_chunk, tools_chunks),+ total=len(tools_chunks), desc="Tokenizing tools"))all_tools_tokens = [tok for chunk in tools_results for tok in chunk]ans_chunks = [ans_texts[i:i + chunk_size] for i in range(0, len(ans_texts), chunk_size)]- print(f"Tokenizing answers ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,initargs=(model_path, max_dec_len)) as pool:- ans_results = pool.map(_tokenize_chunk, ans_chunks)+ ans_results = list(tqdm(pool.imap(_tokenize_chunk, ans_chunks),+ total=len(ans_chunks), desc="Tokenizing answers"))all_ans_tokens = [tok for chunk in ans_results for tok in chunk]n = len(ds)@@ -568,7 +568,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_loss_mask = np.zeros((n, max_dec_len), dtype=np.float32)skipped = 0- for i in range(n):+ for i in tqdm(range(n), desc="Building pairs"):if not _fill_sample(i, all_enc_tokens[i], all_tools_tokens[i],all_ans_tokens[i], ans_texts[i], enc_inputs,dec_inputs, dec_targets, loss_mask):