Skip to content

[DRAFT] Token level timestamps with DTW (#375) #1485

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Mar 20, 2024
Merged

[DRAFT] Token level timestamps with DTW (#375) #1485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
cd52c5a
whisper.cpp: impl dtw algo
denersc Nov 13, 2023
b69f5b4
WIP: producing and placing DTW timestamps on tokens
denersc Dec 4, 2023
f11ff92
Fix compile and assertion errors. Attempt to DTW timestamp with singl…
denersc Dec 7, 2023
93eb345
Fix mistake causing incorrect alignment of dtw timestamps
denersc Dec 7, 2023
3a5f368
implement N_TOP_MOST and CUSTOM alignment heads setting
denersc Dec 11, 2023
dfb24a4
whisper: fix typo on alignment heads enum
denersc Dec 14, 2023
4de1ed4
Fix issues related to changes in whisper.cpp
denersc Jan 11, 2024
641fb2c
Fixed excessive memory use when using DTW timestamps. Other minor fix…
denersc Mar 1, 2024
9a19200
decoder: save cross QKs only if requested
denersc Mar 5, 2024
3016444
Calling median filter with ggml_map_custom1
denersc Mar 5, 2024
eb531c7
Reimpl aheads n_top_most and custom. Sanity checks on chosen aheads
denersc Mar 6, 2024
87f2620
Copying cross QKs from decoder backend correctly
denersc Mar 7, 2024
10b0304
dtw: cleanup
denersc Mar 12, 2024
9fa298f
Fix incorrect n_frames passed to dtw when near end of audio
denersc Mar 14, 2024
5610253
Fix aheads_masks_init for backend != CPU
denersc Mar 14, 2024
d9bfe29
whisper : minor style
ggerganov Mar 18, 2024
c46712b
main : add dtw (wip)
ggerganov Mar 18, 2024
59c133a
whisper: fix invalid memory access in aheads_masks_init
denersc Mar 19, 2024
b55925d
main : add dtw (cont)
ggerganov Mar 20, 2024
a3c94f3
whisper : minor
ggerganov Mar 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 39 additions & 12 deletions examples/main/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,17 @@ void replace_all(std::string & s, const std::string & search, const std::string

// command-line parameters
struct whisper_params {
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
int32_t n_processors = 1;
int32_t offset_t_ms = 0;
int32_t offset_n = 0;
int32_t duration_ms = 0;
int32_t progress_step = 5;
int32_t max_context = -1;
int32_t max_len = 0;
int32_t best_of = whisper_full_default_params(WHISPER_SAMPLING_GREEDY).greedy.best_of;
int32_t beam_size = whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH).beam_search.beam_size;
int32_t audio_ctx = 0;
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
int32_t n_processors = 1;
int32_t offset_t_ms = 0;
int32_t offset_n = 0;
int32_t duration_ms = 0;
int32_t progress_step = 5;
int32_t max_context = -1;
int32_t max_len = 0;
int32_t best_of = whisper_full_default_params(WHISPER_SAMPLING_GREEDY).greedy.best_of;
int32_t beam_size = whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH).beam_search.beam_size;
int32_t audio_ctx = 0;

float word_thold = 0.01f;
float entropy_thold = 2.40f;
Expand Down Expand Up @@ -76,6 +76,8 @@ struct whisper_params {

std::string openvino_encode_device = "CPU";

std::string dtw = "";

std::vector<std::string> fname_inp = {};
std::vector<std::string> fname_out = {};
};
Expand Down Expand Up @@ -149,6 +151,7 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
else if (arg == "-m" || arg == "--model") { params.model = argv[++i]; }
else if (arg == "-f" || arg == "--file") { params.fname_inp.emplace_back(argv[++i]); }
else if (arg == "-oved" || arg == "--ov-e-device") { params.openvino_encode_device = argv[++i]; }
else if (arg == "-dtw" || arg == "--dtw") { params.dtw = argv[++i]; }
else if (arg == "-ls" || arg == "--log-score") { params.log_score = true; }
else if (arg == "-ng" || arg == "--no-gpu") { params.use_gpu = false; }
else {
Expand Down Expand Up @@ -208,6 +211,7 @@ void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & para
fprintf(stderr, " -m FNAME, --model FNAME [%-7s] model path\n", params.model.c_str());
fprintf(stderr, " -f FNAME, --file FNAME [%-7s] input WAV file path\n", "");
fprintf(stderr, " -oved D, --ov-e-device DNAME [%-7s] the OpenVINO device used for encode inference\n", params.openvino_encode_device.c_str());
fprintf(stderr, " -dtw MODEL --dtw MODEL [%-7s] compute token-level timestamps\n", params.dtw.c_str());
fprintf(stderr, " -ls, --log-score [%-7s] log best decoder scores of tokens\n", params.log_score?"true":"false");
fprintf(stderr, " -ng, --no-gpu [%-7s] disable GPU\n", params.use_gpu ? "false" : "true");
fprintf(stderr, "\n");
Expand Down Expand Up @@ -649,7 +653,8 @@ bool output_json(
times_o(token.t0, token.t1, false);
}
value_i("id", token.id, false);
value_f("p", token.p, true);
value_f("p", token.p, false);
value_f("t_dtw", token.t_dtw, true);
end_obj(j == (n - 1));
}
end_arr(!params.diarize && !params.tinydiarize);
Expand Down Expand Up @@ -889,6 +894,28 @@ int main(int argc, char ** argv) {
struct whisper_context_params cparams = whisper_context_default_params();
cparams.use_gpu = params.use_gpu;

if (!params.dtw.empty()) {
cparams.dtw_token_timestamps = true;
cparams.dtw_aheads_preset = WHISPER_AHEADS_NONE;

if (params.dtw == "tiny") cparams.dtw_aheads_preset = WHISPER_AHEADS_TINY;
if (params.dtw == "tiny.en") cparams.dtw_aheads_preset = WHISPER_AHEADS_TINY_EN;
if (params.dtw == "base") cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE;
if (params.dtw == "base.en") cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE_EN;
if (params.dtw == "small") cparams.dtw_aheads_preset = WHISPER_AHEADS_SMALL;
if (params.dtw == "small.en") cparams.dtw_aheads_preset = WHISPER_AHEADS_SMALL_EN;
if (params.dtw == "medium") cparams.dtw_aheads_preset = WHISPER_AHEADS_MEDIUM;
if (params.dtw == "medium.en") cparams.dtw_aheads_preset = WHISPER_AHEADS_MEDIUM_EN;
if (params.dtw == "large.v1") cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V1;
if (params.dtw == "large.v2") cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V2;
if (params.dtw == "large.v3") cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3;

if (cparams.dtw_aheads_preset == WHISPER_AHEADS_NONE) {
fprintf(stderr, "error: unknown DTW preset '%s'\n", params.dtw.c_str());
return 3;
}
}

struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);

if (ctx == nullptr) {
Expand Down
Loading